1 /* 2 * linux/mm/memory.c 3 * 4 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 5 */ 6 7 /* 8 * demand-loading started 01.12.91 - seems it is high on the list of 9 * things wanted, and it should be easy to implement. - Linus 10 */ 11 12 /* 13 * Ok, demand-loading was easy, shared pages a little bit tricker. Shared 14 * pages started 02.12.91, seems to work. - Linus. 15 * 16 * Tested sharing by executing about 30 /bin/sh: under the old kernel it 17 * would have taken more than the 6M I have free, but it worked well as 18 * far as I could see. 19 * 20 * Also corrected some "invalidate()"s - I wasn't doing enough of them. 21 */ 22 23 /* 24 * Real VM (paging to/from disk) started 18.12.91. Much more work and 25 * thought has to go into this. Oh, well.. 26 * 19.12.91 - works, somewhat. Sometimes I get faults, don't know why. 27 * Found it. Everything seems to work now. 28 * 20.12.91 - Ok, making the swap-device changeable like the root. 29 */ 30 31 /* 32 * 05.04.94 - Multi-page memory management added for v1.1. 33 * Idea by Alex Bligh (alex@cconcepts.co.uk) 34 * 35 * 16.07.99 - Support of BIGMEM added by Gerhard Wichert, Siemens AG 36 * (Gerhard.Wichert@pdb.siemens.de) 37 * 38 * Aug/Sep 2004 Changed to four level page tables (Andi Kleen) 39 */ 40 41 #include <linux/kernel_stat.h> 42 #include <linux/mm.h> 43 #include <linux/hugetlb.h> 44 #include <linux/mman.h> 45 #include <linux/swap.h> 46 #include <linux/highmem.h> 47 #include <linux/pagemap.h> 48 #include <linux/ksm.h> 49 #include <linux/rmap.h> 50 #include <linux/export.h> 51 #include <linux/delayacct.h> 52 #include <linux/init.h> 53 #include <linux/writeback.h> 54 #include <linux/memcontrol.h> 55 #include <linux/mmu_notifier.h> 56 #include <linux/kallsyms.h> 57 #include <linux/swapops.h> 58 #include <linux/elf.h> 59 #include <linux/gfp.h> 60 #include <linux/migrate.h> 61 #include <linux/string.h> 62 63 #include <asm/io.h> 64 #include <asm/pgalloc.h> 65 #include <asm/uaccess.h> 66 #include <asm/tlb.h> 67 #include <asm/tlbflush.h> 68 #include <asm/pgtable.h> 69 70 #include "internal.h" 71 72 #ifdef LAST_NID_NOT_IN_PAGE_FLAGS 73 #warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_nid. 74 #endif 75 76 #ifndef CONFIG_NEED_MULTIPLE_NODES 77 /* use the per-pgdat data instead for discontigmem - mbligh */ 78 unsigned long max_mapnr; 79 struct page *mem_map; 80 81 EXPORT_SYMBOL(max_mapnr); 82 EXPORT_SYMBOL(mem_map); 83 #endif 84 85 unsigned long num_physpages; 86 /* 87 * A number of key systems in x86 including ioremap() rely on the assumption 88 * that high_memory defines the upper bound on direct map memory, then end 89 * of ZONE_NORMAL. Under CONFIG_DISCONTIG this means that max_low_pfn and 90 * highstart_pfn must be the same; there must be no gap between ZONE_NORMAL 91 * and ZONE_HIGHMEM. 92 */ 93 void * high_memory; 94 95 EXPORT_SYMBOL(num_physpages); 96 EXPORT_SYMBOL(high_memory); 97 98 /* 99 * Randomize the address space (stacks, mmaps, brk, etc.). 100 * 101 * ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization, 102 * as ancient (libc5 based) binaries can segfault. ) 103 */ 104 int randomize_va_space __read_mostly = 105 #ifdef CONFIG_COMPAT_BRK 106 1; 107 #else 108 2; 109 #endif 110 111 static int __init disable_randmaps(char *s) 112 { 113 randomize_va_space = 0; 114 return 1; 115 } 116 __setup("norandmaps", disable_randmaps); 117 118 unsigned long zero_pfn __read_mostly; 119 unsigned long highest_memmap_pfn __read_mostly; 120 121 /* 122 * CONFIG_MMU architectures set up ZERO_PAGE in their paging_init() 123 */ 124 static int __init init_zero_pfn(void) 125 { 126 zero_pfn = page_to_pfn(ZERO_PAGE(0)); 127 return 0; 128 } 129 core_initcall(init_zero_pfn); 130 131 132 #if defined(SPLIT_RSS_COUNTING) 133 134 void sync_mm_rss(struct mm_struct *mm) 135 { 136 int i; 137 138 for (i = 0; i < NR_MM_COUNTERS; i++) { 139 if (current->rss_stat.count[i]) { 140 add_mm_counter(mm, i, current->rss_stat.count[i]); 141 current->rss_stat.count[i] = 0; 142 } 143 } 144 current->rss_stat.events = 0; 145 } 146 147 static void add_mm_counter_fast(struct mm_struct *mm, int member, int val) 148 { 149 struct task_struct *task = current; 150 151 if (likely(task->mm == mm)) 152 task->rss_stat.count[member] += val; 153 else 154 add_mm_counter(mm, member, val); 155 } 156 #define inc_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, 1) 157 #define dec_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, -1) 158 159 /* sync counter once per 64 page faults */ 160 #define TASK_RSS_EVENTS_THRESH (64) 161 static void check_sync_rss_stat(struct task_struct *task) 162 { 163 if (unlikely(task != current)) 164 return; 165 if (unlikely(task->rss_stat.events++ > TASK_RSS_EVENTS_THRESH)) 166 sync_mm_rss(task->mm); 167 } 168 #else /* SPLIT_RSS_COUNTING */ 169 170 #define inc_mm_counter_fast(mm, member) inc_mm_counter(mm, member) 171 #define dec_mm_counter_fast(mm, member) dec_mm_counter(mm, member) 172 173 static void check_sync_rss_stat(struct task_struct *task) 174 { 175 } 176 177 #endif /* SPLIT_RSS_COUNTING */ 178 179 #ifdef HAVE_GENERIC_MMU_GATHER 180 181 static int tlb_next_batch(struct mmu_gather *tlb) 182 { 183 struct mmu_gather_batch *batch; 184 185 batch = tlb->active; 186 if (batch->next) { 187 tlb->active = batch->next; 188 return 1; 189 } 190 191 if (tlb->batch_count == MAX_GATHER_BATCH_COUNT) 192 return 0; 193 194 batch = (void *)__get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0); 195 if (!batch) 196 return 0; 197 198 tlb->batch_count++; 199 batch->next = NULL; 200 batch->nr = 0; 201 batch->max = MAX_GATHER_BATCH; 202 203 tlb->active->next = batch; 204 tlb->active = batch; 205 206 return 1; 207 } 208 209 /* tlb_gather_mmu 210 * Called to initialize an (on-stack) mmu_gather structure for page-table 211 * tear-down from @mm. The @fullmm argument is used when @mm is without 212 * users and we're going to destroy the full address space (exit/execve). 213 */ 214 void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm) 215 { 216 tlb->mm = mm; 217 218 tlb->fullmm = fullmm; 219 tlb->start = -1UL; 220 tlb->end = 0; 221 tlb->need_flush = 0; 222 tlb->fast_mode = (num_possible_cpus() == 1); 223 tlb->local.next = NULL; 224 tlb->local.nr = 0; 225 tlb->local.max = ARRAY_SIZE(tlb->__pages); 226 tlb->active = &tlb->local; 227 tlb->batch_count = 0; 228 229 #ifdef CONFIG_HAVE_RCU_TABLE_FREE 230 tlb->batch = NULL; 231 #endif 232 } 233 234 void tlb_flush_mmu(struct mmu_gather *tlb) 235 { 236 struct mmu_gather_batch *batch; 237 238 if (!tlb->need_flush) 239 return; 240 tlb->need_flush = 0; 241 tlb_flush(tlb); 242 #ifdef CONFIG_HAVE_RCU_TABLE_FREE 243 tlb_table_flush(tlb); 244 #endif 245 246 if (tlb_fast_mode(tlb)) 247 return; 248 249 for (batch = &tlb->local; batch; batch = batch->next) { 250 free_pages_and_swap_cache(batch->pages, batch->nr); 251 batch->nr = 0; 252 } 253 tlb->active = &tlb->local; 254 } 255 256 /* tlb_finish_mmu 257 * Called at the end of the shootdown operation to free up any resources 258 * that were required. 259 */ 260 void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) 261 { 262 struct mmu_gather_batch *batch, *next; 263 264 tlb->start = start; 265 tlb->end = end; 266 tlb_flush_mmu(tlb); 267 268 /* keep the page table cache within bounds */ 269 check_pgt_cache(); 270 271 for (batch = tlb->local.next; batch; batch = next) { 272 next = batch->next; 273 free_pages((unsigned long)batch, 0); 274 } 275 tlb->local.next = NULL; 276 } 277 278 /* __tlb_remove_page 279 * Must perform the equivalent to __free_pte(pte_get_and_clear(ptep)), while 280 * handling the additional races in SMP caused by other CPUs caching valid 281 * mappings in their TLBs. Returns the number of free page slots left. 282 * When out of page slots we must call tlb_flush_mmu(). 283 */ 284 int __tlb_remove_page(struct mmu_gather *tlb, struct page *page) 285 { 286 struct mmu_gather_batch *batch; 287 288 VM_BUG_ON(!tlb->need_flush); 289 290 if (tlb_fast_mode(tlb)) { 291 free_page_and_swap_cache(page); 292 return 1; /* avoid calling tlb_flush_mmu() */ 293 } 294 295 batch = tlb->active; 296 batch->pages[batch->nr++] = page; 297 if (batch->nr == batch->max) { 298 if (!tlb_next_batch(tlb)) 299 return 0; 300 batch = tlb->active; 301 } 302 VM_BUG_ON(batch->nr > batch->max); 303 304 return batch->max - batch->nr; 305 } 306 307 #endif /* HAVE_GENERIC_MMU_GATHER */ 308 309 #ifdef CONFIG_HAVE_RCU_TABLE_FREE 310 311 /* 312 * See the comment near struct mmu_table_batch. 313 */ 314 315 static void tlb_remove_table_smp_sync(void *arg) 316 { 317 /* Simply deliver the interrupt */ 318 } 319 320 static void tlb_remove_table_one(void *table) 321 { 322 /* 323 * This isn't an RCU grace period and hence the page-tables cannot be 324 * assumed to be actually RCU-freed. 325 * 326 * It is however sufficient for software page-table walkers that rely on 327 * IRQ disabling. See the comment near struct mmu_table_batch. 328 */ 329 smp_call_function(tlb_remove_table_smp_sync, NULL, 1); 330 __tlb_remove_table(table); 331 } 332 333 static void tlb_remove_table_rcu(struct rcu_head *head) 334 { 335 struct mmu_table_batch *batch; 336 int i; 337 338 batch = container_of(head, struct mmu_table_batch, rcu); 339 340 for (i = 0; i < batch->nr; i++) 341 __tlb_remove_table(batch->tables[i]); 342 343 free_page((unsigned long)batch); 344 } 345 346 void tlb_table_flush(struct mmu_gather *tlb) 347 { 348 struct mmu_table_batch **batch = &tlb->batch; 349 350 if (*batch) { 351 call_rcu_sched(&(*batch)->rcu, tlb_remove_table_rcu); 352 *batch = NULL; 353 } 354 } 355 356 void tlb_remove_table(struct mmu_gather *tlb, void *table) 357 { 358 struct mmu_table_batch **batch = &tlb->batch; 359 360 tlb->need_flush = 1; 361 362 /* 363 * When there's less then two users of this mm there cannot be a 364 * concurrent page-table walk. 365 */ 366 if (atomic_read(&tlb->mm->mm_users) < 2) { 367 __tlb_remove_table(table); 368 return; 369 } 370 371 if (*batch == NULL) { 372 *batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN); 373 if (*batch == NULL) { 374 tlb_remove_table_one(table); 375 return; 376 } 377 (*batch)->nr = 0; 378 } 379 (*batch)->tables[(*batch)->nr++] = table; 380 if ((*batch)->nr == MAX_TABLE_BATCH) 381 tlb_table_flush(tlb); 382 } 383 384 #endif /* CONFIG_HAVE_RCU_TABLE_FREE */ 385 386 /* 387 * If a p?d_bad entry is found while walking page tables, report 388 * the error, before resetting entry to p?d_none. Usually (but 389 * very seldom) called out from the p?d_none_or_clear_bad macros. 390 */ 391 392 void pgd_clear_bad(pgd_t *pgd) 393 { 394 pgd_ERROR(*pgd); 395 pgd_clear(pgd); 396 } 397 398 void pud_clear_bad(pud_t *pud) 399 { 400 pud_ERROR(*pud); 401 pud_clear(pud); 402 } 403 404 void pmd_clear_bad(pmd_t *pmd) 405 { 406 pmd_ERROR(*pmd); 407 pmd_clear(pmd); 408 } 409 410 /* 411 * Note: this doesn't free the actual pages themselves. That 412 * has been handled earlier when unmapping all the memory regions. 413 */ 414 static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd, 415 unsigned long addr) 416 { 417 pgtable_t token = pmd_pgtable(*pmd); 418 pmd_clear(pmd); 419 pte_free_tlb(tlb, token, addr); 420 tlb->mm->nr_ptes--; 421 } 422 423 static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud, 424 unsigned long addr, unsigned long end, 425 unsigned long floor, unsigned long ceiling) 426 { 427 pmd_t *pmd; 428 unsigned long next; 429 unsigned long start; 430 431 start = addr; 432 pmd = pmd_offset(pud, addr); 433 do { 434 next = pmd_addr_end(addr, end); 435 if (pmd_none_or_clear_bad(pmd)) 436 continue; 437 free_pte_range(tlb, pmd, addr); 438 } while (pmd++, addr = next, addr != end); 439 440 start &= PUD_MASK; 441 if (start < floor) 442 return; 443 if (ceiling) { 444 ceiling &= PUD_MASK; 445 if (!ceiling) 446 return; 447 } 448 if (end - 1 > ceiling - 1) 449 return; 450 451 pmd = pmd_offset(pud, start); 452 pud_clear(pud); 453 pmd_free_tlb(tlb, pmd, start); 454 } 455 456 static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, 457 unsigned long addr, unsigned long end, 458 unsigned long floor, unsigned long ceiling) 459 { 460 pud_t *pud; 461 unsigned long next; 462 unsigned long start; 463 464 start = addr; 465 pud = pud_offset(pgd, addr); 466 do { 467 next = pud_addr_end(addr, end); 468 if (pud_none_or_clear_bad(pud)) 469 continue; 470 free_pmd_range(tlb, pud, addr, next, floor, ceiling); 471 } while (pud++, addr = next, addr != end); 472 473 start &= PGDIR_MASK; 474 if (start < floor) 475 return; 476 if (ceiling) { 477 ceiling &= PGDIR_MASK; 478 if (!ceiling) 479 return; 480 } 481 if (end - 1 > ceiling - 1) 482 return; 483 484 pud = pud_offset(pgd, start); 485 pgd_clear(pgd); 486 pud_free_tlb(tlb, pud, start); 487 } 488 489 /* 490 * This function frees user-level page tables of a process. 491 * 492 * Must be called with pagetable lock held. 493 */ 494 void free_pgd_range(struct mmu_gather *tlb, 495 unsigned long addr, unsigned long end, 496 unsigned long floor, unsigned long ceiling) 497 { 498 pgd_t *pgd; 499 unsigned long next; 500 501 /* 502 * The next few lines have given us lots of grief... 503 * 504 * Why are we testing PMD* at this top level? Because often 505 * there will be no work to do at all, and we'd prefer not to 506 * go all the way down to the bottom just to discover that. 507 * 508 * Why all these "- 1"s? Because 0 represents both the bottom 509 * of the address space and the top of it (using -1 for the 510 * top wouldn't help much: the masks would do the wrong thing). 511 * The rule is that addr 0 and floor 0 refer to the bottom of 512 * the address space, but end 0 and ceiling 0 refer to the top 513 * Comparisons need to use "end - 1" and "ceiling - 1" (though 514 * that end 0 case should be mythical). 515 * 516 * Wherever addr is brought up or ceiling brought down, we must 517 * be careful to reject "the opposite 0" before it confuses the 518 * subsequent tests. But what about where end is brought down 519 * by PMD_SIZE below? no, end can't go down to 0 there. 520 * 521 * Whereas we round start (addr) and ceiling down, by different 522 * masks at different levels, in order to test whether a table 523 * now has no other vmas using it, so can be freed, we don't 524 * bother to round floor or end up - the tests don't need that. 525 */ 526 527 addr &= PMD_MASK; 528 if (addr < floor) { 529 addr += PMD_SIZE; 530 if (!addr) 531 return; 532 } 533 if (ceiling) { 534 ceiling &= PMD_MASK; 535 if (!ceiling) 536 return; 537 } 538 if (end - 1 > ceiling - 1) 539 end -= PMD_SIZE; 540 if (addr > end - 1) 541 return; 542 543 pgd = pgd_offset(tlb->mm, addr); 544 do { 545 next = pgd_addr_end(addr, end); 546 if (pgd_none_or_clear_bad(pgd)) 547 continue; 548 free_pud_range(tlb, pgd, addr, next, floor, ceiling); 549 } while (pgd++, addr = next, addr != end); 550 } 551 552 void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, 553 unsigned long floor, unsigned long ceiling) 554 { 555 while (vma) { 556 struct vm_area_struct *next = vma->vm_next; 557 unsigned long addr = vma->vm_start; 558 559 /* 560 * Hide vma from rmap and truncate_pagecache before freeing 561 * pgtables 562 */ 563 unlink_anon_vmas(vma); 564 unlink_file_vma(vma); 565 566 if (is_vm_hugetlb_page(vma)) { 567 hugetlb_free_pgd_range(tlb, addr, vma->vm_end, 568 floor, next? next->vm_start: ceiling); 569 } else { 570 /* 571 * Optimization: gather nearby vmas into one call down 572 */ 573 while (next && next->vm_start <= vma->vm_end + PMD_SIZE 574 && !is_vm_hugetlb_page(next)) { 575 vma = next; 576 next = vma->vm_next; 577 unlink_anon_vmas(vma); 578 unlink_file_vma(vma); 579 } 580 free_pgd_range(tlb, addr, vma->vm_end, 581 floor, next? next->vm_start: ceiling); 582 } 583 vma = next; 584 } 585 } 586 587 int __pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma, 588 pmd_t *pmd, unsigned long address) 589 { 590 pgtable_t new = pte_alloc_one(mm, address); 591 int wait_split_huge_page; 592 if (!new) 593 return -ENOMEM; 594 595 /* 596 * Ensure all pte setup (eg. pte page lock and page clearing) are 597 * visible before the pte is made visible to other CPUs by being 598 * put into page tables. 599 * 600 * The other side of the story is the pointer chasing in the page 601 * table walking code (when walking the page table without locking; 602 * ie. most of the time). Fortunately, these data accesses consist 603 * of a chain of data-dependent loads, meaning most CPUs (alpha 604 * being the notable exception) will already guarantee loads are 605 * seen in-order. See the alpha page table accessors for the 606 * smp_read_barrier_depends() barriers in page table walking code. 607 */ 608 smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */ 609 610 spin_lock(&mm->page_table_lock); 611 wait_split_huge_page = 0; 612 if (likely(pmd_none(*pmd))) { /* Has another populated it ? */ 613 mm->nr_ptes++; 614 pmd_populate(mm, pmd, new); 615 new = NULL; 616 } else if (unlikely(pmd_trans_splitting(*pmd))) 617 wait_split_huge_page = 1; 618 spin_unlock(&mm->page_table_lock); 619 if (new) 620 pte_free(mm, new); 621 if (wait_split_huge_page) 622 wait_split_huge_page(vma->anon_vma, pmd); 623 return 0; 624 } 625 626 int __pte_alloc_kernel(pmd_t *pmd, unsigned long address) 627 { 628 pte_t *new = pte_alloc_one_kernel(&init_mm, address); 629 if (!new) 630 return -ENOMEM; 631 632 smp_wmb(); /* See comment in __pte_alloc */ 633 634 spin_lock(&init_mm.page_table_lock); 635 if (likely(pmd_none(*pmd))) { /* Has another populated it ? */ 636 pmd_populate_kernel(&init_mm, pmd, new); 637 new = NULL; 638 } else 639 VM_BUG_ON(pmd_trans_splitting(*pmd)); 640 spin_unlock(&init_mm.page_table_lock); 641 if (new) 642 pte_free_kernel(&init_mm, new); 643 return 0; 644 } 645 646 static inline void init_rss_vec(int *rss) 647 { 648 memset(rss, 0, sizeof(int) * NR_MM_COUNTERS); 649 } 650 651 static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss) 652 { 653 int i; 654 655 if (current->mm == mm) 656 sync_mm_rss(mm); 657 for (i = 0; i < NR_MM_COUNTERS; i++) 658 if (rss[i]) 659 add_mm_counter(mm, i, rss[i]); 660 } 661 662 /* 663 * This function is called to print an error when a bad pte 664 * is found. For example, we might have a PFN-mapped pte in 665 * a region that doesn't allow it. 666 * 667 * The calling function must still handle the error. 668 */ 669 static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr, 670 pte_t pte, struct page *page) 671 { 672 pgd_t *pgd = pgd_offset(vma->vm_mm, addr); 673 pud_t *pud = pud_offset(pgd, addr); 674 pmd_t *pmd = pmd_offset(pud, addr); 675 struct address_space *mapping; 676 pgoff_t index; 677 static unsigned long resume; 678 static unsigned long nr_shown; 679 static unsigned long nr_unshown; 680 681 /* 682 * Allow a burst of 60 reports, then keep quiet for that minute; 683 * or allow a steady drip of one report per second. 684 */ 685 if (nr_shown == 60) { 686 if (time_before(jiffies, resume)) { 687 nr_unshown++; 688 return; 689 } 690 if (nr_unshown) { 691 printk(KERN_ALERT 692 "BUG: Bad page map: %lu messages suppressed\n", 693 nr_unshown); 694 nr_unshown = 0; 695 } 696 nr_shown = 0; 697 } 698 if (nr_shown++ == 0) 699 resume = jiffies + 60 * HZ; 700 701 mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL; 702 index = linear_page_index(vma, addr); 703 704 printk(KERN_ALERT 705 "BUG: Bad page map in process %s pte:%08llx pmd:%08llx\n", 706 current->comm, 707 (long long)pte_val(pte), (long long)pmd_val(*pmd)); 708 if (page) 709 dump_page(page); 710 printk(KERN_ALERT 711 "addr:%p vm_flags:%08lx anon_vma:%p mapping:%p index:%lx\n", 712 (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index); 713 /* 714 * Choose text because data symbols depend on CONFIG_KALLSYMS_ALL=y 715 */ 716 if (vma->vm_ops) 717 print_symbol(KERN_ALERT "vma->vm_ops->fault: %s\n", 718 (unsigned long)vma->vm_ops->fault); 719 if (vma->vm_file && vma->vm_file->f_op) 720 print_symbol(KERN_ALERT "vma->vm_file->f_op->mmap: %s\n", 721 (unsigned long)vma->vm_file->f_op->mmap); 722 dump_stack(); 723 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 724 } 725 726 static inline bool is_cow_mapping(vm_flags_t flags) 727 { 728 return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE; 729 } 730 731 /* 732 * vm_normal_page -- This function gets the "struct page" associated with a pte. 733 * 734 * "Special" mappings do not wish to be associated with a "struct page" (either 735 * it doesn't exist, or it exists but they don't want to touch it). In this 736 * case, NULL is returned here. "Normal" mappings do have a struct page. 737 * 738 * There are 2 broad cases. Firstly, an architecture may define a pte_special() 739 * pte bit, in which case this function is trivial. Secondly, an architecture 740 * may not have a spare pte bit, which requires a more complicated scheme, 741 * described below. 742 * 743 * A raw VM_PFNMAP mapping (ie. one that is not COWed) is always considered a 744 * special mapping (even if there are underlying and valid "struct pages"). 745 * COWed pages of a VM_PFNMAP are always normal. 746 * 747 * The way we recognize COWed pages within VM_PFNMAP mappings is through the 748 * rules set up by "remap_pfn_range()": the vma will have the VM_PFNMAP bit 749 * set, and the vm_pgoff will point to the first PFN mapped: thus every special 750 * mapping will always honor the rule 751 * 752 * pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT) 753 * 754 * And for normal mappings this is false. 755 * 756 * This restricts such mappings to be a linear translation from virtual address 757 * to pfn. To get around this restriction, we allow arbitrary mappings so long 758 * as the vma is not a COW mapping; in that case, we know that all ptes are 759 * special (because none can have been COWed). 760 * 761 * 762 * In order to support COW of arbitrary special mappings, we have VM_MIXEDMAP. 763 * 764 * VM_MIXEDMAP mappings can likewise contain memory with or without "struct 765 * page" backing, however the difference is that _all_ pages with a struct 766 * page (that is, those where pfn_valid is true) are refcounted and considered 767 * normal pages by the VM. The disadvantage is that pages are refcounted 768 * (which can be slower and simply not an option for some PFNMAP users). The 769 * advantage is that we don't have to follow the strict linearity rule of 770 * PFNMAP mappings in order to support COWable mappings. 771 * 772 */ 773 #ifdef __HAVE_ARCH_PTE_SPECIAL 774 # define HAVE_PTE_SPECIAL 1 775 #else 776 # define HAVE_PTE_SPECIAL 0 777 #endif 778 struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr, 779 pte_t pte) 780 { 781 unsigned long pfn = pte_pfn(pte); 782 783 if (HAVE_PTE_SPECIAL) { 784 if (likely(!pte_special(pte))) 785 goto check_pfn; 786 if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)) 787 return NULL; 788 if (!is_zero_pfn(pfn)) 789 print_bad_pte(vma, addr, pte, NULL); 790 return NULL; 791 } 792 793 /* !HAVE_PTE_SPECIAL case follows: */ 794 795 if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) { 796 if (vma->vm_flags & VM_MIXEDMAP) { 797 if (!pfn_valid(pfn)) 798 return NULL; 799 goto out; 800 } else { 801 unsigned long off; 802 off = (addr - vma->vm_start) >> PAGE_SHIFT; 803 if (pfn == vma->vm_pgoff + off) 804 return NULL; 805 if (!is_cow_mapping(vma->vm_flags)) 806 return NULL; 807 } 808 } 809 810 if (is_zero_pfn(pfn)) 811 return NULL; 812 check_pfn: 813 if (unlikely(pfn > highest_memmap_pfn)) { 814 print_bad_pte(vma, addr, pte, NULL); 815 return NULL; 816 } 817 818 /* 819 * NOTE! We still have PageReserved() pages in the page tables. 820 * eg. VDSO mappings can cause them to exist. 821 */ 822 out: 823 return pfn_to_page(pfn); 824 } 825 826 /* 827 * copy one vm_area from one task to the other. Assumes the page tables 828 * already present in the new task to be cleared in the whole range 829 * covered by this vma. 830 */ 831 832 static inline unsigned long 833 copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, 834 pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma, 835 unsigned long addr, int *rss) 836 { 837 unsigned long vm_flags = vma->vm_flags; 838 pte_t pte = *src_pte; 839 struct page *page; 840 841 /* pte contains position in swap or file, so copy. */ 842 if (unlikely(!pte_present(pte))) { 843 if (!pte_file(pte)) { 844 swp_entry_t entry = pte_to_swp_entry(pte); 845 846 if (swap_duplicate(entry) < 0) 847 return entry.val; 848 849 /* make sure dst_mm is on swapoff's mmlist. */ 850 if (unlikely(list_empty(&dst_mm->mmlist))) { 851 spin_lock(&mmlist_lock); 852 if (list_empty(&dst_mm->mmlist)) 853 list_add(&dst_mm->mmlist, 854 &src_mm->mmlist); 855 spin_unlock(&mmlist_lock); 856 } 857 if (likely(!non_swap_entry(entry))) 858 rss[MM_SWAPENTS]++; 859 else if (is_migration_entry(entry)) { 860 page = migration_entry_to_page(entry); 861 862 if (PageAnon(page)) 863 rss[MM_ANONPAGES]++; 864 else 865 rss[MM_FILEPAGES]++; 866 867 if (is_write_migration_entry(entry) && 868 is_cow_mapping(vm_flags)) { 869 /* 870 * COW mappings require pages in both 871 * parent and child to be set to read. 872 */ 873 make_migration_entry_read(&entry); 874 pte = swp_entry_to_pte(entry); 875 set_pte_at(src_mm, addr, src_pte, pte); 876 } 877 } 878 } 879 goto out_set_pte; 880 } 881 882 /* 883 * If it's a COW mapping, write protect it both 884 * in the parent and the child 885 */ 886 if (is_cow_mapping(vm_flags)) { 887 ptep_set_wrprotect(src_mm, addr, src_pte); 888 pte = pte_wrprotect(pte); 889 } 890 891 /* 892 * If it's a shared mapping, mark it clean in 893 * the child 894 */ 895 if (vm_flags & VM_SHARED) 896 pte = pte_mkclean(pte); 897 pte = pte_mkold(pte); 898 899 page = vm_normal_page(vma, addr, pte); 900 if (page) { 901 get_page(page); 902 page_dup_rmap(page); 903 if (PageAnon(page)) 904 rss[MM_ANONPAGES]++; 905 else 906 rss[MM_FILEPAGES]++; 907 } 908 909 out_set_pte: 910 set_pte_at(dst_mm, addr, dst_pte, pte); 911 return 0; 912 } 913 914 int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 915 pmd_t *dst_pmd, pmd_t *src_pmd, struct vm_area_struct *vma, 916 unsigned long addr, unsigned long end) 917 { 918 pte_t *orig_src_pte, *orig_dst_pte; 919 pte_t *src_pte, *dst_pte; 920 spinlock_t *src_ptl, *dst_ptl; 921 int progress = 0; 922 int rss[NR_MM_COUNTERS]; 923 swp_entry_t entry = (swp_entry_t){0}; 924 925 again: 926 init_rss_vec(rss); 927 928 dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl); 929 if (!dst_pte) 930 return -ENOMEM; 931 src_pte = pte_offset_map(src_pmd, addr); 932 src_ptl = pte_lockptr(src_mm, src_pmd); 933 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 934 orig_src_pte = src_pte; 935 orig_dst_pte = dst_pte; 936 arch_enter_lazy_mmu_mode(); 937 938 do { 939 /* 940 * We are holding two locks at this point - either of them 941 * could generate latencies in another task on another CPU. 942 */ 943 if (progress >= 32) { 944 progress = 0; 945 if (need_resched() || 946 spin_needbreak(src_ptl) || spin_needbreak(dst_ptl)) 947 break; 948 } 949 if (pte_none(*src_pte)) { 950 progress++; 951 continue; 952 } 953 entry.val = copy_one_pte(dst_mm, src_mm, dst_pte, src_pte, 954 vma, addr, rss); 955 if (entry.val) 956 break; 957 progress += 8; 958 } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end); 959 960 arch_leave_lazy_mmu_mode(); 961 spin_unlock(src_ptl); 962 pte_unmap(orig_src_pte); 963 add_mm_rss_vec(dst_mm, rss); 964 pte_unmap_unlock(orig_dst_pte, dst_ptl); 965 cond_resched(); 966 967 if (entry.val) { 968 if (add_swap_count_continuation(entry, GFP_KERNEL) < 0) 969 return -ENOMEM; 970 progress = 0; 971 } 972 if (addr != end) 973 goto again; 974 return 0; 975 } 976 977 static inline int copy_pmd_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 978 pud_t *dst_pud, pud_t *src_pud, struct vm_area_struct *vma, 979 unsigned long addr, unsigned long end) 980 { 981 pmd_t *src_pmd, *dst_pmd; 982 unsigned long next; 983 984 dst_pmd = pmd_alloc(dst_mm, dst_pud, addr); 985 if (!dst_pmd) 986 return -ENOMEM; 987 src_pmd = pmd_offset(src_pud, addr); 988 do { 989 next = pmd_addr_end(addr, end); 990 if (pmd_trans_huge(*src_pmd)) { 991 int err; 992 VM_BUG_ON(next-addr != HPAGE_PMD_SIZE); 993 err = copy_huge_pmd(dst_mm, src_mm, 994 dst_pmd, src_pmd, addr, vma); 995 if (err == -ENOMEM) 996 return -ENOMEM; 997 if (!err) 998 continue; 999 /* fall through */ 1000 } 1001 if (pmd_none_or_clear_bad(src_pmd)) 1002 continue; 1003 if (copy_pte_range(dst_mm, src_mm, dst_pmd, src_pmd, 1004 vma, addr, next)) 1005 return -ENOMEM; 1006 } while (dst_pmd++, src_pmd++, addr = next, addr != end); 1007 return 0; 1008 } 1009 1010 static inline int copy_pud_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 1011 pgd_t *dst_pgd, pgd_t *src_pgd, struct vm_area_struct *vma, 1012 unsigned long addr, unsigned long end) 1013 { 1014 pud_t *src_pud, *dst_pud; 1015 unsigned long next; 1016 1017 dst_pud = pud_alloc(dst_mm, dst_pgd, addr); 1018 if (!dst_pud) 1019 return -ENOMEM; 1020 src_pud = pud_offset(src_pgd, addr); 1021 do { 1022 next = pud_addr_end(addr, end); 1023 if (pud_none_or_clear_bad(src_pud)) 1024 continue; 1025 if (copy_pmd_range(dst_mm, src_mm, dst_pud, src_pud, 1026 vma, addr, next)) 1027 return -ENOMEM; 1028 } while (dst_pud++, src_pud++, addr = next, addr != end); 1029 return 0; 1030 } 1031 1032 int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 1033 struct vm_area_struct *vma) 1034 { 1035 pgd_t *src_pgd, *dst_pgd; 1036 unsigned long next; 1037 unsigned long addr = vma->vm_start; 1038 unsigned long end = vma->vm_end; 1039 unsigned long mmun_start; /* For mmu_notifiers */ 1040 unsigned long mmun_end; /* For mmu_notifiers */ 1041 bool is_cow; 1042 int ret; 1043 1044 /* 1045 * Don't copy ptes where a page fault will fill them correctly. 1046 * Fork becomes much lighter when there are big shared or private 1047 * readonly mappings. The tradeoff is that copy_page_range is more 1048 * efficient than faulting. 1049 */ 1050 if (!(vma->vm_flags & (VM_HUGETLB | VM_NONLINEAR | 1051 VM_PFNMAP | VM_MIXEDMAP))) { 1052 if (!vma->anon_vma) 1053 return 0; 1054 } 1055 1056 if (is_vm_hugetlb_page(vma)) 1057 return copy_hugetlb_page_range(dst_mm, src_mm, vma); 1058 1059 if (unlikely(vma->vm_flags & VM_PFNMAP)) { 1060 /* 1061 * We do not free on error cases below as remove_vma 1062 * gets called on error from higher level routine 1063 */ 1064 ret = track_pfn_copy(vma); 1065 if (ret) 1066 return ret; 1067 } 1068 1069 /* 1070 * We need to invalidate the secondary MMU mappings only when 1071 * there could be a permission downgrade on the ptes of the 1072 * parent mm. And a permission downgrade will only happen if 1073 * is_cow_mapping() returns true. 1074 */ 1075 is_cow = is_cow_mapping(vma->vm_flags); 1076 mmun_start = addr; 1077 mmun_end = end; 1078 if (is_cow) 1079 mmu_notifier_invalidate_range_start(src_mm, mmun_start, 1080 mmun_end); 1081 1082 ret = 0; 1083 dst_pgd = pgd_offset(dst_mm, addr); 1084 src_pgd = pgd_offset(src_mm, addr); 1085 do { 1086 next = pgd_addr_end(addr, end); 1087 if (pgd_none_or_clear_bad(src_pgd)) 1088 continue; 1089 if (unlikely(copy_pud_range(dst_mm, src_mm, dst_pgd, src_pgd, 1090 vma, addr, next))) { 1091 ret = -ENOMEM; 1092 break; 1093 } 1094 } while (dst_pgd++, src_pgd++, addr = next, addr != end); 1095 1096 if (is_cow) 1097 mmu_notifier_invalidate_range_end(src_mm, mmun_start, mmun_end); 1098 return ret; 1099 } 1100 1101 static unsigned long zap_pte_range(struct mmu_gather *tlb, 1102 struct vm_area_struct *vma, pmd_t *pmd, 1103 unsigned long addr, unsigned long end, 1104 struct zap_details *details) 1105 { 1106 struct mm_struct *mm = tlb->mm; 1107 int force_flush = 0; 1108 int rss[NR_MM_COUNTERS]; 1109 spinlock_t *ptl; 1110 pte_t *start_pte; 1111 pte_t *pte; 1112 1113 again: 1114 init_rss_vec(rss); 1115 start_pte = pte_offset_map_lock(mm, pmd, addr, &ptl); 1116 pte = start_pte; 1117 arch_enter_lazy_mmu_mode(); 1118 do { 1119 pte_t ptent = *pte; 1120 if (pte_none(ptent)) { 1121 continue; 1122 } 1123 1124 if (pte_present(ptent)) { 1125 struct page *page; 1126 1127 page = vm_normal_page(vma, addr, ptent); 1128 if (unlikely(details) && page) { 1129 /* 1130 * unmap_shared_mapping_pages() wants to 1131 * invalidate cache without truncating: 1132 * unmap shared but keep private pages. 1133 */ 1134 if (details->check_mapping && 1135 details->check_mapping != page->mapping) 1136 continue; 1137 /* 1138 * Each page->index must be checked when 1139 * invalidating or truncating nonlinear. 1140 */ 1141 if (details->nonlinear_vma && 1142 (page->index < details->first_index || 1143 page->index > details->last_index)) 1144 continue; 1145 } 1146 ptent = ptep_get_and_clear_full(mm, addr, pte, 1147 tlb->fullmm); 1148 tlb_remove_tlb_entry(tlb, pte, addr); 1149 if (unlikely(!page)) 1150 continue; 1151 if (unlikely(details) && details->nonlinear_vma 1152 && linear_page_index(details->nonlinear_vma, 1153 addr) != page->index) 1154 set_pte_at(mm, addr, pte, 1155 pgoff_to_pte(page->index)); 1156 if (PageAnon(page)) 1157 rss[MM_ANONPAGES]--; 1158 else { 1159 if (pte_dirty(ptent)) 1160 set_page_dirty(page); 1161 if (pte_young(ptent) && 1162 likely(!VM_SequentialReadHint(vma))) 1163 mark_page_accessed(page); 1164 rss[MM_FILEPAGES]--; 1165 } 1166 page_remove_rmap(page); 1167 if (unlikely(page_mapcount(page) < 0)) 1168 print_bad_pte(vma, addr, ptent, page); 1169 force_flush = !__tlb_remove_page(tlb, page); 1170 if (force_flush) 1171 break; 1172 continue; 1173 } 1174 /* 1175 * If details->check_mapping, we leave swap entries; 1176 * if details->nonlinear_vma, we leave file entries. 1177 */ 1178 if (unlikely(details)) 1179 continue; 1180 if (pte_file(ptent)) { 1181 if (unlikely(!(vma->vm_flags & VM_NONLINEAR))) 1182 print_bad_pte(vma, addr, ptent, NULL); 1183 } else { 1184 swp_entry_t entry = pte_to_swp_entry(ptent); 1185 1186 if (!non_swap_entry(entry)) 1187 rss[MM_SWAPENTS]--; 1188 else if (is_migration_entry(entry)) { 1189 struct page *page; 1190 1191 page = migration_entry_to_page(entry); 1192 1193 if (PageAnon(page)) 1194 rss[MM_ANONPAGES]--; 1195 else 1196 rss[MM_FILEPAGES]--; 1197 } 1198 if (unlikely(!free_swap_and_cache(entry))) 1199 print_bad_pte(vma, addr, ptent, NULL); 1200 } 1201 pte_clear_not_present_full(mm, addr, pte, tlb->fullmm); 1202 } while (pte++, addr += PAGE_SIZE, addr != end); 1203 1204 add_mm_rss_vec(mm, rss); 1205 arch_leave_lazy_mmu_mode(); 1206 pte_unmap_unlock(start_pte, ptl); 1207 1208 /* 1209 * mmu_gather ran out of room to batch pages, we break out of 1210 * the PTE lock to avoid doing the potential expensive TLB invalidate 1211 * and page-free while holding it. 1212 */ 1213 if (force_flush) { 1214 force_flush = 0; 1215 1216 #ifdef HAVE_GENERIC_MMU_GATHER 1217 tlb->start = addr; 1218 tlb->end = end; 1219 #endif 1220 tlb_flush_mmu(tlb); 1221 if (addr != end) 1222 goto again; 1223 } 1224 1225 return addr; 1226 } 1227 1228 static inline unsigned long zap_pmd_range(struct mmu_gather *tlb, 1229 struct vm_area_struct *vma, pud_t *pud, 1230 unsigned long addr, unsigned long end, 1231 struct zap_details *details) 1232 { 1233 pmd_t *pmd; 1234 unsigned long next; 1235 1236 pmd = pmd_offset(pud, addr); 1237 do { 1238 next = pmd_addr_end(addr, end); 1239 if (pmd_trans_huge(*pmd)) { 1240 if (next - addr != HPAGE_PMD_SIZE) { 1241 #ifdef CONFIG_DEBUG_VM 1242 if (!rwsem_is_locked(&tlb->mm->mmap_sem)) { 1243 pr_err("%s: mmap_sem is unlocked! addr=0x%lx end=0x%lx vma->vm_start=0x%lx vma->vm_end=0x%lx\n", 1244 __func__, addr, end, 1245 vma->vm_start, 1246 vma->vm_end); 1247 BUG(); 1248 } 1249 #endif 1250 split_huge_page_pmd(vma, addr, pmd); 1251 } else if (zap_huge_pmd(tlb, vma, pmd, addr)) 1252 goto next; 1253 /* fall through */ 1254 } 1255 /* 1256 * Here there can be other concurrent MADV_DONTNEED or 1257 * trans huge page faults running, and if the pmd is 1258 * none or trans huge it can change under us. This is 1259 * because MADV_DONTNEED holds the mmap_sem in read 1260 * mode. 1261 */ 1262 if (pmd_none_or_trans_huge_or_clear_bad(pmd)) 1263 goto next; 1264 next = zap_pte_range(tlb, vma, pmd, addr, next, details); 1265 next: 1266 cond_resched(); 1267 } while (pmd++, addr = next, addr != end); 1268 1269 return addr; 1270 } 1271 1272 static inline unsigned long zap_pud_range(struct mmu_gather *tlb, 1273 struct vm_area_struct *vma, pgd_t *pgd, 1274 unsigned long addr, unsigned long end, 1275 struct zap_details *details) 1276 { 1277 pud_t *pud; 1278 unsigned long next; 1279 1280 pud = pud_offset(pgd, addr); 1281 do { 1282 next = pud_addr_end(addr, end); 1283 if (pud_none_or_clear_bad(pud)) 1284 continue; 1285 next = zap_pmd_range(tlb, vma, pud, addr, next, details); 1286 } while (pud++, addr = next, addr != end); 1287 1288 return addr; 1289 } 1290 1291 static void unmap_page_range(struct mmu_gather *tlb, 1292 struct vm_area_struct *vma, 1293 unsigned long addr, unsigned long end, 1294 struct zap_details *details) 1295 { 1296 pgd_t *pgd; 1297 unsigned long next; 1298 1299 if (details && !details->check_mapping && !details->nonlinear_vma) 1300 details = NULL; 1301 1302 BUG_ON(addr >= end); 1303 mem_cgroup_uncharge_start(); 1304 tlb_start_vma(tlb, vma); 1305 pgd = pgd_offset(vma->vm_mm, addr); 1306 do { 1307 next = pgd_addr_end(addr, end); 1308 if (pgd_none_or_clear_bad(pgd)) 1309 continue; 1310 next = zap_pud_range(tlb, vma, pgd, addr, next, details); 1311 } while (pgd++, addr = next, addr != end); 1312 tlb_end_vma(tlb, vma); 1313 mem_cgroup_uncharge_end(); 1314 } 1315 1316 1317 static void unmap_single_vma(struct mmu_gather *tlb, 1318 struct vm_area_struct *vma, unsigned long start_addr, 1319 unsigned long end_addr, 1320 struct zap_details *details) 1321 { 1322 unsigned long start = max(vma->vm_start, start_addr); 1323 unsigned long end; 1324 1325 if (start >= vma->vm_end) 1326 return; 1327 end = min(vma->vm_end, end_addr); 1328 if (end <= vma->vm_start) 1329 return; 1330 1331 if (vma->vm_file) 1332 uprobe_munmap(vma, start, end); 1333 1334 if (unlikely(vma->vm_flags & VM_PFNMAP)) 1335 untrack_pfn(vma, 0, 0); 1336 1337 if (start != end) { 1338 if (unlikely(is_vm_hugetlb_page(vma))) { 1339 /* 1340 * It is undesirable to test vma->vm_file as it 1341 * should be non-null for valid hugetlb area. 1342 * However, vm_file will be NULL in the error 1343 * cleanup path of do_mmap_pgoff. When 1344 * hugetlbfs ->mmap method fails, 1345 * do_mmap_pgoff() nullifies vma->vm_file 1346 * before calling this function to clean up. 1347 * Since no pte has actually been setup, it is 1348 * safe to do nothing in this case. 1349 */ 1350 if (vma->vm_file) { 1351 mutex_lock(&vma->vm_file->f_mapping->i_mmap_mutex); 1352 __unmap_hugepage_range_final(tlb, vma, start, end, NULL); 1353 mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex); 1354 } 1355 } else 1356 unmap_page_range(tlb, vma, start, end, details); 1357 } 1358 } 1359 1360 /** 1361 * unmap_vmas - unmap a range of memory covered by a list of vma's 1362 * @tlb: address of the caller's struct mmu_gather 1363 * @vma: the starting vma 1364 * @start_addr: virtual address at which to start unmapping 1365 * @end_addr: virtual address at which to end unmapping 1366 * 1367 * Unmap all pages in the vma list. 1368 * 1369 * Only addresses between `start' and `end' will be unmapped. 1370 * 1371 * The VMA list must be sorted in ascending virtual address order. 1372 * 1373 * unmap_vmas() assumes that the caller will flush the whole unmapped address 1374 * range after unmap_vmas() returns. So the only responsibility here is to 1375 * ensure that any thus-far unmapped pages are flushed before unmap_vmas() 1376 * drops the lock and schedules. 1377 */ 1378 void unmap_vmas(struct mmu_gather *tlb, 1379 struct vm_area_struct *vma, unsigned long start_addr, 1380 unsigned long end_addr) 1381 { 1382 struct mm_struct *mm = vma->vm_mm; 1383 1384 mmu_notifier_invalidate_range_start(mm, start_addr, end_addr); 1385 for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next) 1386 unmap_single_vma(tlb, vma, start_addr, end_addr, NULL); 1387 mmu_notifier_invalidate_range_end(mm, start_addr, end_addr); 1388 } 1389 1390 /** 1391 * zap_page_range - remove user pages in a given range 1392 * @vma: vm_area_struct holding the applicable pages 1393 * @start: starting address of pages to zap 1394 * @size: number of bytes to zap 1395 * @details: details of nonlinear truncation or shared cache invalidation 1396 * 1397 * Caller must protect the VMA list 1398 */ 1399 void zap_page_range(struct vm_area_struct *vma, unsigned long start, 1400 unsigned long size, struct zap_details *details) 1401 { 1402 struct mm_struct *mm = vma->vm_mm; 1403 struct mmu_gather tlb; 1404 unsigned long end = start + size; 1405 1406 lru_add_drain(); 1407 tlb_gather_mmu(&tlb, mm, 0); 1408 update_hiwater_rss(mm); 1409 mmu_notifier_invalidate_range_start(mm, start, end); 1410 for ( ; vma && vma->vm_start < end; vma = vma->vm_next) 1411 unmap_single_vma(&tlb, vma, start, end, details); 1412 mmu_notifier_invalidate_range_end(mm, start, end); 1413 tlb_finish_mmu(&tlb, start, end); 1414 } 1415 1416 /** 1417 * zap_page_range_single - remove user pages in a given range 1418 * @vma: vm_area_struct holding the applicable pages 1419 * @address: starting address of pages to zap 1420 * @size: number of bytes to zap 1421 * @details: details of nonlinear truncation or shared cache invalidation 1422 * 1423 * The range must fit into one VMA. 1424 */ 1425 static void zap_page_range_single(struct vm_area_struct *vma, unsigned long address, 1426 unsigned long size, struct zap_details *details) 1427 { 1428 struct mm_struct *mm = vma->vm_mm; 1429 struct mmu_gather tlb; 1430 unsigned long end = address + size; 1431 1432 lru_add_drain(); 1433 tlb_gather_mmu(&tlb, mm, 0); 1434 update_hiwater_rss(mm); 1435 mmu_notifier_invalidate_range_start(mm, address, end); 1436 unmap_single_vma(&tlb, vma, address, end, details); 1437 mmu_notifier_invalidate_range_end(mm, address, end); 1438 tlb_finish_mmu(&tlb, address, end); 1439 } 1440 1441 /** 1442 * zap_vma_ptes - remove ptes mapping the vma 1443 * @vma: vm_area_struct holding ptes to be zapped 1444 * @address: starting address of pages to zap 1445 * @size: number of bytes to zap 1446 * 1447 * This function only unmaps ptes assigned to VM_PFNMAP vmas. 1448 * 1449 * The entire address range must be fully contained within the vma. 1450 * 1451 * Returns 0 if successful. 1452 */ 1453 int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, 1454 unsigned long size) 1455 { 1456 if (address < vma->vm_start || address + size > vma->vm_end || 1457 !(vma->vm_flags & VM_PFNMAP)) 1458 return -1; 1459 zap_page_range_single(vma, address, size, NULL); 1460 return 0; 1461 } 1462 EXPORT_SYMBOL_GPL(zap_vma_ptes); 1463 1464 /** 1465 * follow_page_mask - look up a page descriptor from a user-virtual address 1466 * @vma: vm_area_struct mapping @address 1467 * @address: virtual address to look up 1468 * @flags: flags modifying lookup behaviour 1469 * @page_mask: on output, *page_mask is set according to the size of the page 1470 * 1471 * @flags can have FOLL_ flags set, defined in <linux/mm.h> 1472 * 1473 * Returns the mapped (struct page *), %NULL if no mapping exists, or 1474 * an error pointer if there is a mapping to something not represented 1475 * by a page descriptor (see also vm_normal_page()). 1476 */ 1477 struct page *follow_page_mask(struct vm_area_struct *vma, 1478 unsigned long address, unsigned int flags, 1479 unsigned int *page_mask) 1480 { 1481 pgd_t *pgd; 1482 pud_t *pud; 1483 pmd_t *pmd; 1484 pte_t *ptep, pte; 1485 spinlock_t *ptl; 1486 struct page *page; 1487 struct mm_struct *mm = vma->vm_mm; 1488 1489 *page_mask = 0; 1490 1491 page = follow_huge_addr(mm, address, flags & FOLL_WRITE); 1492 if (!IS_ERR(page)) { 1493 BUG_ON(flags & FOLL_GET); 1494 goto out; 1495 } 1496 1497 page = NULL; 1498 pgd = pgd_offset(mm, address); 1499 if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) 1500 goto no_page_table; 1501 1502 pud = pud_offset(pgd, address); 1503 if (pud_none(*pud)) 1504 goto no_page_table; 1505 if (pud_huge(*pud) && vma->vm_flags & VM_HUGETLB) { 1506 BUG_ON(flags & FOLL_GET); 1507 page = follow_huge_pud(mm, address, pud, flags & FOLL_WRITE); 1508 goto out; 1509 } 1510 if (unlikely(pud_bad(*pud))) 1511 goto no_page_table; 1512 1513 pmd = pmd_offset(pud, address); 1514 if (pmd_none(*pmd)) 1515 goto no_page_table; 1516 if (pmd_huge(*pmd) && vma->vm_flags & VM_HUGETLB) { 1517 BUG_ON(flags & FOLL_GET); 1518 page = follow_huge_pmd(mm, address, pmd, flags & FOLL_WRITE); 1519 goto out; 1520 } 1521 if ((flags & FOLL_NUMA) && pmd_numa(*pmd)) 1522 goto no_page_table; 1523 if (pmd_trans_huge(*pmd)) { 1524 if (flags & FOLL_SPLIT) { 1525 split_huge_page_pmd(vma, address, pmd); 1526 goto split_fallthrough; 1527 } 1528 spin_lock(&mm->page_table_lock); 1529 if (likely(pmd_trans_huge(*pmd))) { 1530 if (unlikely(pmd_trans_splitting(*pmd))) { 1531 spin_unlock(&mm->page_table_lock); 1532 wait_split_huge_page(vma->anon_vma, pmd); 1533 } else { 1534 page = follow_trans_huge_pmd(vma, address, 1535 pmd, flags); 1536 spin_unlock(&mm->page_table_lock); 1537 *page_mask = HPAGE_PMD_NR - 1; 1538 goto out; 1539 } 1540 } else 1541 spin_unlock(&mm->page_table_lock); 1542 /* fall through */ 1543 } 1544 split_fallthrough: 1545 if (unlikely(pmd_bad(*pmd))) 1546 goto no_page_table; 1547 1548 ptep = pte_offset_map_lock(mm, pmd, address, &ptl); 1549 1550 pte = *ptep; 1551 if (!pte_present(pte)) { 1552 swp_entry_t entry; 1553 /* 1554 * KSM's break_ksm() relies upon recognizing a ksm page 1555 * even while it is being migrated, so for that case we 1556 * need migration_entry_wait(). 1557 */ 1558 if (likely(!(flags & FOLL_MIGRATION))) 1559 goto no_page; 1560 if (pte_none(pte) || pte_file(pte)) 1561 goto no_page; 1562 entry = pte_to_swp_entry(pte); 1563 if (!is_migration_entry(entry)) 1564 goto no_page; 1565 pte_unmap_unlock(ptep, ptl); 1566 migration_entry_wait(mm, pmd, address); 1567 goto split_fallthrough; 1568 } 1569 if ((flags & FOLL_NUMA) && pte_numa(pte)) 1570 goto no_page; 1571 if ((flags & FOLL_WRITE) && !pte_write(pte)) 1572 goto unlock; 1573 1574 page = vm_normal_page(vma, address, pte); 1575 if (unlikely(!page)) { 1576 if ((flags & FOLL_DUMP) || 1577 !is_zero_pfn(pte_pfn(pte))) 1578 goto bad_page; 1579 page = pte_page(pte); 1580 } 1581 1582 if (flags & FOLL_GET) 1583 get_page_foll(page); 1584 if (flags & FOLL_TOUCH) { 1585 if ((flags & FOLL_WRITE) && 1586 !pte_dirty(pte) && !PageDirty(page)) 1587 set_page_dirty(page); 1588 /* 1589 * pte_mkyoung() would be more correct here, but atomic care 1590 * is needed to avoid losing the dirty bit: it is easier to use 1591 * mark_page_accessed(). 1592 */ 1593 mark_page_accessed(page); 1594 } 1595 if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) { 1596 /* 1597 * The preliminary mapping check is mainly to avoid the 1598 * pointless overhead of lock_page on the ZERO_PAGE 1599 * which might bounce very badly if there is contention. 1600 * 1601 * If the page is already locked, we don't need to 1602 * handle it now - vmscan will handle it later if and 1603 * when it attempts to reclaim the page. 1604 */ 1605 if (page->mapping && trylock_page(page)) { 1606 lru_add_drain(); /* push cached pages to LRU */ 1607 /* 1608 * Because we lock page here, and migration is 1609 * blocked by the pte's page reference, and we 1610 * know the page is still mapped, we don't even 1611 * need to check for file-cache page truncation. 1612 */ 1613 mlock_vma_page(page); 1614 unlock_page(page); 1615 } 1616 } 1617 unlock: 1618 pte_unmap_unlock(ptep, ptl); 1619 out: 1620 return page; 1621 1622 bad_page: 1623 pte_unmap_unlock(ptep, ptl); 1624 return ERR_PTR(-EFAULT); 1625 1626 no_page: 1627 pte_unmap_unlock(ptep, ptl); 1628 if (!pte_none(pte)) 1629 return page; 1630 1631 no_page_table: 1632 /* 1633 * When core dumping an enormous anonymous area that nobody 1634 * has touched so far, we don't want to allocate unnecessary pages or 1635 * page tables. Return error instead of NULL to skip handle_mm_fault, 1636 * then get_dump_page() will return NULL to leave a hole in the dump. 1637 * But we can only make this optimization where a hole would surely 1638 * be zero-filled if handle_mm_fault() actually did handle it. 1639 */ 1640 if ((flags & FOLL_DUMP) && 1641 (!vma->vm_ops || !vma->vm_ops->fault)) 1642 return ERR_PTR(-EFAULT); 1643 return page; 1644 } 1645 1646 static inline int stack_guard_page(struct vm_area_struct *vma, unsigned long addr) 1647 { 1648 return stack_guard_page_start(vma, addr) || 1649 stack_guard_page_end(vma, addr+PAGE_SIZE); 1650 } 1651 1652 /** 1653 * __get_user_pages() - pin user pages in memory 1654 * @tsk: task_struct of target task 1655 * @mm: mm_struct of target mm 1656 * @start: starting user address 1657 * @nr_pages: number of pages from start to pin 1658 * @gup_flags: flags modifying pin behaviour 1659 * @pages: array that receives pointers to the pages pinned. 1660 * Should be at least nr_pages long. Or NULL, if caller 1661 * only intends to ensure the pages are faulted in. 1662 * @vmas: array of pointers to vmas corresponding to each page. 1663 * Or NULL if the caller does not require them. 1664 * @nonblocking: whether waiting for disk IO or mmap_sem contention 1665 * 1666 * Returns number of pages pinned. This may be fewer than the number 1667 * requested. If nr_pages is 0 or negative, returns 0. If no pages 1668 * were pinned, returns -errno. Each page returned must be released 1669 * with a put_page() call when it is finished with. vmas will only 1670 * remain valid while mmap_sem is held. 1671 * 1672 * Must be called with mmap_sem held for read or write. 1673 * 1674 * __get_user_pages walks a process's page tables and takes a reference to 1675 * each struct page that each user address corresponds to at a given 1676 * instant. That is, it takes the page that would be accessed if a user 1677 * thread accesses the given user virtual address at that instant. 1678 * 1679 * This does not guarantee that the page exists in the user mappings when 1680 * __get_user_pages returns, and there may even be a completely different 1681 * page there in some cases (eg. if mmapped pagecache has been invalidated 1682 * and subsequently re faulted). However it does guarantee that the page 1683 * won't be freed completely. And mostly callers simply care that the page 1684 * contains data that was valid *at some point in time*. Typically, an IO 1685 * or similar operation cannot guarantee anything stronger anyway because 1686 * locks can't be held over the syscall boundary. 1687 * 1688 * If @gup_flags & FOLL_WRITE == 0, the page must not be written to. If 1689 * the page is written to, set_page_dirty (or set_page_dirty_lock, as 1690 * appropriate) must be called after the page is finished with, and 1691 * before put_page is called. 1692 * 1693 * If @nonblocking != NULL, __get_user_pages will not wait for disk IO 1694 * or mmap_sem contention, and if waiting is needed to pin all pages, 1695 * *@nonblocking will be set to 0. 1696 * 1697 * In most cases, get_user_pages or get_user_pages_fast should be used 1698 * instead of __get_user_pages. __get_user_pages should be used only if 1699 * you need some special @gup_flags. 1700 */ 1701 long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, 1702 unsigned long start, unsigned long nr_pages, 1703 unsigned int gup_flags, struct page **pages, 1704 struct vm_area_struct **vmas, int *nonblocking) 1705 { 1706 long i; 1707 unsigned long vm_flags; 1708 unsigned int page_mask; 1709 1710 if (!nr_pages) 1711 return 0; 1712 1713 VM_BUG_ON(!!pages != !!(gup_flags & FOLL_GET)); 1714 1715 /* 1716 * Require read or write permissions. 1717 * If FOLL_FORCE is set, we only require the "MAY" flags. 1718 */ 1719 vm_flags = (gup_flags & FOLL_WRITE) ? 1720 (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD); 1721 vm_flags &= (gup_flags & FOLL_FORCE) ? 1722 (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE); 1723 1724 /* 1725 * If FOLL_FORCE and FOLL_NUMA are both set, handle_mm_fault 1726 * would be called on PROT_NONE ranges. We must never invoke 1727 * handle_mm_fault on PROT_NONE ranges or the NUMA hinting 1728 * page faults would unprotect the PROT_NONE ranges if 1729 * _PAGE_NUMA and _PAGE_PROTNONE are sharing the same pte/pmd 1730 * bitflag. So to avoid that, don't set FOLL_NUMA if 1731 * FOLL_FORCE is set. 1732 */ 1733 if (!(gup_flags & FOLL_FORCE)) 1734 gup_flags |= FOLL_NUMA; 1735 1736 i = 0; 1737 1738 do { 1739 struct vm_area_struct *vma; 1740 1741 vma = find_extend_vma(mm, start); 1742 if (!vma && in_gate_area(mm, start)) { 1743 unsigned long pg = start & PAGE_MASK; 1744 pgd_t *pgd; 1745 pud_t *pud; 1746 pmd_t *pmd; 1747 pte_t *pte; 1748 1749 /* user gate pages are read-only */ 1750 if (gup_flags & FOLL_WRITE) 1751 return i ? : -EFAULT; 1752 if (pg > TASK_SIZE) 1753 pgd = pgd_offset_k(pg); 1754 else 1755 pgd = pgd_offset_gate(mm, pg); 1756 BUG_ON(pgd_none(*pgd)); 1757 pud = pud_offset(pgd, pg); 1758 BUG_ON(pud_none(*pud)); 1759 pmd = pmd_offset(pud, pg); 1760 if (pmd_none(*pmd)) 1761 return i ? : -EFAULT; 1762 VM_BUG_ON(pmd_trans_huge(*pmd)); 1763 pte = pte_offset_map(pmd, pg); 1764 if (pte_none(*pte)) { 1765 pte_unmap(pte); 1766 return i ? : -EFAULT; 1767 } 1768 vma = get_gate_vma(mm); 1769 if (pages) { 1770 struct page *page; 1771 1772 page = vm_normal_page(vma, start, *pte); 1773 if (!page) { 1774 if (!(gup_flags & FOLL_DUMP) && 1775 is_zero_pfn(pte_pfn(*pte))) 1776 page = pte_page(*pte); 1777 else { 1778 pte_unmap(pte); 1779 return i ? : -EFAULT; 1780 } 1781 } 1782 pages[i] = page; 1783 get_page(page); 1784 } 1785 pte_unmap(pte); 1786 page_mask = 0; 1787 goto next_page; 1788 } 1789 1790 if (!vma || 1791 (vma->vm_flags & (VM_IO | VM_PFNMAP)) || 1792 !(vm_flags & vma->vm_flags)) 1793 return i ? : -EFAULT; 1794 1795 if (is_vm_hugetlb_page(vma)) { 1796 i = follow_hugetlb_page(mm, vma, pages, vmas, 1797 &start, &nr_pages, i, gup_flags); 1798 continue; 1799 } 1800 1801 do { 1802 struct page *page; 1803 unsigned int foll_flags = gup_flags; 1804 unsigned int page_increm; 1805 1806 /* 1807 * If we have a pending SIGKILL, don't keep faulting 1808 * pages and potentially allocating memory. 1809 */ 1810 if (unlikely(fatal_signal_pending(current))) 1811 return i ? i : -ERESTARTSYS; 1812 1813 cond_resched(); 1814 while (!(page = follow_page_mask(vma, start, 1815 foll_flags, &page_mask))) { 1816 int ret; 1817 unsigned int fault_flags = 0; 1818 1819 /* For mlock, just skip the stack guard page. */ 1820 if (foll_flags & FOLL_MLOCK) { 1821 if (stack_guard_page(vma, start)) 1822 goto next_page; 1823 } 1824 if (foll_flags & FOLL_WRITE) 1825 fault_flags |= FAULT_FLAG_WRITE; 1826 if (nonblocking) 1827 fault_flags |= FAULT_FLAG_ALLOW_RETRY; 1828 if (foll_flags & FOLL_NOWAIT) 1829 fault_flags |= (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT); 1830 1831 ret = handle_mm_fault(mm, vma, start, 1832 fault_flags); 1833 1834 if (ret & VM_FAULT_ERROR) { 1835 if (ret & VM_FAULT_OOM) 1836 return i ? i : -ENOMEM; 1837 if (ret & (VM_FAULT_HWPOISON | 1838 VM_FAULT_HWPOISON_LARGE)) { 1839 if (i) 1840 return i; 1841 else if (gup_flags & FOLL_HWPOISON) 1842 return -EHWPOISON; 1843 else 1844 return -EFAULT; 1845 } 1846 if (ret & VM_FAULT_SIGBUS) 1847 return i ? i : -EFAULT; 1848 BUG(); 1849 } 1850 1851 if (tsk) { 1852 if (ret & VM_FAULT_MAJOR) 1853 tsk->maj_flt++; 1854 else 1855 tsk->min_flt++; 1856 } 1857 1858 if (ret & VM_FAULT_RETRY) { 1859 if (nonblocking) 1860 *nonblocking = 0; 1861 return i; 1862 } 1863 1864 /* 1865 * The VM_FAULT_WRITE bit tells us that 1866 * do_wp_page has broken COW when necessary, 1867 * even if maybe_mkwrite decided not to set 1868 * pte_write. We can thus safely do subsequent 1869 * page lookups as if they were reads. But only 1870 * do so when looping for pte_write is futile: 1871 * in some cases userspace may also be wanting 1872 * to write to the gotten user page, which a 1873 * read fault here might prevent (a readonly 1874 * page might get reCOWed by userspace write). 1875 */ 1876 if ((ret & VM_FAULT_WRITE) && 1877 !(vma->vm_flags & VM_WRITE)) 1878 foll_flags &= ~FOLL_WRITE; 1879 1880 cond_resched(); 1881 } 1882 if (IS_ERR(page)) 1883 return i ? i : PTR_ERR(page); 1884 if (pages) { 1885 pages[i] = page; 1886 1887 flush_anon_page(vma, page, start); 1888 flush_dcache_page(page); 1889 page_mask = 0; 1890 } 1891 next_page: 1892 if (vmas) { 1893 vmas[i] = vma; 1894 page_mask = 0; 1895 } 1896 page_increm = 1 + (~(start >> PAGE_SHIFT) & page_mask); 1897 if (page_increm > nr_pages) 1898 page_increm = nr_pages; 1899 i += page_increm; 1900 start += page_increm * PAGE_SIZE; 1901 nr_pages -= page_increm; 1902 } while (nr_pages && start < vma->vm_end); 1903 } while (nr_pages); 1904 return i; 1905 } 1906 EXPORT_SYMBOL(__get_user_pages); 1907 1908 /* 1909 * fixup_user_fault() - manually resolve a user page fault 1910 * @tsk: the task_struct to use for page fault accounting, or 1911 * NULL if faults are not to be recorded. 1912 * @mm: mm_struct of target mm 1913 * @address: user address 1914 * @fault_flags:flags to pass down to handle_mm_fault() 1915 * 1916 * This is meant to be called in the specific scenario where for locking reasons 1917 * we try to access user memory in atomic context (within a pagefault_disable() 1918 * section), this returns -EFAULT, and we want to resolve the user fault before 1919 * trying again. 1920 * 1921 * Typically this is meant to be used by the futex code. 1922 * 1923 * The main difference with get_user_pages() is that this function will 1924 * unconditionally call handle_mm_fault() which will in turn perform all the 1925 * necessary SW fixup of the dirty and young bits in the PTE, while 1926 * handle_mm_fault() only guarantees to update these in the struct page. 1927 * 1928 * This is important for some architectures where those bits also gate the 1929 * access permission to the page because they are maintained in software. On 1930 * such architectures, gup() will not be enough to make a subsequent access 1931 * succeed. 1932 * 1933 * This should be called with the mm_sem held for read. 1934 */ 1935 int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm, 1936 unsigned long address, unsigned int fault_flags) 1937 { 1938 struct vm_area_struct *vma; 1939 int ret; 1940 1941 vma = find_extend_vma(mm, address); 1942 if (!vma || address < vma->vm_start) 1943 return -EFAULT; 1944 1945 ret = handle_mm_fault(mm, vma, address, fault_flags); 1946 if (ret & VM_FAULT_ERROR) { 1947 if (ret & VM_FAULT_OOM) 1948 return -ENOMEM; 1949 if (ret & (VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE)) 1950 return -EHWPOISON; 1951 if (ret & VM_FAULT_SIGBUS) 1952 return -EFAULT; 1953 BUG(); 1954 } 1955 if (tsk) { 1956 if (ret & VM_FAULT_MAJOR) 1957 tsk->maj_flt++; 1958 else 1959 tsk->min_flt++; 1960 } 1961 return 0; 1962 } 1963 1964 /* 1965 * get_user_pages() - pin user pages in memory 1966 * @tsk: the task_struct to use for page fault accounting, or 1967 * NULL if faults are not to be recorded. 1968 * @mm: mm_struct of target mm 1969 * @start: starting user address 1970 * @nr_pages: number of pages from start to pin 1971 * @write: whether pages will be written to by the caller 1972 * @force: whether to force write access even if user mapping is 1973 * readonly. This will result in the page being COWed even 1974 * in MAP_SHARED mappings. You do not want this. 1975 * @pages: array that receives pointers to the pages pinned. 1976 * Should be at least nr_pages long. Or NULL, if caller 1977 * only intends to ensure the pages are faulted in. 1978 * @vmas: array of pointers to vmas corresponding to each page. 1979 * Or NULL if the caller does not require them. 1980 * 1981 * Returns number of pages pinned. This may be fewer than the number 1982 * requested. If nr_pages is 0 or negative, returns 0. If no pages 1983 * were pinned, returns -errno. Each page returned must be released 1984 * with a put_page() call when it is finished with. vmas will only 1985 * remain valid while mmap_sem is held. 1986 * 1987 * Must be called with mmap_sem held for read or write. 1988 * 1989 * get_user_pages walks a process's page tables and takes a reference to 1990 * each struct page that each user address corresponds to at a given 1991 * instant. That is, it takes the page that would be accessed if a user 1992 * thread accesses the given user virtual address at that instant. 1993 * 1994 * This does not guarantee that the page exists in the user mappings when 1995 * get_user_pages returns, and there may even be a completely different 1996 * page there in some cases (eg. if mmapped pagecache has been invalidated 1997 * and subsequently re faulted). However it does guarantee that the page 1998 * won't be freed completely. And mostly callers simply care that the page 1999 * contains data that was valid *at some point in time*. Typically, an IO 2000 * or similar operation cannot guarantee anything stronger anyway because 2001 * locks can't be held over the syscall boundary. 2002 * 2003 * If write=0, the page must not be written to. If the page is written to, 2004 * set_page_dirty (or set_page_dirty_lock, as appropriate) must be called 2005 * after the page is finished with, and before put_page is called. 2006 * 2007 * get_user_pages is typically used for fewer-copy IO operations, to get a 2008 * handle on the memory by some means other than accesses via the user virtual 2009 * addresses. The pages may be submitted for DMA to devices or accessed via 2010 * their kernel linear mapping (via the kmap APIs). Care should be taken to 2011 * use the correct cache flushing APIs. 2012 * 2013 * See also get_user_pages_fast, for performance critical applications. 2014 */ 2015 long get_user_pages(struct task_struct *tsk, struct mm_struct *mm, 2016 unsigned long start, unsigned long nr_pages, int write, 2017 int force, struct page **pages, struct vm_area_struct **vmas) 2018 { 2019 int flags = FOLL_TOUCH; 2020 2021 if (pages) 2022 flags |= FOLL_GET; 2023 if (write) 2024 flags |= FOLL_WRITE; 2025 if (force) 2026 flags |= FOLL_FORCE; 2027 2028 return __get_user_pages(tsk, mm, start, nr_pages, flags, pages, vmas, 2029 NULL); 2030 } 2031 EXPORT_SYMBOL(get_user_pages); 2032 2033 /** 2034 * get_dump_page() - pin user page in memory while writing it to core dump 2035 * @addr: user address 2036 * 2037 * Returns struct page pointer of user page pinned for dump, 2038 * to be freed afterwards by page_cache_release() or put_page(). 2039 * 2040 * Returns NULL on any kind of failure - a hole must then be inserted into 2041 * the corefile, to preserve alignment with its headers; and also returns 2042 * NULL wherever the ZERO_PAGE, or an anonymous pte_none, has been found - 2043 * allowing a hole to be left in the corefile to save diskspace. 2044 * 2045 * Called without mmap_sem, but after all other threads have been killed. 2046 */ 2047 #ifdef CONFIG_ELF_CORE 2048 struct page *get_dump_page(unsigned long addr) 2049 { 2050 struct vm_area_struct *vma; 2051 struct page *page; 2052 2053 if (__get_user_pages(current, current->mm, addr, 1, 2054 FOLL_FORCE | FOLL_DUMP | FOLL_GET, &page, &vma, 2055 NULL) < 1) 2056 return NULL; 2057 flush_cache_page(vma, addr, page_to_pfn(page)); 2058 return page; 2059 } 2060 #endif /* CONFIG_ELF_CORE */ 2061 2062 pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr, 2063 spinlock_t **ptl) 2064 { 2065 pgd_t * pgd = pgd_offset(mm, addr); 2066 pud_t * pud = pud_alloc(mm, pgd, addr); 2067 if (pud) { 2068 pmd_t * pmd = pmd_alloc(mm, pud, addr); 2069 if (pmd) { 2070 VM_BUG_ON(pmd_trans_huge(*pmd)); 2071 return pte_alloc_map_lock(mm, pmd, addr, ptl); 2072 } 2073 } 2074 return NULL; 2075 } 2076 2077 /* 2078 * This is the old fallback for page remapping. 2079 * 2080 * For historical reasons, it only allows reserved pages. Only 2081 * old drivers should use this, and they needed to mark their 2082 * pages reserved for the old functions anyway. 2083 */ 2084 static int insert_page(struct vm_area_struct *vma, unsigned long addr, 2085 struct page *page, pgprot_t prot) 2086 { 2087 struct mm_struct *mm = vma->vm_mm; 2088 int retval; 2089 pte_t *pte; 2090 spinlock_t *ptl; 2091 2092 retval = -EINVAL; 2093 if (PageAnon(page)) 2094 goto out; 2095 retval = -ENOMEM; 2096 flush_dcache_page(page); 2097 pte = get_locked_pte(mm, addr, &ptl); 2098 if (!pte) 2099 goto out; 2100 retval = -EBUSY; 2101 if (!pte_none(*pte)) 2102 goto out_unlock; 2103 2104 /* Ok, finally just insert the thing.. */ 2105 get_page(page); 2106 inc_mm_counter_fast(mm, MM_FILEPAGES); 2107 page_add_file_rmap(page); 2108 set_pte_at(mm, addr, pte, mk_pte(page, prot)); 2109 2110 retval = 0; 2111 pte_unmap_unlock(pte, ptl); 2112 return retval; 2113 out_unlock: 2114 pte_unmap_unlock(pte, ptl); 2115 out: 2116 return retval; 2117 } 2118 2119 /** 2120 * vm_insert_page - insert single page into user vma 2121 * @vma: user vma to map to 2122 * @addr: target user address of this page 2123 * @page: source kernel page 2124 * 2125 * This allows drivers to insert individual pages they've allocated 2126 * into a user vma. 2127 * 2128 * The page has to be a nice clean _individual_ kernel allocation. 2129 * If you allocate a compound page, you need to have marked it as 2130 * such (__GFP_COMP), or manually just split the page up yourself 2131 * (see split_page()). 2132 * 2133 * NOTE! Traditionally this was done with "remap_pfn_range()" which 2134 * took an arbitrary page protection parameter. This doesn't allow 2135 * that. Your vma protection will have to be set up correctly, which 2136 * means that if you want a shared writable mapping, you'd better 2137 * ask for a shared writable mapping! 2138 * 2139 * The page does not need to be reserved. 2140 * 2141 * Usually this function is called from f_op->mmap() handler 2142 * under mm->mmap_sem write-lock, so it can change vma->vm_flags. 2143 * Caller must set VM_MIXEDMAP on vma if it wants to call this 2144 * function from other places, for example from page-fault handler. 2145 */ 2146 int vm_insert_page(struct vm_area_struct *vma, unsigned long addr, 2147 struct page *page) 2148 { 2149 if (addr < vma->vm_start || addr >= vma->vm_end) 2150 return -EFAULT; 2151 if (!page_count(page)) 2152 return -EINVAL; 2153 if (!(vma->vm_flags & VM_MIXEDMAP)) { 2154 BUG_ON(down_read_trylock(&vma->vm_mm->mmap_sem)); 2155 BUG_ON(vma->vm_flags & VM_PFNMAP); 2156 vma->vm_flags |= VM_MIXEDMAP; 2157 } 2158 return insert_page(vma, addr, page, vma->vm_page_prot); 2159 } 2160 EXPORT_SYMBOL(vm_insert_page); 2161 2162 static int insert_pfn(struct vm_area_struct *vma, unsigned long addr, 2163 unsigned long pfn, pgprot_t prot) 2164 { 2165 struct mm_struct *mm = vma->vm_mm; 2166 int retval; 2167 pte_t *pte, entry; 2168 spinlock_t *ptl; 2169 2170 retval = -ENOMEM; 2171 pte = get_locked_pte(mm, addr, &ptl); 2172 if (!pte) 2173 goto out; 2174 retval = -EBUSY; 2175 if (!pte_none(*pte)) 2176 goto out_unlock; 2177 2178 /* Ok, finally just insert the thing.. */ 2179 entry = pte_mkspecial(pfn_pte(pfn, prot)); 2180 set_pte_at(mm, addr, pte, entry); 2181 update_mmu_cache(vma, addr, pte); /* XXX: why not for insert_page? */ 2182 2183 retval = 0; 2184 out_unlock: 2185 pte_unmap_unlock(pte, ptl); 2186 out: 2187 return retval; 2188 } 2189 2190 /** 2191 * vm_insert_pfn - insert single pfn into user vma 2192 * @vma: user vma to map to 2193 * @addr: target user address of this page 2194 * @pfn: source kernel pfn 2195 * 2196 * Similar to vm_insert_page, this allows drivers to insert individual pages 2197 * they've allocated into a user vma. Same comments apply. 2198 * 2199 * This function should only be called from a vm_ops->fault handler, and 2200 * in that case the handler should return NULL. 2201 * 2202 * vma cannot be a COW mapping. 2203 * 2204 * As this is called only for pages that do not currently exist, we 2205 * do not need to flush old virtual caches or the TLB. 2206 */ 2207 int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr, 2208 unsigned long pfn) 2209 { 2210 int ret; 2211 pgprot_t pgprot = vma->vm_page_prot; 2212 /* 2213 * Technically, architectures with pte_special can avoid all these 2214 * restrictions (same for remap_pfn_range). However we would like 2215 * consistency in testing and feature parity among all, so we should 2216 * try to keep these invariants in place for everybody. 2217 */ 2218 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); 2219 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 2220 (VM_PFNMAP|VM_MIXEDMAP)); 2221 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 2222 BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn)); 2223 2224 if (addr < vma->vm_start || addr >= vma->vm_end) 2225 return -EFAULT; 2226 if (track_pfn_insert(vma, &pgprot, pfn)) 2227 return -EINVAL; 2228 2229 ret = insert_pfn(vma, addr, pfn, pgprot); 2230 2231 return ret; 2232 } 2233 EXPORT_SYMBOL(vm_insert_pfn); 2234 2235 int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr, 2236 unsigned long pfn) 2237 { 2238 BUG_ON(!(vma->vm_flags & VM_MIXEDMAP)); 2239 2240 if (addr < vma->vm_start || addr >= vma->vm_end) 2241 return -EFAULT; 2242 2243 /* 2244 * If we don't have pte special, then we have to use the pfn_valid() 2245 * based VM_MIXEDMAP scheme (see vm_normal_page), and thus we *must* 2246 * refcount the page if pfn_valid is true (hence insert_page rather 2247 * than insert_pfn). If a zero_pfn were inserted into a VM_MIXEDMAP 2248 * without pte special, it would there be refcounted as a normal page. 2249 */ 2250 if (!HAVE_PTE_SPECIAL && pfn_valid(pfn)) { 2251 struct page *page; 2252 2253 page = pfn_to_page(pfn); 2254 return insert_page(vma, addr, page, vma->vm_page_prot); 2255 } 2256 return insert_pfn(vma, addr, pfn, vma->vm_page_prot); 2257 } 2258 EXPORT_SYMBOL(vm_insert_mixed); 2259 2260 /* 2261 * maps a range of physical memory into the requested pages. the old 2262 * mappings are removed. any references to nonexistent pages results 2263 * in null mappings (currently treated as "copy-on-access") 2264 */ 2265 static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd, 2266 unsigned long addr, unsigned long end, 2267 unsigned long pfn, pgprot_t prot) 2268 { 2269 pte_t *pte; 2270 spinlock_t *ptl; 2271 2272 pte = pte_alloc_map_lock(mm, pmd, addr, &ptl); 2273 if (!pte) 2274 return -ENOMEM; 2275 arch_enter_lazy_mmu_mode(); 2276 do { 2277 BUG_ON(!pte_none(*pte)); 2278 set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot))); 2279 pfn++; 2280 } while (pte++, addr += PAGE_SIZE, addr != end); 2281 arch_leave_lazy_mmu_mode(); 2282 pte_unmap_unlock(pte - 1, ptl); 2283 return 0; 2284 } 2285 2286 static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud, 2287 unsigned long addr, unsigned long end, 2288 unsigned long pfn, pgprot_t prot) 2289 { 2290 pmd_t *pmd; 2291 unsigned long next; 2292 2293 pfn -= addr >> PAGE_SHIFT; 2294 pmd = pmd_alloc(mm, pud, addr); 2295 if (!pmd) 2296 return -ENOMEM; 2297 VM_BUG_ON(pmd_trans_huge(*pmd)); 2298 do { 2299 next = pmd_addr_end(addr, end); 2300 if (remap_pte_range(mm, pmd, addr, next, 2301 pfn + (addr >> PAGE_SHIFT), prot)) 2302 return -ENOMEM; 2303 } while (pmd++, addr = next, addr != end); 2304 return 0; 2305 } 2306 2307 static inline int remap_pud_range(struct mm_struct *mm, pgd_t *pgd, 2308 unsigned long addr, unsigned long end, 2309 unsigned long pfn, pgprot_t prot) 2310 { 2311 pud_t *pud; 2312 unsigned long next; 2313 2314 pfn -= addr >> PAGE_SHIFT; 2315 pud = pud_alloc(mm, pgd, addr); 2316 if (!pud) 2317 return -ENOMEM; 2318 do { 2319 next = pud_addr_end(addr, end); 2320 if (remap_pmd_range(mm, pud, addr, next, 2321 pfn + (addr >> PAGE_SHIFT), prot)) 2322 return -ENOMEM; 2323 } while (pud++, addr = next, addr != end); 2324 return 0; 2325 } 2326 2327 /** 2328 * remap_pfn_range - remap kernel memory to userspace 2329 * @vma: user vma to map to 2330 * @addr: target user address to start at 2331 * @pfn: physical address of kernel memory 2332 * @size: size of map area 2333 * @prot: page protection flags for this mapping 2334 * 2335 * Note: this is only safe if the mm semaphore is held when called. 2336 */ 2337 int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, 2338 unsigned long pfn, unsigned long size, pgprot_t prot) 2339 { 2340 pgd_t *pgd; 2341 unsigned long next; 2342 unsigned long end = addr + PAGE_ALIGN(size); 2343 struct mm_struct *mm = vma->vm_mm; 2344 int err; 2345 2346 /* 2347 * Physically remapped pages are special. Tell the 2348 * rest of the world about it: 2349 * VM_IO tells people not to look at these pages 2350 * (accesses can have side effects). 2351 * VM_PFNMAP tells the core MM that the base pages are just 2352 * raw PFN mappings, and do not have a "struct page" associated 2353 * with them. 2354 * VM_DONTEXPAND 2355 * Disable vma merging and expanding with mremap(). 2356 * VM_DONTDUMP 2357 * Omit vma from core dump, even when VM_IO turned off. 2358 * 2359 * There's a horrible special case to handle copy-on-write 2360 * behaviour that some programs depend on. We mark the "original" 2361 * un-COW'ed pages by matching them up with "vma->vm_pgoff". 2362 * See vm_normal_page() for details. 2363 */ 2364 if (is_cow_mapping(vma->vm_flags)) { 2365 if (addr != vma->vm_start || end != vma->vm_end) 2366 return -EINVAL; 2367 vma->vm_pgoff = pfn; 2368 } 2369 2370 err = track_pfn_remap(vma, &prot, pfn, addr, PAGE_ALIGN(size)); 2371 if (err) 2372 return -EINVAL; 2373 2374 vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP; 2375 2376 BUG_ON(addr >= end); 2377 pfn -= addr >> PAGE_SHIFT; 2378 pgd = pgd_offset(mm, addr); 2379 flush_cache_range(vma, addr, end); 2380 do { 2381 next = pgd_addr_end(addr, end); 2382 err = remap_pud_range(mm, pgd, addr, next, 2383 pfn + (addr >> PAGE_SHIFT), prot); 2384 if (err) 2385 break; 2386 } while (pgd++, addr = next, addr != end); 2387 2388 if (err) 2389 untrack_pfn(vma, pfn, PAGE_ALIGN(size)); 2390 2391 return err; 2392 } 2393 EXPORT_SYMBOL(remap_pfn_range); 2394 2395 static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd, 2396 unsigned long addr, unsigned long end, 2397 pte_fn_t fn, void *data) 2398 { 2399 pte_t *pte; 2400 int err; 2401 pgtable_t token; 2402 spinlock_t *uninitialized_var(ptl); 2403 2404 pte = (mm == &init_mm) ? 2405 pte_alloc_kernel(pmd, addr) : 2406 pte_alloc_map_lock(mm, pmd, addr, &ptl); 2407 if (!pte) 2408 return -ENOMEM; 2409 2410 BUG_ON(pmd_huge(*pmd)); 2411 2412 arch_enter_lazy_mmu_mode(); 2413 2414 token = pmd_pgtable(*pmd); 2415 2416 do { 2417 err = fn(pte++, token, addr, data); 2418 if (err) 2419 break; 2420 } while (addr += PAGE_SIZE, addr != end); 2421 2422 arch_leave_lazy_mmu_mode(); 2423 2424 if (mm != &init_mm) 2425 pte_unmap_unlock(pte-1, ptl); 2426 return err; 2427 } 2428 2429 static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud, 2430 unsigned long addr, unsigned long end, 2431 pte_fn_t fn, void *data) 2432 { 2433 pmd_t *pmd; 2434 unsigned long next; 2435 int err; 2436 2437 BUG_ON(pud_huge(*pud)); 2438 2439 pmd = pmd_alloc(mm, pud, addr); 2440 if (!pmd) 2441 return -ENOMEM; 2442 do { 2443 next = pmd_addr_end(addr, end); 2444 err = apply_to_pte_range(mm, pmd, addr, next, fn, data); 2445 if (err) 2446 break; 2447 } while (pmd++, addr = next, addr != end); 2448 return err; 2449 } 2450 2451 static int apply_to_pud_range(struct mm_struct *mm, pgd_t *pgd, 2452 unsigned long addr, unsigned long end, 2453 pte_fn_t fn, void *data) 2454 { 2455 pud_t *pud; 2456 unsigned long next; 2457 int err; 2458 2459 pud = pud_alloc(mm, pgd, addr); 2460 if (!pud) 2461 return -ENOMEM; 2462 do { 2463 next = pud_addr_end(addr, end); 2464 err = apply_to_pmd_range(mm, pud, addr, next, fn, data); 2465 if (err) 2466 break; 2467 } while (pud++, addr = next, addr != end); 2468 return err; 2469 } 2470 2471 /* 2472 * Scan a region of virtual memory, filling in page tables as necessary 2473 * and calling a provided function on each leaf page table. 2474 */ 2475 int apply_to_page_range(struct mm_struct *mm, unsigned long addr, 2476 unsigned long size, pte_fn_t fn, void *data) 2477 { 2478 pgd_t *pgd; 2479 unsigned long next; 2480 unsigned long end = addr + size; 2481 int err; 2482 2483 BUG_ON(addr >= end); 2484 pgd = pgd_offset(mm, addr); 2485 do { 2486 next = pgd_addr_end(addr, end); 2487 err = apply_to_pud_range(mm, pgd, addr, next, fn, data); 2488 if (err) 2489 break; 2490 } while (pgd++, addr = next, addr != end); 2491 2492 return err; 2493 } 2494 EXPORT_SYMBOL_GPL(apply_to_page_range); 2495 2496 /* 2497 * handle_pte_fault chooses page fault handler according to an entry 2498 * which was read non-atomically. Before making any commitment, on 2499 * those architectures or configurations (e.g. i386 with PAE) which 2500 * might give a mix of unmatched parts, do_swap_page and do_nonlinear_fault 2501 * must check under lock before unmapping the pte and proceeding 2502 * (but do_wp_page is only called after already making such a check; 2503 * and do_anonymous_page can safely check later on). 2504 */ 2505 static inline int pte_unmap_same(struct mm_struct *mm, pmd_t *pmd, 2506 pte_t *page_table, pte_t orig_pte) 2507 { 2508 int same = 1; 2509 #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT) 2510 if (sizeof(pte_t) > sizeof(unsigned long)) { 2511 spinlock_t *ptl = pte_lockptr(mm, pmd); 2512 spin_lock(ptl); 2513 same = pte_same(*page_table, orig_pte); 2514 spin_unlock(ptl); 2515 } 2516 #endif 2517 pte_unmap(page_table); 2518 return same; 2519 } 2520 2521 static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma) 2522 { 2523 /* 2524 * If the source page was a PFN mapping, we don't have 2525 * a "struct page" for it. We do a best-effort copy by 2526 * just copying from the original user address. If that 2527 * fails, we just zero-fill it. Live with it. 2528 */ 2529 if (unlikely(!src)) { 2530 void *kaddr = kmap_atomic(dst); 2531 void __user *uaddr = (void __user *)(va & PAGE_MASK); 2532 2533 /* 2534 * This really shouldn't fail, because the page is there 2535 * in the page tables. But it might just be unreadable, 2536 * in which case we just give up and fill the result with 2537 * zeroes. 2538 */ 2539 if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) 2540 clear_page(kaddr); 2541 kunmap_atomic(kaddr); 2542 flush_dcache_page(dst); 2543 } else 2544 copy_user_highpage(dst, src, va, vma); 2545 } 2546 2547 /* 2548 * This routine handles present pages, when users try to write 2549 * to a shared page. It is done by copying the page to a new address 2550 * and decrementing the shared-page counter for the old page. 2551 * 2552 * Note that this routine assumes that the protection checks have been 2553 * done by the caller (the low-level page fault routine in most cases). 2554 * Thus we can safely just mark it writable once we've done any necessary 2555 * COW. 2556 * 2557 * We also mark the page dirty at this point even though the page will 2558 * change only once the write actually happens. This avoids a few races, 2559 * and potentially makes it more efficient. 2560 * 2561 * We enter with non-exclusive mmap_sem (to exclude vma changes, 2562 * but allow concurrent faults), with pte both mapped and locked. 2563 * We return with mmap_sem still held, but pte unmapped and unlocked. 2564 */ 2565 static int do_wp_page(struct mm_struct *mm, struct vm_area_struct *vma, 2566 unsigned long address, pte_t *page_table, pmd_t *pmd, 2567 spinlock_t *ptl, pte_t orig_pte) 2568 __releases(ptl) 2569 { 2570 struct page *old_page, *new_page = NULL; 2571 pte_t entry; 2572 int ret = 0; 2573 int page_mkwrite = 0; 2574 struct page *dirty_page = NULL; 2575 unsigned long mmun_start = 0; /* For mmu_notifiers */ 2576 unsigned long mmun_end = 0; /* For mmu_notifiers */ 2577 2578 old_page = vm_normal_page(vma, address, orig_pte); 2579 if (!old_page) { 2580 /* 2581 * VM_MIXEDMAP !pfn_valid() case 2582 * 2583 * We should not cow pages in a shared writeable mapping. 2584 * Just mark the pages writable as we can't do any dirty 2585 * accounting on raw pfn maps. 2586 */ 2587 if ((vma->vm_flags & (VM_WRITE|VM_SHARED)) == 2588 (VM_WRITE|VM_SHARED)) 2589 goto reuse; 2590 goto gotten; 2591 } 2592 2593 /* 2594 * Take out anonymous pages first, anonymous shared vmas are 2595 * not dirty accountable. 2596 */ 2597 if (PageAnon(old_page) && !PageKsm(old_page)) { 2598 if (!trylock_page(old_page)) { 2599 page_cache_get(old_page); 2600 pte_unmap_unlock(page_table, ptl); 2601 lock_page(old_page); 2602 page_table = pte_offset_map_lock(mm, pmd, address, 2603 &ptl); 2604 if (!pte_same(*page_table, orig_pte)) { 2605 unlock_page(old_page); 2606 goto unlock; 2607 } 2608 page_cache_release(old_page); 2609 } 2610 if (reuse_swap_page(old_page)) { 2611 /* 2612 * The page is all ours. Move it to our anon_vma so 2613 * the rmap code will not search our parent or siblings. 2614 * Protected against the rmap code by the page lock. 2615 */ 2616 page_move_anon_rmap(old_page, vma, address); 2617 unlock_page(old_page); 2618 goto reuse; 2619 } 2620 unlock_page(old_page); 2621 } else if (unlikely((vma->vm_flags & (VM_WRITE|VM_SHARED)) == 2622 (VM_WRITE|VM_SHARED))) { 2623 /* 2624 * Only catch write-faults on shared writable pages, 2625 * read-only shared pages can get COWed by 2626 * get_user_pages(.write=1, .force=1). 2627 */ 2628 if (vma->vm_ops && vma->vm_ops->page_mkwrite) { 2629 struct vm_fault vmf; 2630 int tmp; 2631 2632 vmf.virtual_address = (void __user *)(address & 2633 PAGE_MASK); 2634 vmf.pgoff = old_page->index; 2635 vmf.flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE; 2636 vmf.page = old_page; 2637 2638 /* 2639 * Notify the address space that the page is about to 2640 * become writable so that it can prohibit this or wait 2641 * for the page to get into an appropriate state. 2642 * 2643 * We do this without the lock held, so that it can 2644 * sleep if it needs to. 2645 */ 2646 page_cache_get(old_page); 2647 pte_unmap_unlock(page_table, ptl); 2648 2649 tmp = vma->vm_ops->page_mkwrite(vma, &vmf); 2650 if (unlikely(tmp & 2651 (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) { 2652 ret = tmp; 2653 goto unwritable_page; 2654 } 2655 if (unlikely(!(tmp & VM_FAULT_LOCKED))) { 2656 lock_page(old_page); 2657 if (!old_page->mapping) { 2658 ret = 0; /* retry the fault */ 2659 unlock_page(old_page); 2660 goto unwritable_page; 2661 } 2662 } else 2663 VM_BUG_ON(!PageLocked(old_page)); 2664 2665 /* 2666 * Since we dropped the lock we need to revalidate 2667 * the PTE as someone else may have changed it. If 2668 * they did, we just return, as we can count on the 2669 * MMU to tell us if they didn't also make it writable. 2670 */ 2671 page_table = pte_offset_map_lock(mm, pmd, address, 2672 &ptl); 2673 if (!pte_same(*page_table, orig_pte)) { 2674 unlock_page(old_page); 2675 goto unlock; 2676 } 2677 2678 page_mkwrite = 1; 2679 } 2680 dirty_page = old_page; 2681 get_page(dirty_page); 2682 2683 reuse: 2684 flush_cache_page(vma, address, pte_pfn(orig_pte)); 2685 entry = pte_mkyoung(orig_pte); 2686 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 2687 if (ptep_set_access_flags(vma, address, page_table, entry,1)) 2688 update_mmu_cache(vma, address, page_table); 2689 pte_unmap_unlock(page_table, ptl); 2690 ret |= VM_FAULT_WRITE; 2691 2692 if (!dirty_page) 2693 return ret; 2694 2695 /* 2696 * Yes, Virginia, this is actually required to prevent a race 2697 * with clear_page_dirty_for_io() from clearing the page dirty 2698 * bit after it clear all dirty ptes, but before a racing 2699 * do_wp_page installs a dirty pte. 2700 * 2701 * __do_fault is protected similarly. 2702 */ 2703 if (!page_mkwrite) { 2704 wait_on_page_locked(dirty_page); 2705 set_page_dirty_balance(dirty_page, page_mkwrite); 2706 /* file_update_time outside page_lock */ 2707 if (vma->vm_file) 2708 file_update_time(vma->vm_file); 2709 } 2710 put_page(dirty_page); 2711 if (page_mkwrite) { 2712 struct address_space *mapping = dirty_page->mapping; 2713 2714 set_page_dirty(dirty_page); 2715 unlock_page(dirty_page); 2716 page_cache_release(dirty_page); 2717 if (mapping) { 2718 /* 2719 * Some device drivers do not set page.mapping 2720 * but still dirty their pages 2721 */ 2722 balance_dirty_pages_ratelimited(mapping); 2723 } 2724 } 2725 2726 return ret; 2727 } 2728 2729 /* 2730 * Ok, we need to copy. Oh, well.. 2731 */ 2732 page_cache_get(old_page); 2733 gotten: 2734 pte_unmap_unlock(page_table, ptl); 2735 2736 if (unlikely(anon_vma_prepare(vma))) 2737 goto oom; 2738 2739 if (is_zero_pfn(pte_pfn(orig_pte))) { 2740 new_page = alloc_zeroed_user_highpage_movable(vma, address); 2741 if (!new_page) 2742 goto oom; 2743 } else { 2744 new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address); 2745 if (!new_page) 2746 goto oom; 2747 cow_user_page(new_page, old_page, address, vma); 2748 } 2749 __SetPageUptodate(new_page); 2750 2751 if (mem_cgroup_newpage_charge(new_page, mm, GFP_KERNEL)) 2752 goto oom_free_new; 2753 2754 mmun_start = address & PAGE_MASK; 2755 mmun_end = mmun_start + PAGE_SIZE; 2756 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 2757 2758 /* 2759 * Re-check the pte - we dropped the lock 2760 */ 2761 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 2762 if (likely(pte_same(*page_table, orig_pte))) { 2763 if (old_page) { 2764 if (!PageAnon(old_page)) { 2765 dec_mm_counter_fast(mm, MM_FILEPAGES); 2766 inc_mm_counter_fast(mm, MM_ANONPAGES); 2767 } 2768 } else 2769 inc_mm_counter_fast(mm, MM_ANONPAGES); 2770 flush_cache_page(vma, address, pte_pfn(orig_pte)); 2771 entry = mk_pte(new_page, vma->vm_page_prot); 2772 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 2773 /* 2774 * Clear the pte entry and flush it first, before updating the 2775 * pte with the new entry. This will avoid a race condition 2776 * seen in the presence of one thread doing SMC and another 2777 * thread doing COW. 2778 */ 2779 ptep_clear_flush(vma, address, page_table); 2780 page_add_new_anon_rmap(new_page, vma, address); 2781 /* 2782 * We call the notify macro here because, when using secondary 2783 * mmu page tables (such as kvm shadow page tables), we want the 2784 * new page to be mapped directly into the secondary page table. 2785 */ 2786 set_pte_at_notify(mm, address, page_table, entry); 2787 update_mmu_cache(vma, address, page_table); 2788 if (old_page) { 2789 /* 2790 * Only after switching the pte to the new page may 2791 * we remove the mapcount here. Otherwise another 2792 * process may come and find the rmap count decremented 2793 * before the pte is switched to the new page, and 2794 * "reuse" the old page writing into it while our pte 2795 * here still points into it and can be read by other 2796 * threads. 2797 * 2798 * The critical issue is to order this 2799 * page_remove_rmap with the ptp_clear_flush above. 2800 * Those stores are ordered by (if nothing else,) 2801 * the barrier present in the atomic_add_negative 2802 * in page_remove_rmap. 2803 * 2804 * Then the TLB flush in ptep_clear_flush ensures that 2805 * no process can access the old page before the 2806 * decremented mapcount is visible. And the old page 2807 * cannot be reused until after the decremented 2808 * mapcount is visible. So transitively, TLBs to 2809 * old page will be flushed before it can be reused. 2810 */ 2811 page_remove_rmap(old_page); 2812 } 2813 2814 /* Free the old page.. */ 2815 new_page = old_page; 2816 ret |= VM_FAULT_WRITE; 2817 } else 2818 mem_cgroup_uncharge_page(new_page); 2819 2820 if (new_page) 2821 page_cache_release(new_page); 2822 unlock: 2823 pte_unmap_unlock(page_table, ptl); 2824 if (mmun_end > mmun_start) 2825 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 2826 if (old_page) { 2827 /* 2828 * Don't let another task, with possibly unlocked vma, 2829 * keep the mlocked page. 2830 */ 2831 if ((ret & VM_FAULT_WRITE) && (vma->vm_flags & VM_LOCKED)) { 2832 lock_page(old_page); /* LRU manipulation */ 2833 munlock_vma_page(old_page); 2834 unlock_page(old_page); 2835 } 2836 page_cache_release(old_page); 2837 } 2838 return ret; 2839 oom_free_new: 2840 page_cache_release(new_page); 2841 oom: 2842 if (old_page) 2843 page_cache_release(old_page); 2844 return VM_FAULT_OOM; 2845 2846 unwritable_page: 2847 page_cache_release(old_page); 2848 return ret; 2849 } 2850 2851 static void unmap_mapping_range_vma(struct vm_area_struct *vma, 2852 unsigned long start_addr, unsigned long end_addr, 2853 struct zap_details *details) 2854 { 2855 zap_page_range_single(vma, start_addr, end_addr - start_addr, details); 2856 } 2857 2858 static inline void unmap_mapping_range_tree(struct rb_root *root, 2859 struct zap_details *details) 2860 { 2861 struct vm_area_struct *vma; 2862 pgoff_t vba, vea, zba, zea; 2863 2864 vma_interval_tree_foreach(vma, root, 2865 details->first_index, details->last_index) { 2866 2867 vba = vma->vm_pgoff; 2868 vea = vba + ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT) - 1; 2869 /* Assume for now that PAGE_CACHE_SHIFT == PAGE_SHIFT */ 2870 zba = details->first_index; 2871 if (zba < vba) 2872 zba = vba; 2873 zea = details->last_index; 2874 if (zea > vea) 2875 zea = vea; 2876 2877 unmap_mapping_range_vma(vma, 2878 ((zba - vba) << PAGE_SHIFT) + vma->vm_start, 2879 ((zea - vba + 1) << PAGE_SHIFT) + vma->vm_start, 2880 details); 2881 } 2882 } 2883 2884 static inline void unmap_mapping_range_list(struct list_head *head, 2885 struct zap_details *details) 2886 { 2887 struct vm_area_struct *vma; 2888 2889 /* 2890 * In nonlinear VMAs there is no correspondence between virtual address 2891 * offset and file offset. So we must perform an exhaustive search 2892 * across *all* the pages in each nonlinear VMA, not just the pages 2893 * whose virtual address lies outside the file truncation point. 2894 */ 2895 list_for_each_entry(vma, head, shared.nonlinear) { 2896 details->nonlinear_vma = vma; 2897 unmap_mapping_range_vma(vma, vma->vm_start, vma->vm_end, details); 2898 } 2899 } 2900 2901 /** 2902 * unmap_mapping_range - unmap the portion of all mmaps in the specified address_space corresponding to the specified page range in the underlying file. 2903 * @mapping: the address space containing mmaps to be unmapped. 2904 * @holebegin: byte in first page to unmap, relative to the start of 2905 * the underlying file. This will be rounded down to a PAGE_SIZE 2906 * boundary. Note that this is different from truncate_pagecache(), which 2907 * must keep the partial page. In contrast, we must get rid of 2908 * partial pages. 2909 * @holelen: size of prospective hole in bytes. This will be rounded 2910 * up to a PAGE_SIZE boundary. A holelen of zero truncates to the 2911 * end of the file. 2912 * @even_cows: 1 when truncating a file, unmap even private COWed pages; 2913 * but 0 when invalidating pagecache, don't throw away private data. 2914 */ 2915 void unmap_mapping_range(struct address_space *mapping, 2916 loff_t const holebegin, loff_t const holelen, int even_cows) 2917 { 2918 struct zap_details details; 2919 pgoff_t hba = holebegin >> PAGE_SHIFT; 2920 pgoff_t hlen = (holelen + PAGE_SIZE - 1) >> PAGE_SHIFT; 2921 2922 /* Check for overflow. */ 2923 if (sizeof(holelen) > sizeof(hlen)) { 2924 long long holeend = 2925 (holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT; 2926 if (holeend & ~(long long)ULONG_MAX) 2927 hlen = ULONG_MAX - hba + 1; 2928 } 2929 2930 details.check_mapping = even_cows? NULL: mapping; 2931 details.nonlinear_vma = NULL; 2932 details.first_index = hba; 2933 details.last_index = hba + hlen - 1; 2934 if (details.last_index < details.first_index) 2935 details.last_index = ULONG_MAX; 2936 2937 2938 mutex_lock(&mapping->i_mmap_mutex); 2939 if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap))) 2940 unmap_mapping_range_tree(&mapping->i_mmap, &details); 2941 if (unlikely(!list_empty(&mapping->i_mmap_nonlinear))) 2942 unmap_mapping_range_list(&mapping->i_mmap_nonlinear, &details); 2943 mutex_unlock(&mapping->i_mmap_mutex); 2944 } 2945 EXPORT_SYMBOL(unmap_mapping_range); 2946 2947 /* 2948 * We enter with non-exclusive mmap_sem (to exclude vma changes, 2949 * but allow concurrent faults), and pte mapped but not yet locked. 2950 * We return with mmap_sem still held, but pte unmapped and unlocked. 2951 */ 2952 static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma, 2953 unsigned long address, pte_t *page_table, pmd_t *pmd, 2954 unsigned int flags, pte_t orig_pte) 2955 { 2956 spinlock_t *ptl; 2957 struct page *page, *swapcache; 2958 swp_entry_t entry; 2959 pte_t pte; 2960 int locked; 2961 struct mem_cgroup *ptr; 2962 int exclusive = 0; 2963 int ret = 0; 2964 2965 if (!pte_unmap_same(mm, pmd, page_table, orig_pte)) 2966 goto out; 2967 2968 entry = pte_to_swp_entry(orig_pte); 2969 if (unlikely(non_swap_entry(entry))) { 2970 if (is_migration_entry(entry)) { 2971 migration_entry_wait(mm, pmd, address); 2972 } else if (is_hwpoison_entry(entry)) { 2973 ret = VM_FAULT_HWPOISON; 2974 } else { 2975 print_bad_pte(vma, address, orig_pte, NULL); 2976 ret = VM_FAULT_SIGBUS; 2977 } 2978 goto out; 2979 } 2980 delayacct_set_flag(DELAYACCT_PF_SWAPIN); 2981 page = lookup_swap_cache(entry); 2982 if (!page) { 2983 page = swapin_readahead(entry, 2984 GFP_HIGHUSER_MOVABLE, vma, address); 2985 if (!page) { 2986 /* 2987 * Back out if somebody else faulted in this pte 2988 * while we released the pte lock. 2989 */ 2990 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 2991 if (likely(pte_same(*page_table, orig_pte))) 2992 ret = VM_FAULT_OOM; 2993 delayacct_clear_flag(DELAYACCT_PF_SWAPIN); 2994 goto unlock; 2995 } 2996 2997 /* Had to read the page from swap area: Major fault */ 2998 ret = VM_FAULT_MAJOR; 2999 count_vm_event(PGMAJFAULT); 3000 mem_cgroup_count_vm_event(mm, PGMAJFAULT); 3001 } else if (PageHWPoison(page)) { 3002 /* 3003 * hwpoisoned dirty swapcache pages are kept for killing 3004 * owner processes (which may be unknown at hwpoison time) 3005 */ 3006 ret = VM_FAULT_HWPOISON; 3007 delayacct_clear_flag(DELAYACCT_PF_SWAPIN); 3008 swapcache = page; 3009 goto out_release; 3010 } 3011 3012 swapcache = page; 3013 locked = lock_page_or_retry(page, mm, flags); 3014 3015 delayacct_clear_flag(DELAYACCT_PF_SWAPIN); 3016 if (!locked) { 3017 ret |= VM_FAULT_RETRY; 3018 goto out_release; 3019 } 3020 3021 /* 3022 * Make sure try_to_free_swap or reuse_swap_page or swapoff did not 3023 * release the swapcache from under us. The page pin, and pte_same 3024 * test below, are not enough to exclude that. Even if it is still 3025 * swapcache, we need to check that the page's swap has not changed. 3026 */ 3027 if (unlikely(!PageSwapCache(page) || page_private(page) != entry.val)) 3028 goto out_page; 3029 3030 page = ksm_might_need_to_copy(page, vma, address); 3031 if (unlikely(!page)) { 3032 ret = VM_FAULT_OOM; 3033 page = swapcache; 3034 goto out_page; 3035 } 3036 3037 if (mem_cgroup_try_charge_swapin(mm, page, GFP_KERNEL, &ptr)) { 3038 ret = VM_FAULT_OOM; 3039 goto out_page; 3040 } 3041 3042 /* 3043 * Back out if somebody else already faulted in this pte. 3044 */ 3045 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 3046 if (unlikely(!pte_same(*page_table, orig_pte))) 3047 goto out_nomap; 3048 3049 if (unlikely(!PageUptodate(page))) { 3050 ret = VM_FAULT_SIGBUS; 3051 goto out_nomap; 3052 } 3053 3054 /* 3055 * The page isn't present yet, go ahead with the fault. 3056 * 3057 * Be careful about the sequence of operations here. 3058 * To get its accounting right, reuse_swap_page() must be called 3059 * while the page is counted on swap but not yet in mapcount i.e. 3060 * before page_add_anon_rmap() and swap_free(); try_to_free_swap() 3061 * must be called after the swap_free(), or it will never succeed. 3062 * Because delete_from_swap_page() may be called by reuse_swap_page(), 3063 * mem_cgroup_commit_charge_swapin() may not be able to find swp_entry 3064 * in page->private. In this case, a record in swap_cgroup is silently 3065 * discarded at swap_free(). 3066 */ 3067 3068 inc_mm_counter_fast(mm, MM_ANONPAGES); 3069 dec_mm_counter_fast(mm, MM_SWAPENTS); 3070 pte = mk_pte(page, vma->vm_page_prot); 3071 if ((flags & FAULT_FLAG_WRITE) && reuse_swap_page(page)) { 3072 pte = maybe_mkwrite(pte_mkdirty(pte), vma); 3073 flags &= ~FAULT_FLAG_WRITE; 3074 ret |= VM_FAULT_WRITE; 3075 exclusive = 1; 3076 } 3077 flush_icache_page(vma, page); 3078 set_pte_at(mm, address, page_table, pte); 3079 if (page == swapcache) 3080 do_page_add_anon_rmap(page, vma, address, exclusive); 3081 else /* ksm created a completely new copy */ 3082 page_add_new_anon_rmap(page, vma, address); 3083 /* It's better to call commit-charge after rmap is established */ 3084 mem_cgroup_commit_charge_swapin(page, ptr); 3085 3086 swap_free(entry); 3087 if (vm_swap_full() || (vma->vm_flags & VM_LOCKED) || PageMlocked(page)) 3088 try_to_free_swap(page); 3089 unlock_page(page); 3090 if (page != swapcache) { 3091 /* 3092 * Hold the lock to avoid the swap entry to be reused 3093 * until we take the PT lock for the pte_same() check 3094 * (to avoid false positives from pte_same). For 3095 * further safety release the lock after the swap_free 3096 * so that the swap count won't change under a 3097 * parallel locked swapcache. 3098 */ 3099 unlock_page(swapcache); 3100 page_cache_release(swapcache); 3101 } 3102 3103 if (flags & FAULT_FLAG_WRITE) { 3104 ret |= do_wp_page(mm, vma, address, page_table, pmd, ptl, pte); 3105 if (ret & VM_FAULT_ERROR) 3106 ret &= VM_FAULT_ERROR; 3107 goto out; 3108 } 3109 3110 /* No need to invalidate - it was non-present before */ 3111 update_mmu_cache(vma, address, page_table); 3112 unlock: 3113 pte_unmap_unlock(page_table, ptl); 3114 out: 3115 return ret; 3116 out_nomap: 3117 mem_cgroup_cancel_charge_swapin(ptr); 3118 pte_unmap_unlock(page_table, ptl); 3119 out_page: 3120 unlock_page(page); 3121 out_release: 3122 page_cache_release(page); 3123 if (page != swapcache) { 3124 unlock_page(swapcache); 3125 page_cache_release(swapcache); 3126 } 3127 return ret; 3128 } 3129 3130 /* 3131 * This is like a special single-page "expand_{down|up}wards()", 3132 * except we must first make sure that 'address{-|+}PAGE_SIZE' 3133 * doesn't hit another vma. 3134 */ 3135 static inline int check_stack_guard_page(struct vm_area_struct *vma, unsigned long address) 3136 { 3137 address &= PAGE_MASK; 3138 if ((vma->vm_flags & VM_GROWSDOWN) && address == vma->vm_start) { 3139 struct vm_area_struct *prev = vma->vm_prev; 3140 3141 /* 3142 * Is there a mapping abutting this one below? 3143 * 3144 * That's only ok if it's the same stack mapping 3145 * that has gotten split.. 3146 */ 3147 if (prev && prev->vm_end == address) 3148 return prev->vm_flags & VM_GROWSDOWN ? 0 : -ENOMEM; 3149 3150 expand_downwards(vma, address - PAGE_SIZE); 3151 } 3152 if ((vma->vm_flags & VM_GROWSUP) && address + PAGE_SIZE == vma->vm_end) { 3153 struct vm_area_struct *next = vma->vm_next; 3154 3155 /* As VM_GROWSDOWN but s/below/above/ */ 3156 if (next && next->vm_start == address + PAGE_SIZE) 3157 return next->vm_flags & VM_GROWSUP ? 0 : -ENOMEM; 3158 3159 expand_upwards(vma, address + PAGE_SIZE); 3160 } 3161 return 0; 3162 } 3163 3164 /* 3165 * We enter with non-exclusive mmap_sem (to exclude vma changes, 3166 * but allow concurrent faults), and pte mapped but not yet locked. 3167 * We return with mmap_sem still held, but pte unmapped and unlocked. 3168 */ 3169 static int do_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma, 3170 unsigned long address, pte_t *page_table, pmd_t *pmd, 3171 unsigned int flags) 3172 { 3173 struct page *page; 3174 spinlock_t *ptl; 3175 pte_t entry; 3176 3177 pte_unmap(page_table); 3178 3179 /* Check if we need to add a guard page to the stack */ 3180 if (check_stack_guard_page(vma, address) < 0) 3181 return VM_FAULT_SIGBUS; 3182 3183 /* Use the zero-page for reads */ 3184 if (!(flags & FAULT_FLAG_WRITE)) { 3185 entry = pte_mkspecial(pfn_pte(my_zero_pfn(address), 3186 vma->vm_page_prot)); 3187 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 3188 if (!pte_none(*page_table)) 3189 goto unlock; 3190 goto setpte; 3191 } 3192 3193 /* Allocate our own private page. */ 3194 if (unlikely(anon_vma_prepare(vma))) 3195 goto oom; 3196 page = alloc_zeroed_user_highpage_movable(vma, address); 3197 if (!page) 3198 goto oom; 3199 __SetPageUptodate(page); 3200 3201 if (mem_cgroup_newpage_charge(page, mm, GFP_KERNEL)) 3202 goto oom_free_page; 3203 3204 entry = mk_pte(page, vma->vm_page_prot); 3205 if (vma->vm_flags & VM_WRITE) 3206 entry = pte_mkwrite(pte_mkdirty(entry)); 3207 3208 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 3209 if (!pte_none(*page_table)) 3210 goto release; 3211 3212 inc_mm_counter_fast(mm, MM_ANONPAGES); 3213 page_add_new_anon_rmap(page, vma, address); 3214 setpte: 3215 set_pte_at(mm, address, page_table, entry); 3216 3217 /* No need to invalidate - it was non-present before */ 3218 update_mmu_cache(vma, address, page_table); 3219 unlock: 3220 pte_unmap_unlock(page_table, ptl); 3221 return 0; 3222 release: 3223 mem_cgroup_uncharge_page(page); 3224 page_cache_release(page); 3225 goto unlock; 3226 oom_free_page: 3227 page_cache_release(page); 3228 oom: 3229 return VM_FAULT_OOM; 3230 } 3231 3232 /* 3233 * __do_fault() tries to create a new page mapping. It aggressively 3234 * tries to share with existing pages, but makes a separate copy if 3235 * the FAULT_FLAG_WRITE is set in the flags parameter in order to avoid 3236 * the next page fault. 3237 * 3238 * As this is called only for pages that do not currently exist, we 3239 * do not need to flush old virtual caches or the TLB. 3240 * 3241 * We enter with non-exclusive mmap_sem (to exclude vma changes, 3242 * but allow concurrent faults), and pte neither mapped nor locked. 3243 * We return with mmap_sem still held, but pte unmapped and unlocked. 3244 */ 3245 static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma, 3246 unsigned long address, pmd_t *pmd, 3247 pgoff_t pgoff, unsigned int flags, pte_t orig_pte) 3248 { 3249 pte_t *page_table; 3250 spinlock_t *ptl; 3251 struct page *page; 3252 struct page *cow_page; 3253 pte_t entry; 3254 int anon = 0; 3255 struct page *dirty_page = NULL; 3256 struct vm_fault vmf; 3257 int ret; 3258 int page_mkwrite = 0; 3259 3260 /* 3261 * If we do COW later, allocate page befor taking lock_page() 3262 * on the file cache page. This will reduce lock holding time. 3263 */ 3264 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { 3265 3266 if (unlikely(anon_vma_prepare(vma))) 3267 return VM_FAULT_OOM; 3268 3269 cow_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address); 3270 if (!cow_page) 3271 return VM_FAULT_OOM; 3272 3273 if (mem_cgroup_newpage_charge(cow_page, mm, GFP_KERNEL)) { 3274 page_cache_release(cow_page); 3275 return VM_FAULT_OOM; 3276 } 3277 } else 3278 cow_page = NULL; 3279 3280 vmf.virtual_address = (void __user *)(address & PAGE_MASK); 3281 vmf.pgoff = pgoff; 3282 vmf.flags = flags; 3283 vmf.page = NULL; 3284 3285 ret = vma->vm_ops->fault(vma, &vmf); 3286 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | 3287 VM_FAULT_RETRY))) 3288 goto uncharge_out; 3289 3290 if (unlikely(PageHWPoison(vmf.page))) { 3291 if (ret & VM_FAULT_LOCKED) 3292 unlock_page(vmf.page); 3293 ret = VM_FAULT_HWPOISON; 3294 goto uncharge_out; 3295 } 3296 3297 /* 3298 * For consistency in subsequent calls, make the faulted page always 3299 * locked. 3300 */ 3301 if (unlikely(!(ret & VM_FAULT_LOCKED))) 3302 lock_page(vmf.page); 3303 else 3304 VM_BUG_ON(!PageLocked(vmf.page)); 3305 3306 /* 3307 * Should we do an early C-O-W break? 3308 */ 3309 page = vmf.page; 3310 if (flags & FAULT_FLAG_WRITE) { 3311 if (!(vma->vm_flags & VM_SHARED)) { 3312 page = cow_page; 3313 anon = 1; 3314 copy_user_highpage(page, vmf.page, address, vma); 3315 __SetPageUptodate(page); 3316 } else { 3317 /* 3318 * If the page will be shareable, see if the backing 3319 * address space wants to know that the page is about 3320 * to become writable 3321 */ 3322 if (vma->vm_ops->page_mkwrite) { 3323 int tmp; 3324 3325 unlock_page(page); 3326 vmf.flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE; 3327 tmp = vma->vm_ops->page_mkwrite(vma, &vmf); 3328 if (unlikely(tmp & 3329 (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) { 3330 ret = tmp; 3331 goto unwritable_page; 3332 } 3333 if (unlikely(!(tmp & VM_FAULT_LOCKED))) { 3334 lock_page(page); 3335 if (!page->mapping) { 3336 ret = 0; /* retry the fault */ 3337 unlock_page(page); 3338 goto unwritable_page; 3339 } 3340 } else 3341 VM_BUG_ON(!PageLocked(page)); 3342 page_mkwrite = 1; 3343 } 3344 } 3345 3346 } 3347 3348 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 3349 3350 /* 3351 * This silly early PAGE_DIRTY setting removes a race 3352 * due to the bad i386 page protection. But it's valid 3353 * for other architectures too. 3354 * 3355 * Note that if FAULT_FLAG_WRITE is set, we either now have 3356 * an exclusive copy of the page, or this is a shared mapping, 3357 * so we can make it writable and dirty to avoid having to 3358 * handle that later. 3359 */ 3360 /* Only go through if we didn't race with anybody else... */ 3361 if (likely(pte_same(*page_table, orig_pte))) { 3362 flush_icache_page(vma, page); 3363 entry = mk_pte(page, vma->vm_page_prot); 3364 if (flags & FAULT_FLAG_WRITE) 3365 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 3366 if (anon) { 3367 inc_mm_counter_fast(mm, MM_ANONPAGES); 3368 page_add_new_anon_rmap(page, vma, address); 3369 } else { 3370 inc_mm_counter_fast(mm, MM_FILEPAGES); 3371 page_add_file_rmap(page); 3372 if (flags & FAULT_FLAG_WRITE) { 3373 dirty_page = page; 3374 get_page(dirty_page); 3375 } 3376 } 3377 set_pte_at(mm, address, page_table, entry); 3378 3379 /* no need to invalidate: a not-present page won't be cached */ 3380 update_mmu_cache(vma, address, page_table); 3381 } else { 3382 if (cow_page) 3383 mem_cgroup_uncharge_page(cow_page); 3384 if (anon) 3385 page_cache_release(page); 3386 else 3387 anon = 1; /* no anon but release faulted_page */ 3388 } 3389 3390 pte_unmap_unlock(page_table, ptl); 3391 3392 if (dirty_page) { 3393 struct address_space *mapping = page->mapping; 3394 int dirtied = 0; 3395 3396 if (set_page_dirty(dirty_page)) 3397 dirtied = 1; 3398 unlock_page(dirty_page); 3399 put_page(dirty_page); 3400 if ((dirtied || page_mkwrite) && mapping) { 3401 /* 3402 * Some device drivers do not set page.mapping but still 3403 * dirty their pages 3404 */ 3405 balance_dirty_pages_ratelimited(mapping); 3406 } 3407 3408 /* file_update_time outside page_lock */ 3409 if (vma->vm_file && !page_mkwrite) 3410 file_update_time(vma->vm_file); 3411 } else { 3412 unlock_page(vmf.page); 3413 if (anon) 3414 page_cache_release(vmf.page); 3415 } 3416 3417 return ret; 3418 3419 unwritable_page: 3420 page_cache_release(page); 3421 return ret; 3422 uncharge_out: 3423 /* fs's fault handler get error */ 3424 if (cow_page) { 3425 mem_cgroup_uncharge_page(cow_page); 3426 page_cache_release(cow_page); 3427 } 3428 return ret; 3429 } 3430 3431 static int do_linear_fault(struct mm_struct *mm, struct vm_area_struct *vma, 3432 unsigned long address, pte_t *page_table, pmd_t *pmd, 3433 unsigned int flags, pte_t orig_pte) 3434 { 3435 pgoff_t pgoff = (((address & PAGE_MASK) 3436 - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; 3437 3438 pte_unmap(page_table); 3439 return __do_fault(mm, vma, address, pmd, pgoff, flags, orig_pte); 3440 } 3441 3442 /* 3443 * Fault of a previously existing named mapping. Repopulate the pte 3444 * from the encoded file_pte if possible. This enables swappable 3445 * nonlinear vmas. 3446 * 3447 * We enter with non-exclusive mmap_sem (to exclude vma changes, 3448 * but allow concurrent faults), and pte mapped but not yet locked. 3449 * We return with mmap_sem still held, but pte unmapped and unlocked. 3450 */ 3451 static int do_nonlinear_fault(struct mm_struct *mm, struct vm_area_struct *vma, 3452 unsigned long address, pte_t *page_table, pmd_t *pmd, 3453 unsigned int flags, pte_t orig_pte) 3454 { 3455 pgoff_t pgoff; 3456 3457 flags |= FAULT_FLAG_NONLINEAR; 3458 3459 if (!pte_unmap_same(mm, pmd, page_table, orig_pte)) 3460 return 0; 3461 3462 if (unlikely(!(vma->vm_flags & VM_NONLINEAR))) { 3463 /* 3464 * Page table corrupted: show pte and kill process. 3465 */ 3466 print_bad_pte(vma, address, orig_pte, NULL); 3467 return VM_FAULT_SIGBUS; 3468 } 3469 3470 pgoff = pte_to_pgoff(orig_pte); 3471 return __do_fault(mm, vma, address, pmd, pgoff, flags, orig_pte); 3472 } 3473 3474 int numa_migrate_prep(struct page *page, struct vm_area_struct *vma, 3475 unsigned long addr, int current_nid) 3476 { 3477 get_page(page); 3478 3479 count_vm_numa_event(NUMA_HINT_FAULTS); 3480 if (current_nid == numa_node_id()) 3481 count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL); 3482 3483 return mpol_misplaced(page, vma, addr); 3484 } 3485 3486 int do_numa_page(struct mm_struct *mm, struct vm_area_struct *vma, 3487 unsigned long addr, pte_t pte, pte_t *ptep, pmd_t *pmd) 3488 { 3489 struct page *page = NULL; 3490 spinlock_t *ptl; 3491 int current_nid = -1; 3492 int target_nid; 3493 bool migrated = false; 3494 3495 /* 3496 * The "pte" at this point cannot be used safely without 3497 * validation through pte_unmap_same(). It's of NUMA type but 3498 * the pfn may be screwed if the read is non atomic. 3499 * 3500 * ptep_modify_prot_start is not called as this is clearing 3501 * the _PAGE_NUMA bit and it is not really expected that there 3502 * would be concurrent hardware modifications to the PTE. 3503 */ 3504 ptl = pte_lockptr(mm, pmd); 3505 spin_lock(ptl); 3506 if (unlikely(!pte_same(*ptep, pte))) { 3507 pte_unmap_unlock(ptep, ptl); 3508 goto out; 3509 } 3510 3511 pte = pte_mknonnuma(pte); 3512 set_pte_at(mm, addr, ptep, pte); 3513 update_mmu_cache(vma, addr, ptep); 3514 3515 page = vm_normal_page(vma, addr, pte); 3516 if (!page) { 3517 pte_unmap_unlock(ptep, ptl); 3518 return 0; 3519 } 3520 3521 current_nid = page_to_nid(page); 3522 target_nid = numa_migrate_prep(page, vma, addr, current_nid); 3523 pte_unmap_unlock(ptep, ptl); 3524 if (target_nid == -1) { 3525 /* 3526 * Account for the fault against the current node if it not 3527 * being replaced regardless of where the page is located. 3528 */ 3529 current_nid = numa_node_id(); 3530 put_page(page); 3531 goto out; 3532 } 3533 3534 /* Migrate to the requested node */ 3535 migrated = migrate_misplaced_page(page, target_nid); 3536 if (migrated) 3537 current_nid = target_nid; 3538 3539 out: 3540 if (current_nid != -1) 3541 task_numa_fault(current_nid, 1, migrated); 3542 return 0; 3543 } 3544 3545 /* NUMA hinting page fault entry point for regular pmds */ 3546 #ifdef CONFIG_NUMA_BALANCING 3547 static int do_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma, 3548 unsigned long addr, pmd_t *pmdp) 3549 { 3550 pmd_t pmd; 3551 pte_t *pte, *orig_pte; 3552 unsigned long _addr = addr & PMD_MASK; 3553 unsigned long offset; 3554 spinlock_t *ptl; 3555 bool numa = false; 3556 int local_nid = numa_node_id(); 3557 3558 spin_lock(&mm->page_table_lock); 3559 pmd = *pmdp; 3560 if (pmd_numa(pmd)) { 3561 set_pmd_at(mm, _addr, pmdp, pmd_mknonnuma(pmd)); 3562 numa = true; 3563 } 3564 spin_unlock(&mm->page_table_lock); 3565 3566 if (!numa) 3567 return 0; 3568 3569 /* we're in a page fault so some vma must be in the range */ 3570 BUG_ON(!vma); 3571 BUG_ON(vma->vm_start >= _addr + PMD_SIZE); 3572 offset = max(_addr, vma->vm_start) & ~PMD_MASK; 3573 VM_BUG_ON(offset >= PMD_SIZE); 3574 orig_pte = pte = pte_offset_map_lock(mm, pmdp, _addr, &ptl); 3575 pte += offset >> PAGE_SHIFT; 3576 for (addr = _addr + offset; addr < _addr + PMD_SIZE; pte++, addr += PAGE_SIZE) { 3577 pte_t pteval = *pte; 3578 struct page *page; 3579 int curr_nid = local_nid; 3580 int target_nid; 3581 bool migrated; 3582 if (!pte_present(pteval)) 3583 continue; 3584 if (!pte_numa(pteval)) 3585 continue; 3586 if (addr >= vma->vm_end) { 3587 vma = find_vma(mm, addr); 3588 /* there's a pte present so there must be a vma */ 3589 BUG_ON(!vma); 3590 BUG_ON(addr < vma->vm_start); 3591 } 3592 if (pte_numa(pteval)) { 3593 pteval = pte_mknonnuma(pteval); 3594 set_pte_at(mm, addr, pte, pteval); 3595 } 3596 page = vm_normal_page(vma, addr, pteval); 3597 if (unlikely(!page)) 3598 continue; 3599 /* only check non-shared pages */ 3600 if (unlikely(page_mapcount(page) != 1)) 3601 continue; 3602 3603 /* 3604 * Note that the NUMA fault is later accounted to either 3605 * the node that is currently running or where the page is 3606 * migrated to. 3607 */ 3608 curr_nid = local_nid; 3609 target_nid = numa_migrate_prep(page, vma, addr, 3610 page_to_nid(page)); 3611 if (target_nid == -1) { 3612 put_page(page); 3613 continue; 3614 } 3615 3616 /* Migrate to the requested node */ 3617 pte_unmap_unlock(pte, ptl); 3618 migrated = migrate_misplaced_page(page, target_nid); 3619 if (migrated) 3620 curr_nid = target_nid; 3621 task_numa_fault(curr_nid, 1, migrated); 3622 3623 pte = pte_offset_map_lock(mm, pmdp, addr, &ptl); 3624 } 3625 pte_unmap_unlock(orig_pte, ptl); 3626 3627 return 0; 3628 } 3629 #else 3630 static int do_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma, 3631 unsigned long addr, pmd_t *pmdp) 3632 { 3633 BUG(); 3634 return 0; 3635 } 3636 #endif /* CONFIG_NUMA_BALANCING */ 3637 3638 /* 3639 * These routines also need to handle stuff like marking pages dirty 3640 * and/or accessed for architectures that don't do it in hardware (most 3641 * RISC architectures). The early dirtying is also good on the i386. 3642 * 3643 * There is also a hook called "update_mmu_cache()" that architectures 3644 * with external mmu caches can use to update those (ie the Sparc or 3645 * PowerPC hashed page tables that act as extended TLBs). 3646 * 3647 * We enter with non-exclusive mmap_sem (to exclude vma changes, 3648 * but allow concurrent faults), and pte mapped but not yet locked. 3649 * We return with mmap_sem still held, but pte unmapped and unlocked. 3650 */ 3651 int handle_pte_fault(struct mm_struct *mm, 3652 struct vm_area_struct *vma, unsigned long address, 3653 pte_t *pte, pmd_t *pmd, unsigned int flags) 3654 { 3655 pte_t entry; 3656 spinlock_t *ptl; 3657 3658 entry = *pte; 3659 if (!pte_present(entry)) { 3660 if (pte_none(entry)) { 3661 if (vma->vm_ops) { 3662 if (likely(vma->vm_ops->fault)) 3663 return do_linear_fault(mm, vma, address, 3664 pte, pmd, flags, entry); 3665 } 3666 return do_anonymous_page(mm, vma, address, 3667 pte, pmd, flags); 3668 } 3669 if (pte_file(entry)) 3670 return do_nonlinear_fault(mm, vma, address, 3671 pte, pmd, flags, entry); 3672 return do_swap_page(mm, vma, address, 3673 pte, pmd, flags, entry); 3674 } 3675 3676 if (pte_numa(entry)) 3677 return do_numa_page(mm, vma, address, entry, pte, pmd); 3678 3679 ptl = pte_lockptr(mm, pmd); 3680 spin_lock(ptl); 3681 if (unlikely(!pte_same(*pte, entry))) 3682 goto unlock; 3683 if (flags & FAULT_FLAG_WRITE) { 3684 if (!pte_write(entry)) 3685 return do_wp_page(mm, vma, address, 3686 pte, pmd, ptl, entry); 3687 entry = pte_mkdirty(entry); 3688 } 3689 entry = pte_mkyoung(entry); 3690 if (ptep_set_access_flags(vma, address, pte, entry, flags & FAULT_FLAG_WRITE)) { 3691 update_mmu_cache(vma, address, pte); 3692 } else { 3693 /* 3694 * This is needed only for protection faults but the arch code 3695 * is not yet telling us if this is a protection fault or not. 3696 * This still avoids useless tlb flushes for .text page faults 3697 * with threads. 3698 */ 3699 if (flags & FAULT_FLAG_WRITE) 3700 flush_tlb_fix_spurious_fault(vma, address); 3701 } 3702 unlock: 3703 pte_unmap_unlock(pte, ptl); 3704 return 0; 3705 } 3706 3707 /* 3708 * By the time we get here, we already hold the mm semaphore 3709 */ 3710 int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma, 3711 unsigned long address, unsigned int flags) 3712 { 3713 pgd_t *pgd; 3714 pud_t *pud; 3715 pmd_t *pmd; 3716 pte_t *pte; 3717 3718 __set_current_state(TASK_RUNNING); 3719 3720 count_vm_event(PGFAULT); 3721 mem_cgroup_count_vm_event(mm, PGFAULT); 3722 3723 /* do counter updates before entering really critical section. */ 3724 check_sync_rss_stat(current); 3725 3726 if (unlikely(is_vm_hugetlb_page(vma))) 3727 return hugetlb_fault(mm, vma, address, flags); 3728 3729 retry: 3730 pgd = pgd_offset(mm, address); 3731 pud = pud_alloc(mm, pgd, address); 3732 if (!pud) 3733 return VM_FAULT_OOM; 3734 pmd = pmd_alloc(mm, pud, address); 3735 if (!pmd) 3736 return VM_FAULT_OOM; 3737 if (pmd_none(*pmd) && transparent_hugepage_enabled(vma)) { 3738 if (!vma->vm_ops) 3739 return do_huge_pmd_anonymous_page(mm, vma, address, 3740 pmd, flags); 3741 } else { 3742 pmd_t orig_pmd = *pmd; 3743 int ret; 3744 3745 barrier(); 3746 if (pmd_trans_huge(orig_pmd)) { 3747 unsigned int dirty = flags & FAULT_FLAG_WRITE; 3748 3749 /* 3750 * If the pmd is splitting, return and retry the 3751 * the fault. Alternative: wait until the split 3752 * is done, and goto retry. 3753 */ 3754 if (pmd_trans_splitting(orig_pmd)) 3755 return 0; 3756 3757 if (pmd_numa(orig_pmd)) 3758 return do_huge_pmd_numa_page(mm, vma, address, 3759 orig_pmd, pmd); 3760 3761 if (dirty && !pmd_write(orig_pmd)) { 3762 ret = do_huge_pmd_wp_page(mm, vma, address, pmd, 3763 orig_pmd); 3764 /* 3765 * If COW results in an oom, the huge pmd will 3766 * have been split, so retry the fault on the 3767 * pte for a smaller charge. 3768 */ 3769 if (unlikely(ret & VM_FAULT_OOM)) 3770 goto retry; 3771 return ret; 3772 } else { 3773 huge_pmd_set_accessed(mm, vma, address, pmd, 3774 orig_pmd, dirty); 3775 } 3776 3777 return 0; 3778 } 3779 } 3780 3781 if (pmd_numa(*pmd)) 3782 return do_pmd_numa_page(mm, vma, address, pmd); 3783 3784 /* 3785 * Use __pte_alloc instead of pte_alloc_map, because we can't 3786 * run pte_offset_map on the pmd, if an huge pmd could 3787 * materialize from under us from a different thread. 3788 */ 3789 if (unlikely(pmd_none(*pmd)) && 3790 unlikely(__pte_alloc(mm, vma, pmd, address))) 3791 return VM_FAULT_OOM; 3792 /* if an huge pmd materialized from under us just retry later */ 3793 if (unlikely(pmd_trans_huge(*pmd))) 3794 return 0; 3795 /* 3796 * A regular pmd is established and it can't morph into a huge pmd 3797 * from under us anymore at this point because we hold the mmap_sem 3798 * read mode and khugepaged takes it in write mode. So now it's 3799 * safe to run pte_offset_map(). 3800 */ 3801 pte = pte_offset_map(pmd, address); 3802 3803 return handle_pte_fault(mm, vma, address, pte, pmd, flags); 3804 } 3805 3806 #ifndef __PAGETABLE_PUD_FOLDED 3807 /* 3808 * Allocate page upper directory. 3809 * We've already handled the fast-path in-line. 3810 */ 3811 int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address) 3812 { 3813 pud_t *new = pud_alloc_one(mm, address); 3814 if (!new) 3815 return -ENOMEM; 3816 3817 smp_wmb(); /* See comment in __pte_alloc */ 3818 3819 spin_lock(&mm->page_table_lock); 3820 if (pgd_present(*pgd)) /* Another has populated it */ 3821 pud_free(mm, new); 3822 else 3823 pgd_populate(mm, pgd, new); 3824 spin_unlock(&mm->page_table_lock); 3825 return 0; 3826 } 3827 #endif /* __PAGETABLE_PUD_FOLDED */ 3828 3829 #ifndef __PAGETABLE_PMD_FOLDED 3830 /* 3831 * Allocate page middle directory. 3832 * We've already handled the fast-path in-line. 3833 */ 3834 int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) 3835 { 3836 pmd_t *new = pmd_alloc_one(mm, address); 3837 if (!new) 3838 return -ENOMEM; 3839 3840 smp_wmb(); /* See comment in __pte_alloc */ 3841 3842 spin_lock(&mm->page_table_lock); 3843 #ifndef __ARCH_HAS_4LEVEL_HACK 3844 if (pud_present(*pud)) /* Another has populated it */ 3845 pmd_free(mm, new); 3846 else 3847 pud_populate(mm, pud, new); 3848 #else 3849 if (pgd_present(*pud)) /* Another has populated it */ 3850 pmd_free(mm, new); 3851 else 3852 pgd_populate(mm, pud, new); 3853 #endif /* __ARCH_HAS_4LEVEL_HACK */ 3854 spin_unlock(&mm->page_table_lock); 3855 return 0; 3856 } 3857 #endif /* __PAGETABLE_PMD_FOLDED */ 3858 3859 #if !defined(__HAVE_ARCH_GATE_AREA) 3860 3861 #if defined(AT_SYSINFO_EHDR) 3862 static struct vm_area_struct gate_vma; 3863 3864 static int __init gate_vma_init(void) 3865 { 3866 gate_vma.vm_mm = NULL; 3867 gate_vma.vm_start = FIXADDR_USER_START; 3868 gate_vma.vm_end = FIXADDR_USER_END; 3869 gate_vma.vm_flags = VM_READ | VM_MAYREAD | VM_EXEC | VM_MAYEXEC; 3870 gate_vma.vm_page_prot = __P101; 3871 3872 return 0; 3873 } 3874 __initcall(gate_vma_init); 3875 #endif 3876 3877 struct vm_area_struct *get_gate_vma(struct mm_struct *mm) 3878 { 3879 #ifdef AT_SYSINFO_EHDR 3880 return &gate_vma; 3881 #else 3882 return NULL; 3883 #endif 3884 } 3885 3886 int in_gate_area_no_mm(unsigned long addr) 3887 { 3888 #ifdef AT_SYSINFO_EHDR 3889 if ((addr >= FIXADDR_USER_START) && (addr < FIXADDR_USER_END)) 3890 return 1; 3891 #endif 3892 return 0; 3893 } 3894 3895 #endif /* __HAVE_ARCH_GATE_AREA */ 3896 3897 static int __follow_pte(struct mm_struct *mm, unsigned long address, 3898 pte_t **ptepp, spinlock_t **ptlp) 3899 { 3900 pgd_t *pgd; 3901 pud_t *pud; 3902 pmd_t *pmd; 3903 pte_t *ptep; 3904 3905 pgd = pgd_offset(mm, address); 3906 if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) 3907 goto out; 3908 3909 pud = pud_offset(pgd, address); 3910 if (pud_none(*pud) || unlikely(pud_bad(*pud))) 3911 goto out; 3912 3913 pmd = pmd_offset(pud, address); 3914 VM_BUG_ON(pmd_trans_huge(*pmd)); 3915 if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd))) 3916 goto out; 3917 3918 /* We cannot handle huge page PFN maps. Luckily they don't exist. */ 3919 if (pmd_huge(*pmd)) 3920 goto out; 3921 3922 ptep = pte_offset_map_lock(mm, pmd, address, ptlp); 3923 if (!ptep) 3924 goto out; 3925 if (!pte_present(*ptep)) 3926 goto unlock; 3927 *ptepp = ptep; 3928 return 0; 3929 unlock: 3930 pte_unmap_unlock(ptep, *ptlp); 3931 out: 3932 return -EINVAL; 3933 } 3934 3935 static inline int follow_pte(struct mm_struct *mm, unsigned long address, 3936 pte_t **ptepp, spinlock_t **ptlp) 3937 { 3938 int res; 3939 3940 /* (void) is needed to make gcc happy */ 3941 (void) __cond_lock(*ptlp, 3942 !(res = __follow_pte(mm, address, ptepp, ptlp))); 3943 return res; 3944 } 3945 3946 /** 3947 * follow_pfn - look up PFN at a user virtual address 3948 * @vma: memory mapping 3949 * @address: user virtual address 3950 * @pfn: location to store found PFN 3951 * 3952 * Only IO mappings and raw PFN mappings are allowed. 3953 * 3954 * Returns zero and the pfn at @pfn on success, -ve otherwise. 3955 */ 3956 int follow_pfn(struct vm_area_struct *vma, unsigned long address, 3957 unsigned long *pfn) 3958 { 3959 int ret = -EINVAL; 3960 spinlock_t *ptl; 3961 pte_t *ptep; 3962 3963 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) 3964 return ret; 3965 3966 ret = follow_pte(vma->vm_mm, address, &ptep, &ptl); 3967 if (ret) 3968 return ret; 3969 *pfn = pte_pfn(*ptep); 3970 pte_unmap_unlock(ptep, ptl); 3971 return 0; 3972 } 3973 EXPORT_SYMBOL(follow_pfn); 3974 3975 #ifdef CONFIG_HAVE_IOREMAP_PROT 3976 int follow_phys(struct vm_area_struct *vma, 3977 unsigned long address, unsigned int flags, 3978 unsigned long *prot, resource_size_t *phys) 3979 { 3980 int ret = -EINVAL; 3981 pte_t *ptep, pte; 3982 spinlock_t *ptl; 3983 3984 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) 3985 goto out; 3986 3987 if (follow_pte(vma->vm_mm, address, &ptep, &ptl)) 3988 goto out; 3989 pte = *ptep; 3990 3991 if ((flags & FOLL_WRITE) && !pte_write(pte)) 3992 goto unlock; 3993 3994 *prot = pgprot_val(pte_pgprot(pte)); 3995 *phys = (resource_size_t)pte_pfn(pte) << PAGE_SHIFT; 3996 3997 ret = 0; 3998 unlock: 3999 pte_unmap_unlock(ptep, ptl); 4000 out: 4001 return ret; 4002 } 4003 4004 int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, 4005 void *buf, int len, int write) 4006 { 4007 resource_size_t phys_addr; 4008 unsigned long prot = 0; 4009 void __iomem *maddr; 4010 int offset = addr & (PAGE_SIZE-1); 4011 4012 if (follow_phys(vma, addr, write, &prot, &phys_addr)) 4013 return -EINVAL; 4014 4015 maddr = ioremap_prot(phys_addr, PAGE_SIZE, prot); 4016 if (write) 4017 memcpy_toio(maddr + offset, buf, len); 4018 else 4019 memcpy_fromio(buf, maddr + offset, len); 4020 iounmap(maddr); 4021 4022 return len; 4023 } 4024 #endif 4025 4026 /* 4027 * Access another process' address space as given in mm. If non-NULL, use the 4028 * given task for page fault accounting. 4029 */ 4030 static int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm, 4031 unsigned long addr, void *buf, int len, int write) 4032 { 4033 struct vm_area_struct *vma; 4034 void *old_buf = buf; 4035 4036 down_read(&mm->mmap_sem); 4037 /* ignore errors, just check how much was successfully transferred */ 4038 while (len) { 4039 int bytes, ret, offset; 4040 void *maddr; 4041 struct page *page = NULL; 4042 4043 ret = get_user_pages(tsk, mm, addr, 1, 4044 write, 1, &page, &vma); 4045 if (ret <= 0) { 4046 /* 4047 * Check if this is a VM_IO | VM_PFNMAP VMA, which 4048 * we can access using slightly different code. 4049 */ 4050 #ifdef CONFIG_HAVE_IOREMAP_PROT 4051 vma = find_vma(mm, addr); 4052 if (!vma || vma->vm_start > addr) 4053 break; 4054 if (vma->vm_ops && vma->vm_ops->access) 4055 ret = vma->vm_ops->access(vma, addr, buf, 4056 len, write); 4057 if (ret <= 0) 4058 #endif 4059 break; 4060 bytes = ret; 4061 } else { 4062 bytes = len; 4063 offset = addr & (PAGE_SIZE-1); 4064 if (bytes > PAGE_SIZE-offset) 4065 bytes = PAGE_SIZE-offset; 4066 4067 maddr = kmap(page); 4068 if (write) { 4069 copy_to_user_page(vma, page, addr, 4070 maddr + offset, buf, bytes); 4071 set_page_dirty_lock(page); 4072 } else { 4073 copy_from_user_page(vma, page, addr, 4074 buf, maddr + offset, bytes); 4075 } 4076 kunmap(page); 4077 page_cache_release(page); 4078 } 4079 len -= bytes; 4080 buf += bytes; 4081 addr += bytes; 4082 } 4083 up_read(&mm->mmap_sem); 4084 4085 return buf - old_buf; 4086 } 4087 4088 /** 4089 * access_remote_vm - access another process' address space 4090 * @mm: the mm_struct of the target address space 4091 * @addr: start address to access 4092 * @buf: source or destination buffer 4093 * @len: number of bytes to transfer 4094 * @write: whether the access is a write 4095 * 4096 * The caller must hold a reference on @mm. 4097 */ 4098 int access_remote_vm(struct mm_struct *mm, unsigned long addr, 4099 void *buf, int len, int write) 4100 { 4101 return __access_remote_vm(NULL, mm, addr, buf, len, write); 4102 } 4103 4104 /* 4105 * Access another process' address space. 4106 * Source/target buffer must be kernel space, 4107 * Do not walk the page table directly, use get_user_pages 4108 */ 4109 int access_process_vm(struct task_struct *tsk, unsigned long addr, 4110 void *buf, int len, int write) 4111 { 4112 struct mm_struct *mm; 4113 int ret; 4114 4115 mm = get_task_mm(tsk); 4116 if (!mm) 4117 return 0; 4118 4119 ret = __access_remote_vm(tsk, mm, addr, buf, len, write); 4120 mmput(mm); 4121 4122 return ret; 4123 } 4124 4125 /* 4126 * Print the name of a VMA. 4127 */ 4128 void print_vma_addr(char *prefix, unsigned long ip) 4129 { 4130 struct mm_struct *mm = current->mm; 4131 struct vm_area_struct *vma; 4132 4133 /* 4134 * Do not print if we are in atomic 4135 * contexts (in exception stacks, etc.): 4136 */ 4137 if (preempt_count()) 4138 return; 4139 4140 down_read(&mm->mmap_sem); 4141 vma = find_vma(mm, ip); 4142 if (vma && vma->vm_file) { 4143 struct file *f = vma->vm_file; 4144 char *buf = (char *)__get_free_page(GFP_KERNEL); 4145 if (buf) { 4146 char *p; 4147 4148 p = d_path(&f->f_path, buf, PAGE_SIZE); 4149 if (IS_ERR(p)) 4150 p = "?"; 4151 printk("%s%s[%lx+%lx]", prefix, kbasename(p), 4152 vma->vm_start, 4153 vma->vm_end - vma->vm_start); 4154 free_page((unsigned long)buf); 4155 } 4156 } 4157 up_read(&mm->mmap_sem); 4158 } 4159 4160 #ifdef CONFIG_PROVE_LOCKING 4161 void might_fault(void) 4162 { 4163 /* 4164 * Some code (nfs/sunrpc) uses socket ops on kernel memory while 4165 * holding the mmap_sem, this is safe because kernel memory doesn't 4166 * get paged out, therefore we'll never actually fault, and the 4167 * below annotations will generate false positives. 4168 */ 4169 if (segment_eq(get_fs(), KERNEL_DS)) 4170 return; 4171 4172 might_sleep(); 4173 /* 4174 * it would be nicer only to annotate paths which are not under 4175 * pagefault_disable, however that requires a larger audit and 4176 * providing helpers like get_user_atomic. 4177 */ 4178 if (!in_atomic() && current->mm) 4179 might_lock_read(¤t->mm->mmap_sem); 4180 } 4181 EXPORT_SYMBOL(might_fault); 4182 #endif 4183 4184 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS) 4185 static void clear_gigantic_page(struct page *page, 4186 unsigned long addr, 4187 unsigned int pages_per_huge_page) 4188 { 4189 int i; 4190 struct page *p = page; 4191 4192 might_sleep(); 4193 for (i = 0; i < pages_per_huge_page; 4194 i++, p = mem_map_next(p, page, i)) { 4195 cond_resched(); 4196 clear_user_highpage(p, addr + i * PAGE_SIZE); 4197 } 4198 } 4199 void clear_huge_page(struct page *page, 4200 unsigned long addr, unsigned int pages_per_huge_page) 4201 { 4202 int i; 4203 4204 if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) { 4205 clear_gigantic_page(page, addr, pages_per_huge_page); 4206 return; 4207 } 4208 4209 might_sleep(); 4210 for (i = 0; i < pages_per_huge_page; i++) { 4211 cond_resched(); 4212 clear_user_highpage(page + i, addr + i * PAGE_SIZE); 4213 } 4214 } 4215 4216 static void copy_user_gigantic_page(struct page *dst, struct page *src, 4217 unsigned long addr, 4218 struct vm_area_struct *vma, 4219 unsigned int pages_per_huge_page) 4220 { 4221 int i; 4222 struct page *dst_base = dst; 4223 struct page *src_base = src; 4224 4225 for (i = 0; i < pages_per_huge_page; ) { 4226 cond_resched(); 4227 copy_user_highpage(dst, src, addr + i*PAGE_SIZE, vma); 4228 4229 i++; 4230 dst = mem_map_next(dst, dst_base, i); 4231 src = mem_map_next(src, src_base, i); 4232 } 4233 } 4234 4235 void copy_user_huge_page(struct page *dst, struct page *src, 4236 unsigned long addr, struct vm_area_struct *vma, 4237 unsigned int pages_per_huge_page) 4238 { 4239 int i; 4240 4241 if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) { 4242 copy_user_gigantic_page(dst, src, addr, vma, 4243 pages_per_huge_page); 4244 return; 4245 } 4246 4247 might_sleep(); 4248 for (i = 0; i < pages_per_huge_page; i++) { 4249 cond_resched(); 4250 copy_user_highpage(dst + i, src + i, addr + i*PAGE_SIZE, vma); 4251 } 4252 } 4253 #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */ 4254