1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/mm/memory.c 4 * 5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 6 */ 7 8 /* 9 * demand-loading started 01.12.91 - seems it is high on the list of 10 * things wanted, and it should be easy to implement. - Linus 11 */ 12 13 /* 14 * Ok, demand-loading was easy, shared pages a little bit tricker. Shared 15 * pages started 02.12.91, seems to work. - Linus. 16 * 17 * Tested sharing by executing about 30 /bin/sh: under the old kernel it 18 * would have taken more than the 6M I have free, but it worked well as 19 * far as I could see. 20 * 21 * Also corrected some "invalidate()"s - I wasn't doing enough of them. 22 */ 23 24 /* 25 * Real VM (paging to/from disk) started 18.12.91. Much more work and 26 * thought has to go into this. Oh, well.. 27 * 19.12.91 - works, somewhat. Sometimes I get faults, don't know why. 28 * Found it. Everything seems to work now. 29 * 20.12.91 - Ok, making the swap-device changeable like the root. 30 */ 31 32 /* 33 * 05.04.94 - Multi-page memory management added for v1.1. 34 * Idea by Alex Bligh (alex@cconcepts.co.uk) 35 * 36 * 16.07.99 - Support of BIGMEM added by Gerhard Wichert, Siemens AG 37 * (Gerhard.Wichert@pdb.siemens.de) 38 * 39 * Aug/Sep 2004 Changed to four level page tables (Andi Kleen) 40 */ 41 42 #include <linux/kernel_stat.h> 43 #include <linux/mm.h> 44 #include <linux/sched/mm.h> 45 #include <linux/sched/coredump.h> 46 #include <linux/sched/numa_balancing.h> 47 #include <linux/sched/task.h> 48 #include <linux/hugetlb.h> 49 #include <linux/mman.h> 50 #include <linux/swap.h> 51 #include <linux/highmem.h> 52 #include <linux/pagemap.h> 53 #include <linux/memremap.h> 54 #include <linux/ksm.h> 55 #include <linux/rmap.h> 56 #include <linux/export.h> 57 #include <linux/delayacct.h> 58 #include <linux/init.h> 59 #include <linux/pfn_t.h> 60 #include <linux/writeback.h> 61 #include <linux/memcontrol.h> 62 #include <linux/mmu_notifier.h> 63 #include <linux/swapops.h> 64 #include <linux/elf.h> 65 #include <linux/gfp.h> 66 #include <linux/migrate.h> 67 #include <linux/string.h> 68 #include <linux/dma-debug.h> 69 #include <linux/debugfs.h> 70 #include <linux/userfaultfd_k.h> 71 #include <linux/dax.h> 72 #include <linux/oom.h> 73 #include <linux/numa.h> 74 75 #include <trace/events/kmem.h> 76 77 #include <asm/io.h> 78 #include <asm/mmu_context.h> 79 #include <asm/pgalloc.h> 80 #include <linux/uaccess.h> 81 #include <asm/tlb.h> 82 #include <asm/tlbflush.h> 83 #include <asm/pgtable.h> 84 85 #include "internal.h" 86 87 #if defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) && !defined(CONFIG_COMPILE_TEST) 88 #warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid. 89 #endif 90 91 #ifndef CONFIG_NEED_MULTIPLE_NODES 92 /* use the per-pgdat data instead for discontigmem - mbligh */ 93 unsigned long max_mapnr; 94 EXPORT_SYMBOL(max_mapnr); 95 96 struct page *mem_map; 97 EXPORT_SYMBOL(mem_map); 98 #endif 99 100 /* 101 * A number of key systems in x86 including ioremap() rely on the assumption 102 * that high_memory defines the upper bound on direct map memory, then end 103 * of ZONE_NORMAL. Under CONFIG_DISCONTIG this means that max_low_pfn and 104 * highstart_pfn must be the same; there must be no gap between ZONE_NORMAL 105 * and ZONE_HIGHMEM. 106 */ 107 void *high_memory; 108 EXPORT_SYMBOL(high_memory); 109 110 /* 111 * Randomize the address space (stacks, mmaps, brk, etc.). 112 * 113 * ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization, 114 * as ancient (libc5 based) binaries can segfault. ) 115 */ 116 int randomize_va_space __read_mostly = 117 #ifdef CONFIG_COMPAT_BRK 118 1; 119 #else 120 2; 121 #endif 122 123 #ifndef arch_faults_on_old_pte 124 static inline bool arch_faults_on_old_pte(void) 125 { 126 /* 127 * Those arches which don't have hw access flag feature need to 128 * implement their own helper. By default, "true" means pagefault 129 * will be hit on old pte. 130 */ 131 return true; 132 } 133 #endif 134 135 static int __init disable_randmaps(char *s) 136 { 137 randomize_va_space = 0; 138 return 1; 139 } 140 __setup("norandmaps", disable_randmaps); 141 142 unsigned long zero_pfn __read_mostly; 143 EXPORT_SYMBOL(zero_pfn); 144 145 unsigned long highest_memmap_pfn __read_mostly; 146 147 /* 148 * CONFIG_MMU architectures set up ZERO_PAGE in their paging_init() 149 */ 150 static int __init init_zero_pfn(void) 151 { 152 zero_pfn = page_to_pfn(ZERO_PAGE(0)); 153 return 0; 154 } 155 core_initcall(init_zero_pfn); 156 157 void mm_trace_rss_stat(struct mm_struct *mm, int member, long count) 158 { 159 trace_rss_stat(mm, member, count); 160 } 161 162 #if defined(SPLIT_RSS_COUNTING) 163 164 void sync_mm_rss(struct mm_struct *mm) 165 { 166 int i; 167 168 for (i = 0; i < NR_MM_COUNTERS; i++) { 169 if (current->rss_stat.count[i]) { 170 add_mm_counter(mm, i, current->rss_stat.count[i]); 171 current->rss_stat.count[i] = 0; 172 } 173 } 174 current->rss_stat.events = 0; 175 } 176 177 static void add_mm_counter_fast(struct mm_struct *mm, int member, int val) 178 { 179 struct task_struct *task = current; 180 181 if (likely(task->mm == mm)) 182 task->rss_stat.count[member] += val; 183 else 184 add_mm_counter(mm, member, val); 185 } 186 #define inc_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, 1) 187 #define dec_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, -1) 188 189 /* sync counter once per 64 page faults */ 190 #define TASK_RSS_EVENTS_THRESH (64) 191 static void check_sync_rss_stat(struct task_struct *task) 192 { 193 if (unlikely(task != current)) 194 return; 195 if (unlikely(task->rss_stat.events++ > TASK_RSS_EVENTS_THRESH)) 196 sync_mm_rss(task->mm); 197 } 198 #else /* SPLIT_RSS_COUNTING */ 199 200 #define inc_mm_counter_fast(mm, member) inc_mm_counter(mm, member) 201 #define dec_mm_counter_fast(mm, member) dec_mm_counter(mm, member) 202 203 static void check_sync_rss_stat(struct task_struct *task) 204 { 205 } 206 207 #endif /* SPLIT_RSS_COUNTING */ 208 209 /* 210 * Note: this doesn't free the actual pages themselves. That 211 * has been handled earlier when unmapping all the memory regions. 212 */ 213 static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd, 214 unsigned long addr) 215 { 216 pgtable_t token = pmd_pgtable(*pmd); 217 pmd_clear(pmd); 218 pte_free_tlb(tlb, token, addr); 219 mm_dec_nr_ptes(tlb->mm); 220 } 221 222 static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud, 223 unsigned long addr, unsigned long end, 224 unsigned long floor, unsigned long ceiling) 225 { 226 pmd_t *pmd; 227 unsigned long next; 228 unsigned long start; 229 230 start = addr; 231 pmd = pmd_offset(pud, addr); 232 do { 233 next = pmd_addr_end(addr, end); 234 if (pmd_none_or_clear_bad(pmd)) 235 continue; 236 free_pte_range(tlb, pmd, addr); 237 } while (pmd++, addr = next, addr != end); 238 239 start &= PUD_MASK; 240 if (start < floor) 241 return; 242 if (ceiling) { 243 ceiling &= PUD_MASK; 244 if (!ceiling) 245 return; 246 } 247 if (end - 1 > ceiling - 1) 248 return; 249 250 pmd = pmd_offset(pud, start); 251 pud_clear(pud); 252 pmd_free_tlb(tlb, pmd, start); 253 mm_dec_nr_pmds(tlb->mm); 254 } 255 256 static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d, 257 unsigned long addr, unsigned long end, 258 unsigned long floor, unsigned long ceiling) 259 { 260 pud_t *pud; 261 unsigned long next; 262 unsigned long start; 263 264 start = addr; 265 pud = pud_offset(p4d, addr); 266 do { 267 next = pud_addr_end(addr, end); 268 if (pud_none_or_clear_bad(pud)) 269 continue; 270 free_pmd_range(tlb, pud, addr, next, floor, ceiling); 271 } while (pud++, addr = next, addr != end); 272 273 start &= P4D_MASK; 274 if (start < floor) 275 return; 276 if (ceiling) { 277 ceiling &= P4D_MASK; 278 if (!ceiling) 279 return; 280 } 281 if (end - 1 > ceiling - 1) 282 return; 283 284 pud = pud_offset(p4d, start); 285 p4d_clear(p4d); 286 pud_free_tlb(tlb, pud, start); 287 mm_dec_nr_puds(tlb->mm); 288 } 289 290 static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd, 291 unsigned long addr, unsigned long end, 292 unsigned long floor, unsigned long ceiling) 293 { 294 p4d_t *p4d; 295 unsigned long next; 296 unsigned long start; 297 298 start = addr; 299 p4d = p4d_offset(pgd, addr); 300 do { 301 next = p4d_addr_end(addr, end); 302 if (p4d_none_or_clear_bad(p4d)) 303 continue; 304 free_pud_range(tlb, p4d, addr, next, floor, ceiling); 305 } while (p4d++, addr = next, addr != end); 306 307 start &= PGDIR_MASK; 308 if (start < floor) 309 return; 310 if (ceiling) { 311 ceiling &= PGDIR_MASK; 312 if (!ceiling) 313 return; 314 } 315 if (end - 1 > ceiling - 1) 316 return; 317 318 p4d = p4d_offset(pgd, start); 319 pgd_clear(pgd); 320 p4d_free_tlb(tlb, p4d, start); 321 } 322 323 /* 324 * This function frees user-level page tables of a process. 325 */ 326 void free_pgd_range(struct mmu_gather *tlb, 327 unsigned long addr, unsigned long end, 328 unsigned long floor, unsigned long ceiling) 329 { 330 pgd_t *pgd; 331 unsigned long next; 332 333 /* 334 * The next few lines have given us lots of grief... 335 * 336 * Why are we testing PMD* at this top level? Because often 337 * there will be no work to do at all, and we'd prefer not to 338 * go all the way down to the bottom just to discover that. 339 * 340 * Why all these "- 1"s? Because 0 represents both the bottom 341 * of the address space and the top of it (using -1 for the 342 * top wouldn't help much: the masks would do the wrong thing). 343 * The rule is that addr 0 and floor 0 refer to the bottom of 344 * the address space, but end 0 and ceiling 0 refer to the top 345 * Comparisons need to use "end - 1" and "ceiling - 1" (though 346 * that end 0 case should be mythical). 347 * 348 * Wherever addr is brought up or ceiling brought down, we must 349 * be careful to reject "the opposite 0" before it confuses the 350 * subsequent tests. But what about where end is brought down 351 * by PMD_SIZE below? no, end can't go down to 0 there. 352 * 353 * Whereas we round start (addr) and ceiling down, by different 354 * masks at different levels, in order to test whether a table 355 * now has no other vmas using it, so can be freed, we don't 356 * bother to round floor or end up - the tests don't need that. 357 */ 358 359 addr &= PMD_MASK; 360 if (addr < floor) { 361 addr += PMD_SIZE; 362 if (!addr) 363 return; 364 } 365 if (ceiling) { 366 ceiling &= PMD_MASK; 367 if (!ceiling) 368 return; 369 } 370 if (end - 1 > ceiling - 1) 371 end -= PMD_SIZE; 372 if (addr > end - 1) 373 return; 374 /* 375 * We add page table cache pages with PAGE_SIZE, 376 * (see pte_free_tlb()), flush the tlb if we need 377 */ 378 tlb_change_page_size(tlb, PAGE_SIZE); 379 pgd = pgd_offset(tlb->mm, addr); 380 do { 381 next = pgd_addr_end(addr, end); 382 if (pgd_none_or_clear_bad(pgd)) 383 continue; 384 free_p4d_range(tlb, pgd, addr, next, floor, ceiling); 385 } while (pgd++, addr = next, addr != end); 386 } 387 388 void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, 389 unsigned long floor, unsigned long ceiling) 390 { 391 while (vma) { 392 struct vm_area_struct *next = vma->vm_next; 393 unsigned long addr = vma->vm_start; 394 395 /* 396 * Hide vma from rmap and truncate_pagecache before freeing 397 * pgtables 398 */ 399 unlink_anon_vmas(vma); 400 unlink_file_vma(vma); 401 402 if (is_vm_hugetlb_page(vma)) { 403 hugetlb_free_pgd_range(tlb, addr, vma->vm_end, 404 floor, next ? next->vm_start : ceiling); 405 } else { 406 /* 407 * Optimization: gather nearby vmas into one call down 408 */ 409 while (next && next->vm_start <= vma->vm_end + PMD_SIZE 410 && !is_vm_hugetlb_page(next)) { 411 vma = next; 412 next = vma->vm_next; 413 unlink_anon_vmas(vma); 414 unlink_file_vma(vma); 415 } 416 free_pgd_range(tlb, addr, vma->vm_end, 417 floor, next ? next->vm_start : ceiling); 418 } 419 vma = next; 420 } 421 } 422 423 int __pte_alloc(struct mm_struct *mm, pmd_t *pmd) 424 { 425 spinlock_t *ptl; 426 pgtable_t new = pte_alloc_one(mm); 427 if (!new) 428 return -ENOMEM; 429 430 /* 431 * Ensure all pte setup (eg. pte page lock and page clearing) are 432 * visible before the pte is made visible to other CPUs by being 433 * put into page tables. 434 * 435 * The other side of the story is the pointer chasing in the page 436 * table walking code (when walking the page table without locking; 437 * ie. most of the time). Fortunately, these data accesses consist 438 * of a chain of data-dependent loads, meaning most CPUs (alpha 439 * being the notable exception) will already guarantee loads are 440 * seen in-order. See the alpha page table accessors for the 441 * smp_read_barrier_depends() barriers in page table walking code. 442 */ 443 smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */ 444 445 ptl = pmd_lock(mm, pmd); 446 if (likely(pmd_none(*pmd))) { /* Has another populated it ? */ 447 mm_inc_nr_ptes(mm); 448 pmd_populate(mm, pmd, new); 449 new = NULL; 450 } 451 spin_unlock(ptl); 452 if (new) 453 pte_free(mm, new); 454 return 0; 455 } 456 457 int __pte_alloc_kernel(pmd_t *pmd) 458 { 459 pte_t *new = pte_alloc_one_kernel(&init_mm); 460 if (!new) 461 return -ENOMEM; 462 463 smp_wmb(); /* See comment in __pte_alloc */ 464 465 spin_lock(&init_mm.page_table_lock); 466 if (likely(pmd_none(*pmd))) { /* Has another populated it ? */ 467 pmd_populate_kernel(&init_mm, pmd, new); 468 new = NULL; 469 } 470 spin_unlock(&init_mm.page_table_lock); 471 if (new) 472 pte_free_kernel(&init_mm, new); 473 return 0; 474 } 475 476 static inline void init_rss_vec(int *rss) 477 { 478 memset(rss, 0, sizeof(int) * NR_MM_COUNTERS); 479 } 480 481 static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss) 482 { 483 int i; 484 485 if (current->mm == mm) 486 sync_mm_rss(mm); 487 for (i = 0; i < NR_MM_COUNTERS; i++) 488 if (rss[i]) 489 add_mm_counter(mm, i, rss[i]); 490 } 491 492 /* 493 * This function is called to print an error when a bad pte 494 * is found. For example, we might have a PFN-mapped pte in 495 * a region that doesn't allow it. 496 * 497 * The calling function must still handle the error. 498 */ 499 static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr, 500 pte_t pte, struct page *page) 501 { 502 pgd_t *pgd = pgd_offset(vma->vm_mm, addr); 503 p4d_t *p4d = p4d_offset(pgd, addr); 504 pud_t *pud = pud_offset(p4d, addr); 505 pmd_t *pmd = pmd_offset(pud, addr); 506 struct address_space *mapping; 507 pgoff_t index; 508 static unsigned long resume; 509 static unsigned long nr_shown; 510 static unsigned long nr_unshown; 511 512 /* 513 * Allow a burst of 60 reports, then keep quiet for that minute; 514 * or allow a steady drip of one report per second. 515 */ 516 if (nr_shown == 60) { 517 if (time_before(jiffies, resume)) { 518 nr_unshown++; 519 return; 520 } 521 if (nr_unshown) { 522 pr_alert("BUG: Bad page map: %lu messages suppressed\n", 523 nr_unshown); 524 nr_unshown = 0; 525 } 526 nr_shown = 0; 527 } 528 if (nr_shown++ == 0) 529 resume = jiffies + 60 * HZ; 530 531 mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL; 532 index = linear_page_index(vma, addr); 533 534 pr_alert("BUG: Bad page map in process %s pte:%08llx pmd:%08llx\n", 535 current->comm, 536 (long long)pte_val(pte), (long long)pmd_val(*pmd)); 537 if (page) 538 dump_page(page, "bad pte"); 539 pr_alert("addr:%px vm_flags:%08lx anon_vma:%px mapping:%px index:%lx\n", 540 (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index); 541 pr_alert("file:%pD fault:%ps mmap:%ps readpage:%ps\n", 542 vma->vm_file, 543 vma->vm_ops ? vma->vm_ops->fault : NULL, 544 vma->vm_file ? vma->vm_file->f_op->mmap : NULL, 545 mapping ? mapping->a_ops->readpage : NULL); 546 dump_stack(); 547 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 548 } 549 550 /* 551 * vm_normal_page -- This function gets the "struct page" associated with a pte. 552 * 553 * "Special" mappings do not wish to be associated with a "struct page" (either 554 * it doesn't exist, or it exists but they don't want to touch it). In this 555 * case, NULL is returned here. "Normal" mappings do have a struct page. 556 * 557 * There are 2 broad cases. Firstly, an architecture may define a pte_special() 558 * pte bit, in which case this function is trivial. Secondly, an architecture 559 * may not have a spare pte bit, which requires a more complicated scheme, 560 * described below. 561 * 562 * A raw VM_PFNMAP mapping (ie. one that is not COWed) is always considered a 563 * special mapping (even if there are underlying and valid "struct pages"). 564 * COWed pages of a VM_PFNMAP are always normal. 565 * 566 * The way we recognize COWed pages within VM_PFNMAP mappings is through the 567 * rules set up by "remap_pfn_range()": the vma will have the VM_PFNMAP bit 568 * set, and the vm_pgoff will point to the first PFN mapped: thus every special 569 * mapping will always honor the rule 570 * 571 * pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT) 572 * 573 * And for normal mappings this is false. 574 * 575 * This restricts such mappings to be a linear translation from virtual address 576 * to pfn. To get around this restriction, we allow arbitrary mappings so long 577 * as the vma is not a COW mapping; in that case, we know that all ptes are 578 * special (because none can have been COWed). 579 * 580 * 581 * In order to support COW of arbitrary special mappings, we have VM_MIXEDMAP. 582 * 583 * VM_MIXEDMAP mappings can likewise contain memory with or without "struct 584 * page" backing, however the difference is that _all_ pages with a struct 585 * page (that is, those where pfn_valid is true) are refcounted and considered 586 * normal pages by the VM. The disadvantage is that pages are refcounted 587 * (which can be slower and simply not an option for some PFNMAP users). The 588 * advantage is that we don't have to follow the strict linearity rule of 589 * PFNMAP mappings in order to support COWable mappings. 590 * 591 */ 592 struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr, 593 pte_t pte) 594 { 595 unsigned long pfn = pte_pfn(pte); 596 597 if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL)) { 598 if (likely(!pte_special(pte))) 599 goto check_pfn; 600 if (vma->vm_ops && vma->vm_ops->find_special_page) 601 return vma->vm_ops->find_special_page(vma, addr); 602 if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)) 603 return NULL; 604 if (is_zero_pfn(pfn)) 605 return NULL; 606 if (pte_devmap(pte)) 607 return NULL; 608 609 print_bad_pte(vma, addr, pte, NULL); 610 return NULL; 611 } 612 613 /* !CONFIG_ARCH_HAS_PTE_SPECIAL case follows: */ 614 615 if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) { 616 if (vma->vm_flags & VM_MIXEDMAP) { 617 if (!pfn_valid(pfn)) 618 return NULL; 619 goto out; 620 } else { 621 unsigned long off; 622 off = (addr - vma->vm_start) >> PAGE_SHIFT; 623 if (pfn == vma->vm_pgoff + off) 624 return NULL; 625 if (!is_cow_mapping(vma->vm_flags)) 626 return NULL; 627 } 628 } 629 630 if (is_zero_pfn(pfn)) 631 return NULL; 632 633 check_pfn: 634 if (unlikely(pfn > highest_memmap_pfn)) { 635 print_bad_pte(vma, addr, pte, NULL); 636 return NULL; 637 } 638 639 /* 640 * NOTE! We still have PageReserved() pages in the page tables. 641 * eg. VDSO mappings can cause them to exist. 642 */ 643 out: 644 return pfn_to_page(pfn); 645 } 646 647 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 648 struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr, 649 pmd_t pmd) 650 { 651 unsigned long pfn = pmd_pfn(pmd); 652 653 /* 654 * There is no pmd_special() but there may be special pmds, e.g. 655 * in a direct-access (dax) mapping, so let's just replicate the 656 * !CONFIG_ARCH_HAS_PTE_SPECIAL case from vm_normal_page() here. 657 */ 658 if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) { 659 if (vma->vm_flags & VM_MIXEDMAP) { 660 if (!pfn_valid(pfn)) 661 return NULL; 662 goto out; 663 } else { 664 unsigned long off; 665 off = (addr - vma->vm_start) >> PAGE_SHIFT; 666 if (pfn == vma->vm_pgoff + off) 667 return NULL; 668 if (!is_cow_mapping(vma->vm_flags)) 669 return NULL; 670 } 671 } 672 673 if (pmd_devmap(pmd)) 674 return NULL; 675 if (is_huge_zero_pmd(pmd)) 676 return NULL; 677 if (unlikely(pfn > highest_memmap_pfn)) 678 return NULL; 679 680 /* 681 * NOTE! We still have PageReserved() pages in the page tables. 682 * eg. VDSO mappings can cause them to exist. 683 */ 684 out: 685 return pfn_to_page(pfn); 686 } 687 #endif 688 689 /* 690 * copy one vm_area from one task to the other. Assumes the page tables 691 * already present in the new task to be cleared in the whole range 692 * covered by this vma. 693 */ 694 695 static inline unsigned long 696 copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, 697 pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma, 698 unsigned long addr, int *rss) 699 { 700 unsigned long vm_flags = vma->vm_flags; 701 pte_t pte = *src_pte; 702 struct page *page; 703 704 /* pte contains position in swap or file, so copy. */ 705 if (unlikely(!pte_present(pte))) { 706 swp_entry_t entry = pte_to_swp_entry(pte); 707 708 if (likely(!non_swap_entry(entry))) { 709 if (swap_duplicate(entry) < 0) 710 return entry.val; 711 712 /* make sure dst_mm is on swapoff's mmlist. */ 713 if (unlikely(list_empty(&dst_mm->mmlist))) { 714 spin_lock(&mmlist_lock); 715 if (list_empty(&dst_mm->mmlist)) 716 list_add(&dst_mm->mmlist, 717 &src_mm->mmlist); 718 spin_unlock(&mmlist_lock); 719 } 720 rss[MM_SWAPENTS]++; 721 } else if (is_migration_entry(entry)) { 722 page = migration_entry_to_page(entry); 723 724 rss[mm_counter(page)]++; 725 726 if (is_write_migration_entry(entry) && 727 is_cow_mapping(vm_flags)) { 728 /* 729 * COW mappings require pages in both 730 * parent and child to be set to read. 731 */ 732 make_migration_entry_read(&entry); 733 pte = swp_entry_to_pte(entry); 734 if (pte_swp_soft_dirty(*src_pte)) 735 pte = pte_swp_mksoft_dirty(pte); 736 if (pte_swp_uffd_wp(*src_pte)) 737 pte = pte_swp_mkuffd_wp(pte); 738 set_pte_at(src_mm, addr, src_pte, pte); 739 } 740 } else if (is_device_private_entry(entry)) { 741 page = device_private_entry_to_page(entry); 742 743 /* 744 * Update rss count even for unaddressable pages, as 745 * they should treated just like normal pages in this 746 * respect. 747 * 748 * We will likely want to have some new rss counters 749 * for unaddressable pages, at some point. But for now 750 * keep things as they are. 751 */ 752 get_page(page); 753 rss[mm_counter(page)]++; 754 page_dup_rmap(page, false); 755 756 /* 757 * We do not preserve soft-dirty information, because so 758 * far, checkpoint/restore is the only feature that 759 * requires that. And checkpoint/restore does not work 760 * when a device driver is involved (you cannot easily 761 * save and restore device driver state). 762 */ 763 if (is_write_device_private_entry(entry) && 764 is_cow_mapping(vm_flags)) { 765 make_device_private_entry_read(&entry); 766 pte = swp_entry_to_pte(entry); 767 if (pte_swp_uffd_wp(*src_pte)) 768 pte = pte_swp_mkuffd_wp(pte); 769 set_pte_at(src_mm, addr, src_pte, pte); 770 } 771 } 772 goto out_set_pte; 773 } 774 775 /* 776 * If it's a COW mapping, write protect it both 777 * in the parent and the child 778 */ 779 if (is_cow_mapping(vm_flags) && pte_write(pte)) { 780 ptep_set_wrprotect(src_mm, addr, src_pte); 781 pte = pte_wrprotect(pte); 782 } 783 784 /* 785 * If it's a shared mapping, mark it clean in 786 * the child 787 */ 788 if (vm_flags & VM_SHARED) 789 pte = pte_mkclean(pte); 790 pte = pte_mkold(pte); 791 792 /* 793 * Make sure the _PAGE_UFFD_WP bit is cleared if the new VMA 794 * does not have the VM_UFFD_WP, which means that the uffd 795 * fork event is not enabled. 796 */ 797 if (!(vm_flags & VM_UFFD_WP)) 798 pte = pte_clear_uffd_wp(pte); 799 800 page = vm_normal_page(vma, addr, pte); 801 if (page) { 802 get_page(page); 803 page_dup_rmap(page, false); 804 rss[mm_counter(page)]++; 805 } else if (pte_devmap(pte)) { 806 page = pte_page(pte); 807 } 808 809 out_set_pte: 810 set_pte_at(dst_mm, addr, dst_pte, pte); 811 return 0; 812 } 813 814 static int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 815 pmd_t *dst_pmd, pmd_t *src_pmd, struct vm_area_struct *vma, 816 unsigned long addr, unsigned long end) 817 { 818 pte_t *orig_src_pte, *orig_dst_pte; 819 pte_t *src_pte, *dst_pte; 820 spinlock_t *src_ptl, *dst_ptl; 821 int progress = 0; 822 int rss[NR_MM_COUNTERS]; 823 swp_entry_t entry = (swp_entry_t){0}; 824 825 again: 826 init_rss_vec(rss); 827 828 dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl); 829 if (!dst_pte) 830 return -ENOMEM; 831 src_pte = pte_offset_map(src_pmd, addr); 832 src_ptl = pte_lockptr(src_mm, src_pmd); 833 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 834 orig_src_pte = src_pte; 835 orig_dst_pte = dst_pte; 836 arch_enter_lazy_mmu_mode(); 837 838 do { 839 /* 840 * We are holding two locks at this point - either of them 841 * could generate latencies in another task on another CPU. 842 */ 843 if (progress >= 32) { 844 progress = 0; 845 if (need_resched() || 846 spin_needbreak(src_ptl) || spin_needbreak(dst_ptl)) 847 break; 848 } 849 if (pte_none(*src_pte)) { 850 progress++; 851 continue; 852 } 853 entry.val = copy_one_pte(dst_mm, src_mm, dst_pte, src_pte, 854 vma, addr, rss); 855 if (entry.val) 856 break; 857 progress += 8; 858 } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end); 859 860 arch_leave_lazy_mmu_mode(); 861 spin_unlock(src_ptl); 862 pte_unmap(orig_src_pte); 863 add_mm_rss_vec(dst_mm, rss); 864 pte_unmap_unlock(orig_dst_pte, dst_ptl); 865 cond_resched(); 866 867 if (entry.val) { 868 if (add_swap_count_continuation(entry, GFP_KERNEL) < 0) 869 return -ENOMEM; 870 progress = 0; 871 } 872 if (addr != end) 873 goto again; 874 return 0; 875 } 876 877 static inline int copy_pmd_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 878 pud_t *dst_pud, pud_t *src_pud, struct vm_area_struct *vma, 879 unsigned long addr, unsigned long end) 880 { 881 pmd_t *src_pmd, *dst_pmd; 882 unsigned long next; 883 884 dst_pmd = pmd_alloc(dst_mm, dst_pud, addr); 885 if (!dst_pmd) 886 return -ENOMEM; 887 src_pmd = pmd_offset(src_pud, addr); 888 do { 889 next = pmd_addr_end(addr, end); 890 if (is_swap_pmd(*src_pmd) || pmd_trans_huge(*src_pmd) 891 || pmd_devmap(*src_pmd)) { 892 int err; 893 VM_BUG_ON_VMA(next-addr != HPAGE_PMD_SIZE, vma); 894 err = copy_huge_pmd(dst_mm, src_mm, 895 dst_pmd, src_pmd, addr, vma); 896 if (err == -ENOMEM) 897 return -ENOMEM; 898 if (!err) 899 continue; 900 /* fall through */ 901 } 902 if (pmd_none_or_clear_bad(src_pmd)) 903 continue; 904 if (copy_pte_range(dst_mm, src_mm, dst_pmd, src_pmd, 905 vma, addr, next)) 906 return -ENOMEM; 907 } while (dst_pmd++, src_pmd++, addr = next, addr != end); 908 return 0; 909 } 910 911 static inline int copy_pud_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 912 p4d_t *dst_p4d, p4d_t *src_p4d, struct vm_area_struct *vma, 913 unsigned long addr, unsigned long end) 914 { 915 pud_t *src_pud, *dst_pud; 916 unsigned long next; 917 918 dst_pud = pud_alloc(dst_mm, dst_p4d, addr); 919 if (!dst_pud) 920 return -ENOMEM; 921 src_pud = pud_offset(src_p4d, addr); 922 do { 923 next = pud_addr_end(addr, end); 924 if (pud_trans_huge(*src_pud) || pud_devmap(*src_pud)) { 925 int err; 926 927 VM_BUG_ON_VMA(next-addr != HPAGE_PUD_SIZE, vma); 928 err = copy_huge_pud(dst_mm, src_mm, 929 dst_pud, src_pud, addr, vma); 930 if (err == -ENOMEM) 931 return -ENOMEM; 932 if (!err) 933 continue; 934 /* fall through */ 935 } 936 if (pud_none_or_clear_bad(src_pud)) 937 continue; 938 if (copy_pmd_range(dst_mm, src_mm, dst_pud, src_pud, 939 vma, addr, next)) 940 return -ENOMEM; 941 } while (dst_pud++, src_pud++, addr = next, addr != end); 942 return 0; 943 } 944 945 static inline int copy_p4d_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 946 pgd_t *dst_pgd, pgd_t *src_pgd, struct vm_area_struct *vma, 947 unsigned long addr, unsigned long end) 948 { 949 p4d_t *src_p4d, *dst_p4d; 950 unsigned long next; 951 952 dst_p4d = p4d_alloc(dst_mm, dst_pgd, addr); 953 if (!dst_p4d) 954 return -ENOMEM; 955 src_p4d = p4d_offset(src_pgd, addr); 956 do { 957 next = p4d_addr_end(addr, end); 958 if (p4d_none_or_clear_bad(src_p4d)) 959 continue; 960 if (copy_pud_range(dst_mm, src_mm, dst_p4d, src_p4d, 961 vma, addr, next)) 962 return -ENOMEM; 963 } while (dst_p4d++, src_p4d++, addr = next, addr != end); 964 return 0; 965 } 966 967 int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 968 struct vm_area_struct *vma) 969 { 970 pgd_t *src_pgd, *dst_pgd; 971 unsigned long next; 972 unsigned long addr = vma->vm_start; 973 unsigned long end = vma->vm_end; 974 struct mmu_notifier_range range; 975 bool is_cow; 976 int ret; 977 978 /* 979 * Don't copy ptes where a page fault will fill them correctly. 980 * Fork becomes much lighter when there are big shared or private 981 * readonly mappings. The tradeoff is that copy_page_range is more 982 * efficient than faulting. 983 */ 984 if (!(vma->vm_flags & (VM_HUGETLB | VM_PFNMAP | VM_MIXEDMAP)) && 985 !vma->anon_vma) 986 return 0; 987 988 if (is_vm_hugetlb_page(vma)) 989 return copy_hugetlb_page_range(dst_mm, src_mm, vma); 990 991 if (unlikely(vma->vm_flags & VM_PFNMAP)) { 992 /* 993 * We do not free on error cases below as remove_vma 994 * gets called on error from higher level routine 995 */ 996 ret = track_pfn_copy(vma); 997 if (ret) 998 return ret; 999 } 1000 1001 /* 1002 * We need to invalidate the secondary MMU mappings only when 1003 * there could be a permission downgrade on the ptes of the 1004 * parent mm. And a permission downgrade will only happen if 1005 * is_cow_mapping() returns true. 1006 */ 1007 is_cow = is_cow_mapping(vma->vm_flags); 1008 1009 if (is_cow) { 1010 mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE, 1011 0, vma, src_mm, addr, end); 1012 mmu_notifier_invalidate_range_start(&range); 1013 } 1014 1015 ret = 0; 1016 dst_pgd = pgd_offset(dst_mm, addr); 1017 src_pgd = pgd_offset(src_mm, addr); 1018 do { 1019 next = pgd_addr_end(addr, end); 1020 if (pgd_none_or_clear_bad(src_pgd)) 1021 continue; 1022 if (unlikely(copy_p4d_range(dst_mm, src_mm, dst_pgd, src_pgd, 1023 vma, addr, next))) { 1024 ret = -ENOMEM; 1025 break; 1026 } 1027 } while (dst_pgd++, src_pgd++, addr = next, addr != end); 1028 1029 if (is_cow) 1030 mmu_notifier_invalidate_range_end(&range); 1031 return ret; 1032 } 1033 1034 static unsigned long zap_pte_range(struct mmu_gather *tlb, 1035 struct vm_area_struct *vma, pmd_t *pmd, 1036 unsigned long addr, unsigned long end, 1037 struct zap_details *details) 1038 { 1039 struct mm_struct *mm = tlb->mm; 1040 int force_flush = 0; 1041 int rss[NR_MM_COUNTERS]; 1042 spinlock_t *ptl; 1043 pte_t *start_pte; 1044 pte_t *pte; 1045 swp_entry_t entry; 1046 1047 tlb_change_page_size(tlb, PAGE_SIZE); 1048 again: 1049 init_rss_vec(rss); 1050 start_pte = pte_offset_map_lock(mm, pmd, addr, &ptl); 1051 pte = start_pte; 1052 flush_tlb_batched_pending(mm); 1053 arch_enter_lazy_mmu_mode(); 1054 do { 1055 pte_t ptent = *pte; 1056 if (pte_none(ptent)) 1057 continue; 1058 1059 if (need_resched()) 1060 break; 1061 1062 if (pte_present(ptent)) { 1063 struct page *page; 1064 1065 page = vm_normal_page(vma, addr, ptent); 1066 if (unlikely(details) && page) { 1067 /* 1068 * unmap_shared_mapping_pages() wants to 1069 * invalidate cache without truncating: 1070 * unmap shared but keep private pages. 1071 */ 1072 if (details->check_mapping && 1073 details->check_mapping != page_rmapping(page)) 1074 continue; 1075 } 1076 ptent = ptep_get_and_clear_full(mm, addr, pte, 1077 tlb->fullmm); 1078 tlb_remove_tlb_entry(tlb, pte, addr); 1079 if (unlikely(!page)) 1080 continue; 1081 1082 if (!PageAnon(page)) { 1083 if (pte_dirty(ptent)) { 1084 force_flush = 1; 1085 set_page_dirty(page); 1086 } 1087 if (pte_young(ptent) && 1088 likely(!(vma->vm_flags & VM_SEQ_READ))) 1089 mark_page_accessed(page); 1090 } 1091 rss[mm_counter(page)]--; 1092 page_remove_rmap(page, false); 1093 if (unlikely(page_mapcount(page) < 0)) 1094 print_bad_pte(vma, addr, ptent, page); 1095 if (unlikely(__tlb_remove_page(tlb, page))) { 1096 force_flush = 1; 1097 addr += PAGE_SIZE; 1098 break; 1099 } 1100 continue; 1101 } 1102 1103 entry = pte_to_swp_entry(ptent); 1104 if (non_swap_entry(entry) && is_device_private_entry(entry)) { 1105 struct page *page = device_private_entry_to_page(entry); 1106 1107 if (unlikely(details && details->check_mapping)) { 1108 /* 1109 * unmap_shared_mapping_pages() wants to 1110 * invalidate cache without truncating: 1111 * unmap shared but keep private pages. 1112 */ 1113 if (details->check_mapping != 1114 page_rmapping(page)) 1115 continue; 1116 } 1117 1118 pte_clear_not_present_full(mm, addr, pte, tlb->fullmm); 1119 rss[mm_counter(page)]--; 1120 page_remove_rmap(page, false); 1121 put_page(page); 1122 continue; 1123 } 1124 1125 /* If details->check_mapping, we leave swap entries. */ 1126 if (unlikely(details)) 1127 continue; 1128 1129 if (!non_swap_entry(entry)) 1130 rss[MM_SWAPENTS]--; 1131 else if (is_migration_entry(entry)) { 1132 struct page *page; 1133 1134 page = migration_entry_to_page(entry); 1135 rss[mm_counter(page)]--; 1136 } 1137 if (unlikely(!free_swap_and_cache(entry))) 1138 print_bad_pte(vma, addr, ptent, NULL); 1139 pte_clear_not_present_full(mm, addr, pte, tlb->fullmm); 1140 } while (pte++, addr += PAGE_SIZE, addr != end); 1141 1142 add_mm_rss_vec(mm, rss); 1143 arch_leave_lazy_mmu_mode(); 1144 1145 /* Do the actual TLB flush before dropping ptl */ 1146 if (force_flush) 1147 tlb_flush_mmu_tlbonly(tlb); 1148 pte_unmap_unlock(start_pte, ptl); 1149 1150 /* 1151 * If we forced a TLB flush (either due to running out of 1152 * batch buffers or because we needed to flush dirty TLB 1153 * entries before releasing the ptl), free the batched 1154 * memory too. Restart if we didn't do everything. 1155 */ 1156 if (force_flush) { 1157 force_flush = 0; 1158 tlb_flush_mmu(tlb); 1159 } 1160 1161 if (addr != end) { 1162 cond_resched(); 1163 goto again; 1164 } 1165 1166 return addr; 1167 } 1168 1169 static inline unsigned long zap_pmd_range(struct mmu_gather *tlb, 1170 struct vm_area_struct *vma, pud_t *pud, 1171 unsigned long addr, unsigned long end, 1172 struct zap_details *details) 1173 { 1174 pmd_t *pmd; 1175 unsigned long next; 1176 1177 pmd = pmd_offset(pud, addr); 1178 do { 1179 next = pmd_addr_end(addr, end); 1180 if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) { 1181 if (next - addr != HPAGE_PMD_SIZE) 1182 __split_huge_pmd(vma, pmd, addr, false, NULL); 1183 else if (zap_huge_pmd(tlb, vma, pmd, addr)) 1184 goto next; 1185 /* fall through */ 1186 } 1187 /* 1188 * Here there can be other concurrent MADV_DONTNEED or 1189 * trans huge page faults running, and if the pmd is 1190 * none or trans huge it can change under us. This is 1191 * because MADV_DONTNEED holds the mmap_sem in read 1192 * mode. 1193 */ 1194 if (pmd_none_or_trans_huge_or_clear_bad(pmd)) 1195 goto next; 1196 next = zap_pte_range(tlb, vma, pmd, addr, next, details); 1197 next: 1198 cond_resched(); 1199 } while (pmd++, addr = next, addr != end); 1200 1201 return addr; 1202 } 1203 1204 static inline unsigned long zap_pud_range(struct mmu_gather *tlb, 1205 struct vm_area_struct *vma, p4d_t *p4d, 1206 unsigned long addr, unsigned long end, 1207 struct zap_details *details) 1208 { 1209 pud_t *pud; 1210 unsigned long next; 1211 1212 pud = pud_offset(p4d, addr); 1213 do { 1214 next = pud_addr_end(addr, end); 1215 if (pud_trans_huge(*pud) || pud_devmap(*pud)) { 1216 if (next - addr != HPAGE_PUD_SIZE) { 1217 VM_BUG_ON_VMA(!rwsem_is_locked(&tlb->mm->mmap_sem), vma); 1218 split_huge_pud(vma, pud, addr); 1219 } else if (zap_huge_pud(tlb, vma, pud, addr)) 1220 goto next; 1221 /* fall through */ 1222 } 1223 if (pud_none_or_clear_bad(pud)) 1224 continue; 1225 next = zap_pmd_range(tlb, vma, pud, addr, next, details); 1226 next: 1227 cond_resched(); 1228 } while (pud++, addr = next, addr != end); 1229 1230 return addr; 1231 } 1232 1233 static inline unsigned long zap_p4d_range(struct mmu_gather *tlb, 1234 struct vm_area_struct *vma, pgd_t *pgd, 1235 unsigned long addr, unsigned long end, 1236 struct zap_details *details) 1237 { 1238 p4d_t *p4d; 1239 unsigned long next; 1240 1241 p4d = p4d_offset(pgd, addr); 1242 do { 1243 next = p4d_addr_end(addr, end); 1244 if (p4d_none_or_clear_bad(p4d)) 1245 continue; 1246 next = zap_pud_range(tlb, vma, p4d, addr, next, details); 1247 } while (p4d++, addr = next, addr != end); 1248 1249 return addr; 1250 } 1251 1252 void unmap_page_range(struct mmu_gather *tlb, 1253 struct vm_area_struct *vma, 1254 unsigned long addr, unsigned long end, 1255 struct zap_details *details) 1256 { 1257 pgd_t *pgd; 1258 unsigned long next; 1259 1260 BUG_ON(addr >= end); 1261 tlb_start_vma(tlb, vma); 1262 pgd = pgd_offset(vma->vm_mm, addr); 1263 do { 1264 next = pgd_addr_end(addr, end); 1265 if (pgd_none_or_clear_bad(pgd)) 1266 continue; 1267 next = zap_p4d_range(tlb, vma, pgd, addr, next, details); 1268 } while (pgd++, addr = next, addr != end); 1269 tlb_end_vma(tlb, vma); 1270 } 1271 1272 1273 static void unmap_single_vma(struct mmu_gather *tlb, 1274 struct vm_area_struct *vma, unsigned long start_addr, 1275 unsigned long end_addr, 1276 struct zap_details *details) 1277 { 1278 unsigned long start = max(vma->vm_start, start_addr); 1279 unsigned long end; 1280 1281 if (start >= vma->vm_end) 1282 return; 1283 end = min(vma->vm_end, end_addr); 1284 if (end <= vma->vm_start) 1285 return; 1286 1287 if (vma->vm_file) 1288 uprobe_munmap(vma, start, end); 1289 1290 if (unlikely(vma->vm_flags & VM_PFNMAP)) 1291 untrack_pfn(vma, 0, 0); 1292 1293 if (start != end) { 1294 if (unlikely(is_vm_hugetlb_page(vma))) { 1295 /* 1296 * It is undesirable to test vma->vm_file as it 1297 * should be non-null for valid hugetlb area. 1298 * However, vm_file will be NULL in the error 1299 * cleanup path of mmap_region. When 1300 * hugetlbfs ->mmap method fails, 1301 * mmap_region() nullifies vma->vm_file 1302 * before calling this function to clean up. 1303 * Since no pte has actually been setup, it is 1304 * safe to do nothing in this case. 1305 */ 1306 if (vma->vm_file) { 1307 i_mmap_lock_write(vma->vm_file->f_mapping); 1308 __unmap_hugepage_range_final(tlb, vma, start, end, NULL); 1309 i_mmap_unlock_write(vma->vm_file->f_mapping); 1310 } 1311 } else 1312 unmap_page_range(tlb, vma, start, end, details); 1313 } 1314 } 1315 1316 /** 1317 * unmap_vmas - unmap a range of memory covered by a list of vma's 1318 * @tlb: address of the caller's struct mmu_gather 1319 * @vma: the starting vma 1320 * @start_addr: virtual address at which to start unmapping 1321 * @end_addr: virtual address at which to end unmapping 1322 * 1323 * Unmap all pages in the vma list. 1324 * 1325 * Only addresses between `start' and `end' will be unmapped. 1326 * 1327 * The VMA list must be sorted in ascending virtual address order. 1328 * 1329 * unmap_vmas() assumes that the caller will flush the whole unmapped address 1330 * range after unmap_vmas() returns. So the only responsibility here is to 1331 * ensure that any thus-far unmapped pages are flushed before unmap_vmas() 1332 * drops the lock and schedules. 1333 */ 1334 void unmap_vmas(struct mmu_gather *tlb, 1335 struct vm_area_struct *vma, unsigned long start_addr, 1336 unsigned long end_addr) 1337 { 1338 struct mmu_notifier_range range; 1339 1340 mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, vma->vm_mm, 1341 start_addr, end_addr); 1342 mmu_notifier_invalidate_range_start(&range); 1343 for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next) 1344 unmap_single_vma(tlb, vma, start_addr, end_addr, NULL); 1345 mmu_notifier_invalidate_range_end(&range); 1346 } 1347 1348 /** 1349 * zap_page_range - remove user pages in a given range 1350 * @vma: vm_area_struct holding the applicable pages 1351 * @start: starting address of pages to zap 1352 * @size: number of bytes to zap 1353 * 1354 * Caller must protect the VMA list 1355 */ 1356 void zap_page_range(struct vm_area_struct *vma, unsigned long start, 1357 unsigned long size) 1358 { 1359 struct mmu_notifier_range range; 1360 struct mmu_gather tlb; 1361 1362 lru_add_drain(); 1363 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, 1364 start, start + size); 1365 tlb_gather_mmu(&tlb, vma->vm_mm, start, range.end); 1366 update_hiwater_rss(vma->vm_mm); 1367 mmu_notifier_invalidate_range_start(&range); 1368 for ( ; vma && vma->vm_start < range.end; vma = vma->vm_next) 1369 unmap_single_vma(&tlb, vma, start, range.end, NULL); 1370 mmu_notifier_invalidate_range_end(&range); 1371 tlb_finish_mmu(&tlb, start, range.end); 1372 } 1373 1374 /** 1375 * zap_page_range_single - remove user pages in a given range 1376 * @vma: vm_area_struct holding the applicable pages 1377 * @address: starting address of pages to zap 1378 * @size: number of bytes to zap 1379 * @details: details of shared cache invalidation 1380 * 1381 * The range must fit into one VMA. 1382 */ 1383 static void zap_page_range_single(struct vm_area_struct *vma, unsigned long address, 1384 unsigned long size, struct zap_details *details) 1385 { 1386 struct mmu_notifier_range range; 1387 struct mmu_gather tlb; 1388 1389 lru_add_drain(); 1390 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, 1391 address, address + size); 1392 tlb_gather_mmu(&tlb, vma->vm_mm, address, range.end); 1393 update_hiwater_rss(vma->vm_mm); 1394 mmu_notifier_invalidate_range_start(&range); 1395 unmap_single_vma(&tlb, vma, address, range.end, details); 1396 mmu_notifier_invalidate_range_end(&range); 1397 tlb_finish_mmu(&tlb, address, range.end); 1398 } 1399 1400 /** 1401 * zap_vma_ptes - remove ptes mapping the vma 1402 * @vma: vm_area_struct holding ptes to be zapped 1403 * @address: starting address of pages to zap 1404 * @size: number of bytes to zap 1405 * 1406 * This function only unmaps ptes assigned to VM_PFNMAP vmas. 1407 * 1408 * The entire address range must be fully contained within the vma. 1409 * 1410 */ 1411 void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, 1412 unsigned long size) 1413 { 1414 if (address < vma->vm_start || address + size > vma->vm_end || 1415 !(vma->vm_flags & VM_PFNMAP)) 1416 return; 1417 1418 zap_page_range_single(vma, address, size, NULL); 1419 } 1420 EXPORT_SYMBOL_GPL(zap_vma_ptes); 1421 1422 pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr, 1423 spinlock_t **ptl) 1424 { 1425 pgd_t *pgd; 1426 p4d_t *p4d; 1427 pud_t *pud; 1428 pmd_t *pmd; 1429 1430 pgd = pgd_offset(mm, addr); 1431 p4d = p4d_alloc(mm, pgd, addr); 1432 if (!p4d) 1433 return NULL; 1434 pud = pud_alloc(mm, p4d, addr); 1435 if (!pud) 1436 return NULL; 1437 pmd = pmd_alloc(mm, pud, addr); 1438 if (!pmd) 1439 return NULL; 1440 1441 VM_BUG_ON(pmd_trans_huge(*pmd)); 1442 return pte_alloc_map_lock(mm, pmd, addr, ptl); 1443 } 1444 1445 /* 1446 * This is the old fallback for page remapping. 1447 * 1448 * For historical reasons, it only allows reserved pages. Only 1449 * old drivers should use this, and they needed to mark their 1450 * pages reserved for the old functions anyway. 1451 */ 1452 static int insert_page(struct vm_area_struct *vma, unsigned long addr, 1453 struct page *page, pgprot_t prot) 1454 { 1455 struct mm_struct *mm = vma->vm_mm; 1456 int retval; 1457 pte_t *pte; 1458 spinlock_t *ptl; 1459 1460 retval = -EINVAL; 1461 if (PageAnon(page) || PageSlab(page) || page_has_type(page)) 1462 goto out; 1463 retval = -ENOMEM; 1464 flush_dcache_page(page); 1465 pte = get_locked_pte(mm, addr, &ptl); 1466 if (!pte) 1467 goto out; 1468 retval = -EBUSY; 1469 if (!pte_none(*pte)) 1470 goto out_unlock; 1471 1472 /* Ok, finally just insert the thing.. */ 1473 get_page(page); 1474 inc_mm_counter_fast(mm, mm_counter_file(page)); 1475 page_add_file_rmap(page, false); 1476 set_pte_at(mm, addr, pte, mk_pte(page, prot)); 1477 1478 retval = 0; 1479 out_unlock: 1480 pte_unmap_unlock(pte, ptl); 1481 out: 1482 return retval; 1483 } 1484 1485 /** 1486 * vm_insert_page - insert single page into user vma 1487 * @vma: user vma to map to 1488 * @addr: target user address of this page 1489 * @page: source kernel page 1490 * 1491 * This allows drivers to insert individual pages they've allocated 1492 * into a user vma. 1493 * 1494 * The page has to be a nice clean _individual_ kernel allocation. 1495 * If you allocate a compound page, you need to have marked it as 1496 * such (__GFP_COMP), or manually just split the page up yourself 1497 * (see split_page()). 1498 * 1499 * NOTE! Traditionally this was done with "remap_pfn_range()" which 1500 * took an arbitrary page protection parameter. This doesn't allow 1501 * that. Your vma protection will have to be set up correctly, which 1502 * means that if you want a shared writable mapping, you'd better 1503 * ask for a shared writable mapping! 1504 * 1505 * The page does not need to be reserved. 1506 * 1507 * Usually this function is called from f_op->mmap() handler 1508 * under mm->mmap_sem write-lock, so it can change vma->vm_flags. 1509 * Caller must set VM_MIXEDMAP on vma if it wants to call this 1510 * function from other places, for example from page-fault handler. 1511 * 1512 * Return: %0 on success, negative error code otherwise. 1513 */ 1514 int vm_insert_page(struct vm_area_struct *vma, unsigned long addr, 1515 struct page *page) 1516 { 1517 if (addr < vma->vm_start || addr >= vma->vm_end) 1518 return -EFAULT; 1519 if (!page_count(page)) 1520 return -EINVAL; 1521 if (!(vma->vm_flags & VM_MIXEDMAP)) { 1522 BUG_ON(down_read_trylock(&vma->vm_mm->mmap_sem)); 1523 BUG_ON(vma->vm_flags & VM_PFNMAP); 1524 vma->vm_flags |= VM_MIXEDMAP; 1525 } 1526 return insert_page(vma, addr, page, vma->vm_page_prot); 1527 } 1528 EXPORT_SYMBOL(vm_insert_page); 1529 1530 /* 1531 * __vm_map_pages - maps range of kernel pages into user vma 1532 * @vma: user vma to map to 1533 * @pages: pointer to array of source kernel pages 1534 * @num: number of pages in page array 1535 * @offset: user's requested vm_pgoff 1536 * 1537 * This allows drivers to map range of kernel pages into a user vma. 1538 * 1539 * Return: 0 on success and error code otherwise. 1540 */ 1541 static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages, 1542 unsigned long num, unsigned long offset) 1543 { 1544 unsigned long count = vma_pages(vma); 1545 unsigned long uaddr = vma->vm_start; 1546 int ret, i; 1547 1548 /* Fail if the user requested offset is beyond the end of the object */ 1549 if (offset >= num) 1550 return -ENXIO; 1551 1552 /* Fail if the user requested size exceeds available object size */ 1553 if (count > num - offset) 1554 return -ENXIO; 1555 1556 for (i = 0; i < count; i++) { 1557 ret = vm_insert_page(vma, uaddr, pages[offset + i]); 1558 if (ret < 0) 1559 return ret; 1560 uaddr += PAGE_SIZE; 1561 } 1562 1563 return 0; 1564 } 1565 1566 /** 1567 * vm_map_pages - maps range of kernel pages starts with non zero offset 1568 * @vma: user vma to map to 1569 * @pages: pointer to array of source kernel pages 1570 * @num: number of pages in page array 1571 * 1572 * Maps an object consisting of @num pages, catering for the user's 1573 * requested vm_pgoff 1574 * 1575 * If we fail to insert any page into the vma, the function will return 1576 * immediately leaving any previously inserted pages present. Callers 1577 * from the mmap handler may immediately return the error as their caller 1578 * will destroy the vma, removing any successfully inserted pages. Other 1579 * callers should make their own arrangements for calling unmap_region(). 1580 * 1581 * Context: Process context. Called by mmap handlers. 1582 * Return: 0 on success and error code otherwise. 1583 */ 1584 int vm_map_pages(struct vm_area_struct *vma, struct page **pages, 1585 unsigned long num) 1586 { 1587 return __vm_map_pages(vma, pages, num, vma->vm_pgoff); 1588 } 1589 EXPORT_SYMBOL(vm_map_pages); 1590 1591 /** 1592 * vm_map_pages_zero - map range of kernel pages starts with zero offset 1593 * @vma: user vma to map to 1594 * @pages: pointer to array of source kernel pages 1595 * @num: number of pages in page array 1596 * 1597 * Similar to vm_map_pages(), except that it explicitly sets the offset 1598 * to 0. This function is intended for the drivers that did not consider 1599 * vm_pgoff. 1600 * 1601 * Context: Process context. Called by mmap handlers. 1602 * Return: 0 on success and error code otherwise. 1603 */ 1604 int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages, 1605 unsigned long num) 1606 { 1607 return __vm_map_pages(vma, pages, num, 0); 1608 } 1609 EXPORT_SYMBOL(vm_map_pages_zero); 1610 1611 static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr, 1612 pfn_t pfn, pgprot_t prot, bool mkwrite) 1613 { 1614 struct mm_struct *mm = vma->vm_mm; 1615 pte_t *pte, entry; 1616 spinlock_t *ptl; 1617 1618 pte = get_locked_pte(mm, addr, &ptl); 1619 if (!pte) 1620 return VM_FAULT_OOM; 1621 if (!pte_none(*pte)) { 1622 if (mkwrite) { 1623 /* 1624 * For read faults on private mappings the PFN passed 1625 * in may not match the PFN we have mapped if the 1626 * mapped PFN is a writeable COW page. In the mkwrite 1627 * case we are creating a writable PTE for a shared 1628 * mapping and we expect the PFNs to match. If they 1629 * don't match, we are likely racing with block 1630 * allocation and mapping invalidation so just skip the 1631 * update. 1632 */ 1633 if (pte_pfn(*pte) != pfn_t_to_pfn(pfn)) { 1634 WARN_ON_ONCE(!is_zero_pfn(pte_pfn(*pte))); 1635 goto out_unlock; 1636 } 1637 entry = pte_mkyoung(*pte); 1638 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 1639 if (ptep_set_access_flags(vma, addr, pte, entry, 1)) 1640 update_mmu_cache(vma, addr, pte); 1641 } 1642 goto out_unlock; 1643 } 1644 1645 /* Ok, finally just insert the thing.. */ 1646 if (pfn_t_devmap(pfn)) 1647 entry = pte_mkdevmap(pfn_t_pte(pfn, prot)); 1648 else 1649 entry = pte_mkspecial(pfn_t_pte(pfn, prot)); 1650 1651 if (mkwrite) { 1652 entry = pte_mkyoung(entry); 1653 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 1654 } 1655 1656 set_pte_at(mm, addr, pte, entry); 1657 update_mmu_cache(vma, addr, pte); /* XXX: why not for insert_page? */ 1658 1659 out_unlock: 1660 pte_unmap_unlock(pte, ptl); 1661 return VM_FAULT_NOPAGE; 1662 } 1663 1664 /** 1665 * vmf_insert_pfn_prot - insert single pfn into user vma with specified pgprot 1666 * @vma: user vma to map to 1667 * @addr: target user address of this page 1668 * @pfn: source kernel pfn 1669 * @pgprot: pgprot flags for the inserted page 1670 * 1671 * This is exactly like vmf_insert_pfn(), except that it allows drivers to 1672 * to override pgprot on a per-page basis. 1673 * 1674 * This only makes sense for IO mappings, and it makes no sense for 1675 * COW mappings. In general, using multiple vmas is preferable; 1676 * vmf_insert_pfn_prot should only be used if using multiple VMAs is 1677 * impractical. 1678 * 1679 * See vmf_insert_mixed_prot() for a discussion of the implication of using 1680 * a value of @pgprot different from that of @vma->vm_page_prot. 1681 * 1682 * Context: Process context. May allocate using %GFP_KERNEL. 1683 * Return: vm_fault_t value. 1684 */ 1685 vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr, 1686 unsigned long pfn, pgprot_t pgprot) 1687 { 1688 /* 1689 * Technically, architectures with pte_special can avoid all these 1690 * restrictions (same for remap_pfn_range). However we would like 1691 * consistency in testing and feature parity among all, so we should 1692 * try to keep these invariants in place for everybody. 1693 */ 1694 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); 1695 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 1696 (VM_PFNMAP|VM_MIXEDMAP)); 1697 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 1698 BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn)); 1699 1700 if (addr < vma->vm_start || addr >= vma->vm_end) 1701 return VM_FAULT_SIGBUS; 1702 1703 if (!pfn_modify_allowed(pfn, pgprot)) 1704 return VM_FAULT_SIGBUS; 1705 1706 track_pfn_insert(vma, &pgprot, __pfn_to_pfn_t(pfn, PFN_DEV)); 1707 1708 return insert_pfn(vma, addr, __pfn_to_pfn_t(pfn, PFN_DEV), pgprot, 1709 false); 1710 } 1711 EXPORT_SYMBOL(vmf_insert_pfn_prot); 1712 1713 /** 1714 * vmf_insert_pfn - insert single pfn into user vma 1715 * @vma: user vma to map to 1716 * @addr: target user address of this page 1717 * @pfn: source kernel pfn 1718 * 1719 * Similar to vm_insert_page, this allows drivers to insert individual pages 1720 * they've allocated into a user vma. Same comments apply. 1721 * 1722 * This function should only be called from a vm_ops->fault handler, and 1723 * in that case the handler should return the result of this function. 1724 * 1725 * vma cannot be a COW mapping. 1726 * 1727 * As this is called only for pages that do not currently exist, we 1728 * do not need to flush old virtual caches or the TLB. 1729 * 1730 * Context: Process context. May allocate using %GFP_KERNEL. 1731 * Return: vm_fault_t value. 1732 */ 1733 vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, unsigned long addr, 1734 unsigned long pfn) 1735 { 1736 return vmf_insert_pfn_prot(vma, addr, pfn, vma->vm_page_prot); 1737 } 1738 EXPORT_SYMBOL(vmf_insert_pfn); 1739 1740 static bool vm_mixed_ok(struct vm_area_struct *vma, pfn_t pfn) 1741 { 1742 /* these checks mirror the abort conditions in vm_normal_page */ 1743 if (vma->vm_flags & VM_MIXEDMAP) 1744 return true; 1745 if (pfn_t_devmap(pfn)) 1746 return true; 1747 if (pfn_t_special(pfn)) 1748 return true; 1749 if (is_zero_pfn(pfn_t_to_pfn(pfn))) 1750 return true; 1751 return false; 1752 } 1753 1754 static vm_fault_t __vm_insert_mixed(struct vm_area_struct *vma, 1755 unsigned long addr, pfn_t pfn, pgprot_t pgprot, 1756 bool mkwrite) 1757 { 1758 int err; 1759 1760 BUG_ON(!vm_mixed_ok(vma, pfn)); 1761 1762 if (addr < vma->vm_start || addr >= vma->vm_end) 1763 return VM_FAULT_SIGBUS; 1764 1765 track_pfn_insert(vma, &pgprot, pfn); 1766 1767 if (!pfn_modify_allowed(pfn_t_to_pfn(pfn), pgprot)) 1768 return VM_FAULT_SIGBUS; 1769 1770 /* 1771 * If we don't have pte special, then we have to use the pfn_valid() 1772 * based VM_MIXEDMAP scheme (see vm_normal_page), and thus we *must* 1773 * refcount the page if pfn_valid is true (hence insert_page rather 1774 * than insert_pfn). If a zero_pfn were inserted into a VM_MIXEDMAP 1775 * without pte special, it would there be refcounted as a normal page. 1776 */ 1777 if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL) && 1778 !pfn_t_devmap(pfn) && pfn_t_valid(pfn)) { 1779 struct page *page; 1780 1781 /* 1782 * At this point we are committed to insert_page() 1783 * regardless of whether the caller specified flags that 1784 * result in pfn_t_has_page() == false. 1785 */ 1786 page = pfn_to_page(pfn_t_to_pfn(pfn)); 1787 err = insert_page(vma, addr, page, pgprot); 1788 } else { 1789 return insert_pfn(vma, addr, pfn, pgprot, mkwrite); 1790 } 1791 1792 if (err == -ENOMEM) 1793 return VM_FAULT_OOM; 1794 if (err < 0 && err != -EBUSY) 1795 return VM_FAULT_SIGBUS; 1796 1797 return VM_FAULT_NOPAGE; 1798 } 1799 1800 /** 1801 * vmf_insert_mixed_prot - insert single pfn into user vma with specified pgprot 1802 * @vma: user vma to map to 1803 * @addr: target user address of this page 1804 * @pfn: source kernel pfn 1805 * @pgprot: pgprot flags for the inserted page 1806 * 1807 * This is exactly like vmf_insert_mixed(), except that it allows drivers to 1808 * to override pgprot on a per-page basis. 1809 * 1810 * Typically this function should be used by drivers to set caching- and 1811 * encryption bits different than those of @vma->vm_page_prot, because 1812 * the caching- or encryption mode may not be known at mmap() time. 1813 * This is ok as long as @vma->vm_page_prot is not used by the core vm 1814 * to set caching and encryption bits for those vmas (except for COW pages). 1815 * This is ensured by core vm only modifying these page table entries using 1816 * functions that don't touch caching- or encryption bits, using pte_modify() 1817 * if needed. (See for example mprotect()). 1818 * Also when new page-table entries are created, this is only done using the 1819 * fault() callback, and never using the value of vma->vm_page_prot, 1820 * except for page-table entries that point to anonymous pages as the result 1821 * of COW. 1822 * 1823 * Context: Process context. May allocate using %GFP_KERNEL. 1824 * Return: vm_fault_t value. 1825 */ 1826 vm_fault_t vmf_insert_mixed_prot(struct vm_area_struct *vma, unsigned long addr, 1827 pfn_t pfn, pgprot_t pgprot) 1828 { 1829 return __vm_insert_mixed(vma, addr, pfn, pgprot, false); 1830 } 1831 EXPORT_SYMBOL(vmf_insert_mixed_prot); 1832 1833 vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr, 1834 pfn_t pfn) 1835 { 1836 return __vm_insert_mixed(vma, addr, pfn, vma->vm_page_prot, false); 1837 } 1838 EXPORT_SYMBOL(vmf_insert_mixed); 1839 1840 /* 1841 * If the insertion of PTE failed because someone else already added a 1842 * different entry in the mean time, we treat that as success as we assume 1843 * the same entry was actually inserted. 1844 */ 1845 vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma, 1846 unsigned long addr, pfn_t pfn) 1847 { 1848 return __vm_insert_mixed(vma, addr, pfn, vma->vm_page_prot, true); 1849 } 1850 EXPORT_SYMBOL(vmf_insert_mixed_mkwrite); 1851 1852 /* 1853 * maps a range of physical memory into the requested pages. the old 1854 * mappings are removed. any references to nonexistent pages results 1855 * in null mappings (currently treated as "copy-on-access") 1856 */ 1857 static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd, 1858 unsigned long addr, unsigned long end, 1859 unsigned long pfn, pgprot_t prot) 1860 { 1861 pte_t *pte; 1862 spinlock_t *ptl; 1863 int err = 0; 1864 1865 pte = pte_alloc_map_lock(mm, pmd, addr, &ptl); 1866 if (!pte) 1867 return -ENOMEM; 1868 arch_enter_lazy_mmu_mode(); 1869 do { 1870 BUG_ON(!pte_none(*pte)); 1871 if (!pfn_modify_allowed(pfn, prot)) { 1872 err = -EACCES; 1873 break; 1874 } 1875 set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot))); 1876 pfn++; 1877 } while (pte++, addr += PAGE_SIZE, addr != end); 1878 arch_leave_lazy_mmu_mode(); 1879 pte_unmap_unlock(pte - 1, ptl); 1880 return err; 1881 } 1882 1883 static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud, 1884 unsigned long addr, unsigned long end, 1885 unsigned long pfn, pgprot_t prot) 1886 { 1887 pmd_t *pmd; 1888 unsigned long next; 1889 int err; 1890 1891 pfn -= addr >> PAGE_SHIFT; 1892 pmd = pmd_alloc(mm, pud, addr); 1893 if (!pmd) 1894 return -ENOMEM; 1895 VM_BUG_ON(pmd_trans_huge(*pmd)); 1896 do { 1897 next = pmd_addr_end(addr, end); 1898 err = remap_pte_range(mm, pmd, addr, next, 1899 pfn + (addr >> PAGE_SHIFT), prot); 1900 if (err) 1901 return err; 1902 } while (pmd++, addr = next, addr != end); 1903 return 0; 1904 } 1905 1906 static inline int remap_pud_range(struct mm_struct *mm, p4d_t *p4d, 1907 unsigned long addr, unsigned long end, 1908 unsigned long pfn, pgprot_t prot) 1909 { 1910 pud_t *pud; 1911 unsigned long next; 1912 int err; 1913 1914 pfn -= addr >> PAGE_SHIFT; 1915 pud = pud_alloc(mm, p4d, addr); 1916 if (!pud) 1917 return -ENOMEM; 1918 do { 1919 next = pud_addr_end(addr, end); 1920 err = remap_pmd_range(mm, pud, addr, next, 1921 pfn + (addr >> PAGE_SHIFT), prot); 1922 if (err) 1923 return err; 1924 } while (pud++, addr = next, addr != end); 1925 return 0; 1926 } 1927 1928 static inline int remap_p4d_range(struct mm_struct *mm, pgd_t *pgd, 1929 unsigned long addr, unsigned long end, 1930 unsigned long pfn, pgprot_t prot) 1931 { 1932 p4d_t *p4d; 1933 unsigned long next; 1934 int err; 1935 1936 pfn -= addr >> PAGE_SHIFT; 1937 p4d = p4d_alloc(mm, pgd, addr); 1938 if (!p4d) 1939 return -ENOMEM; 1940 do { 1941 next = p4d_addr_end(addr, end); 1942 err = remap_pud_range(mm, p4d, addr, next, 1943 pfn + (addr >> PAGE_SHIFT), prot); 1944 if (err) 1945 return err; 1946 } while (p4d++, addr = next, addr != end); 1947 return 0; 1948 } 1949 1950 /** 1951 * remap_pfn_range - remap kernel memory to userspace 1952 * @vma: user vma to map to 1953 * @addr: target user address to start at 1954 * @pfn: page frame number of kernel physical memory address 1955 * @size: size of mapping area 1956 * @prot: page protection flags for this mapping 1957 * 1958 * Note: this is only safe if the mm semaphore is held when called. 1959 * 1960 * Return: %0 on success, negative error code otherwise. 1961 */ 1962 int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, 1963 unsigned long pfn, unsigned long size, pgprot_t prot) 1964 { 1965 pgd_t *pgd; 1966 unsigned long next; 1967 unsigned long end = addr + PAGE_ALIGN(size); 1968 struct mm_struct *mm = vma->vm_mm; 1969 unsigned long remap_pfn = pfn; 1970 int err; 1971 1972 /* 1973 * Physically remapped pages are special. Tell the 1974 * rest of the world about it: 1975 * VM_IO tells people not to look at these pages 1976 * (accesses can have side effects). 1977 * VM_PFNMAP tells the core MM that the base pages are just 1978 * raw PFN mappings, and do not have a "struct page" associated 1979 * with them. 1980 * VM_DONTEXPAND 1981 * Disable vma merging and expanding with mremap(). 1982 * VM_DONTDUMP 1983 * Omit vma from core dump, even when VM_IO turned off. 1984 * 1985 * There's a horrible special case to handle copy-on-write 1986 * behaviour that some programs depend on. We mark the "original" 1987 * un-COW'ed pages by matching them up with "vma->vm_pgoff". 1988 * See vm_normal_page() for details. 1989 */ 1990 if (is_cow_mapping(vma->vm_flags)) { 1991 if (addr != vma->vm_start || end != vma->vm_end) 1992 return -EINVAL; 1993 vma->vm_pgoff = pfn; 1994 } 1995 1996 err = track_pfn_remap(vma, &prot, remap_pfn, addr, PAGE_ALIGN(size)); 1997 if (err) 1998 return -EINVAL; 1999 2000 vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP; 2001 2002 BUG_ON(addr >= end); 2003 pfn -= addr >> PAGE_SHIFT; 2004 pgd = pgd_offset(mm, addr); 2005 flush_cache_range(vma, addr, end); 2006 do { 2007 next = pgd_addr_end(addr, end); 2008 err = remap_p4d_range(mm, pgd, addr, next, 2009 pfn + (addr >> PAGE_SHIFT), prot); 2010 if (err) 2011 break; 2012 } while (pgd++, addr = next, addr != end); 2013 2014 if (err) 2015 untrack_pfn(vma, remap_pfn, PAGE_ALIGN(size)); 2016 2017 return err; 2018 } 2019 EXPORT_SYMBOL(remap_pfn_range); 2020 2021 /** 2022 * vm_iomap_memory - remap memory to userspace 2023 * @vma: user vma to map to 2024 * @start: start of the physical memory to be mapped 2025 * @len: size of area 2026 * 2027 * This is a simplified io_remap_pfn_range() for common driver use. The 2028 * driver just needs to give us the physical memory range to be mapped, 2029 * we'll figure out the rest from the vma information. 2030 * 2031 * NOTE! Some drivers might want to tweak vma->vm_page_prot first to get 2032 * whatever write-combining details or similar. 2033 * 2034 * Return: %0 on success, negative error code otherwise. 2035 */ 2036 int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len) 2037 { 2038 unsigned long vm_len, pfn, pages; 2039 2040 /* Check that the physical memory area passed in looks valid */ 2041 if (start + len < start) 2042 return -EINVAL; 2043 /* 2044 * You *really* shouldn't map things that aren't page-aligned, 2045 * but we've historically allowed it because IO memory might 2046 * just have smaller alignment. 2047 */ 2048 len += start & ~PAGE_MASK; 2049 pfn = start >> PAGE_SHIFT; 2050 pages = (len + ~PAGE_MASK) >> PAGE_SHIFT; 2051 if (pfn + pages < pfn) 2052 return -EINVAL; 2053 2054 /* We start the mapping 'vm_pgoff' pages into the area */ 2055 if (vma->vm_pgoff > pages) 2056 return -EINVAL; 2057 pfn += vma->vm_pgoff; 2058 pages -= vma->vm_pgoff; 2059 2060 /* Can we fit all of the mapping? */ 2061 vm_len = vma->vm_end - vma->vm_start; 2062 if (vm_len >> PAGE_SHIFT > pages) 2063 return -EINVAL; 2064 2065 /* Ok, let it rip */ 2066 return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot); 2067 } 2068 EXPORT_SYMBOL(vm_iomap_memory); 2069 2070 static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd, 2071 unsigned long addr, unsigned long end, 2072 pte_fn_t fn, void *data, bool create) 2073 { 2074 pte_t *pte; 2075 int err = 0; 2076 spinlock_t *uninitialized_var(ptl); 2077 2078 if (create) { 2079 pte = (mm == &init_mm) ? 2080 pte_alloc_kernel(pmd, addr) : 2081 pte_alloc_map_lock(mm, pmd, addr, &ptl); 2082 if (!pte) 2083 return -ENOMEM; 2084 } else { 2085 pte = (mm == &init_mm) ? 2086 pte_offset_kernel(pmd, addr) : 2087 pte_offset_map_lock(mm, pmd, addr, &ptl); 2088 } 2089 2090 BUG_ON(pmd_huge(*pmd)); 2091 2092 arch_enter_lazy_mmu_mode(); 2093 2094 do { 2095 if (create || !pte_none(*pte)) { 2096 err = fn(pte++, addr, data); 2097 if (err) 2098 break; 2099 } 2100 } while (addr += PAGE_SIZE, addr != end); 2101 2102 arch_leave_lazy_mmu_mode(); 2103 2104 if (mm != &init_mm) 2105 pte_unmap_unlock(pte-1, ptl); 2106 return err; 2107 } 2108 2109 static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud, 2110 unsigned long addr, unsigned long end, 2111 pte_fn_t fn, void *data, bool create) 2112 { 2113 pmd_t *pmd; 2114 unsigned long next; 2115 int err = 0; 2116 2117 BUG_ON(pud_huge(*pud)); 2118 2119 if (create) { 2120 pmd = pmd_alloc(mm, pud, addr); 2121 if (!pmd) 2122 return -ENOMEM; 2123 } else { 2124 pmd = pmd_offset(pud, addr); 2125 } 2126 do { 2127 next = pmd_addr_end(addr, end); 2128 if (create || !pmd_none_or_clear_bad(pmd)) { 2129 err = apply_to_pte_range(mm, pmd, addr, next, fn, data, 2130 create); 2131 if (err) 2132 break; 2133 } 2134 } while (pmd++, addr = next, addr != end); 2135 return err; 2136 } 2137 2138 static int apply_to_pud_range(struct mm_struct *mm, p4d_t *p4d, 2139 unsigned long addr, unsigned long end, 2140 pte_fn_t fn, void *data, bool create) 2141 { 2142 pud_t *pud; 2143 unsigned long next; 2144 int err = 0; 2145 2146 if (create) { 2147 pud = pud_alloc(mm, p4d, addr); 2148 if (!pud) 2149 return -ENOMEM; 2150 } else { 2151 pud = pud_offset(p4d, addr); 2152 } 2153 do { 2154 next = pud_addr_end(addr, end); 2155 if (create || !pud_none_or_clear_bad(pud)) { 2156 err = apply_to_pmd_range(mm, pud, addr, next, fn, data, 2157 create); 2158 if (err) 2159 break; 2160 } 2161 } while (pud++, addr = next, addr != end); 2162 return err; 2163 } 2164 2165 static int apply_to_p4d_range(struct mm_struct *mm, pgd_t *pgd, 2166 unsigned long addr, unsigned long end, 2167 pte_fn_t fn, void *data, bool create) 2168 { 2169 p4d_t *p4d; 2170 unsigned long next; 2171 int err = 0; 2172 2173 if (create) { 2174 p4d = p4d_alloc(mm, pgd, addr); 2175 if (!p4d) 2176 return -ENOMEM; 2177 } else { 2178 p4d = p4d_offset(pgd, addr); 2179 } 2180 do { 2181 next = p4d_addr_end(addr, end); 2182 if (create || !p4d_none_or_clear_bad(p4d)) { 2183 err = apply_to_pud_range(mm, p4d, addr, next, fn, data, 2184 create); 2185 if (err) 2186 break; 2187 } 2188 } while (p4d++, addr = next, addr != end); 2189 return err; 2190 } 2191 2192 static int __apply_to_page_range(struct mm_struct *mm, unsigned long addr, 2193 unsigned long size, pte_fn_t fn, 2194 void *data, bool create) 2195 { 2196 pgd_t *pgd; 2197 unsigned long next; 2198 unsigned long end = addr + size; 2199 int err = 0; 2200 2201 if (WARN_ON(addr >= end)) 2202 return -EINVAL; 2203 2204 pgd = pgd_offset(mm, addr); 2205 do { 2206 next = pgd_addr_end(addr, end); 2207 if (!create && pgd_none_or_clear_bad(pgd)) 2208 continue; 2209 err = apply_to_p4d_range(mm, pgd, addr, next, fn, data, create); 2210 if (err) 2211 break; 2212 } while (pgd++, addr = next, addr != end); 2213 2214 return err; 2215 } 2216 2217 /* 2218 * Scan a region of virtual memory, filling in page tables as necessary 2219 * and calling a provided function on each leaf page table. 2220 */ 2221 int apply_to_page_range(struct mm_struct *mm, unsigned long addr, 2222 unsigned long size, pte_fn_t fn, void *data) 2223 { 2224 return __apply_to_page_range(mm, addr, size, fn, data, true); 2225 } 2226 EXPORT_SYMBOL_GPL(apply_to_page_range); 2227 2228 /* 2229 * Scan a region of virtual memory, calling a provided function on 2230 * each leaf page table where it exists. 2231 * 2232 * Unlike apply_to_page_range, this does _not_ fill in page tables 2233 * where they are absent. 2234 */ 2235 int apply_to_existing_page_range(struct mm_struct *mm, unsigned long addr, 2236 unsigned long size, pte_fn_t fn, void *data) 2237 { 2238 return __apply_to_page_range(mm, addr, size, fn, data, false); 2239 } 2240 EXPORT_SYMBOL_GPL(apply_to_existing_page_range); 2241 2242 /* 2243 * handle_pte_fault chooses page fault handler according to an entry which was 2244 * read non-atomically. Before making any commitment, on those architectures 2245 * or configurations (e.g. i386 with PAE) which might give a mix of unmatched 2246 * parts, do_swap_page must check under lock before unmapping the pte and 2247 * proceeding (but do_wp_page is only called after already making such a check; 2248 * and do_anonymous_page can safely check later on). 2249 */ 2250 static inline int pte_unmap_same(struct mm_struct *mm, pmd_t *pmd, 2251 pte_t *page_table, pte_t orig_pte) 2252 { 2253 int same = 1; 2254 #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPTION) 2255 if (sizeof(pte_t) > sizeof(unsigned long)) { 2256 spinlock_t *ptl = pte_lockptr(mm, pmd); 2257 spin_lock(ptl); 2258 same = pte_same(*page_table, orig_pte); 2259 spin_unlock(ptl); 2260 } 2261 #endif 2262 pte_unmap(page_table); 2263 return same; 2264 } 2265 2266 static inline bool cow_user_page(struct page *dst, struct page *src, 2267 struct vm_fault *vmf) 2268 { 2269 bool ret; 2270 void *kaddr; 2271 void __user *uaddr; 2272 bool locked = false; 2273 struct vm_area_struct *vma = vmf->vma; 2274 struct mm_struct *mm = vma->vm_mm; 2275 unsigned long addr = vmf->address; 2276 2277 debug_dma_assert_idle(src); 2278 2279 if (likely(src)) { 2280 copy_user_highpage(dst, src, addr, vma); 2281 return true; 2282 } 2283 2284 /* 2285 * If the source page was a PFN mapping, we don't have 2286 * a "struct page" for it. We do a best-effort copy by 2287 * just copying from the original user address. If that 2288 * fails, we just zero-fill it. Live with it. 2289 */ 2290 kaddr = kmap_atomic(dst); 2291 uaddr = (void __user *)(addr & PAGE_MASK); 2292 2293 /* 2294 * On architectures with software "accessed" bits, we would 2295 * take a double page fault, so mark it accessed here. 2296 */ 2297 if (arch_faults_on_old_pte() && !pte_young(vmf->orig_pte)) { 2298 pte_t entry; 2299 2300 vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl); 2301 locked = true; 2302 if (!likely(pte_same(*vmf->pte, vmf->orig_pte))) { 2303 /* 2304 * Other thread has already handled the fault 2305 * and we don't need to do anything. If it's 2306 * not the case, the fault will be triggered 2307 * again on the same address. 2308 */ 2309 ret = false; 2310 goto pte_unlock; 2311 } 2312 2313 entry = pte_mkyoung(vmf->orig_pte); 2314 if (ptep_set_access_flags(vma, addr, vmf->pte, entry, 0)) 2315 update_mmu_cache(vma, addr, vmf->pte); 2316 } 2317 2318 /* 2319 * This really shouldn't fail, because the page is there 2320 * in the page tables. But it might just be unreadable, 2321 * in which case we just give up and fill the result with 2322 * zeroes. 2323 */ 2324 if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) { 2325 if (locked) 2326 goto warn; 2327 2328 /* Re-validate under PTL if the page is still mapped */ 2329 vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl); 2330 locked = true; 2331 if (!likely(pte_same(*vmf->pte, vmf->orig_pte))) { 2332 /* The PTE changed under us. Retry page fault. */ 2333 ret = false; 2334 goto pte_unlock; 2335 } 2336 2337 /* 2338 * The same page can be mapped back since last copy attampt. 2339 * Try to copy again under PTL. 2340 */ 2341 if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) { 2342 /* 2343 * Give a warn in case there can be some obscure 2344 * use-case 2345 */ 2346 warn: 2347 WARN_ON_ONCE(1); 2348 clear_page(kaddr); 2349 } 2350 } 2351 2352 ret = true; 2353 2354 pte_unlock: 2355 if (locked) 2356 pte_unmap_unlock(vmf->pte, vmf->ptl); 2357 kunmap_atomic(kaddr); 2358 flush_dcache_page(dst); 2359 2360 return ret; 2361 } 2362 2363 static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma) 2364 { 2365 struct file *vm_file = vma->vm_file; 2366 2367 if (vm_file) 2368 return mapping_gfp_mask(vm_file->f_mapping) | __GFP_FS | __GFP_IO; 2369 2370 /* 2371 * Special mappings (e.g. VDSO) do not have any file so fake 2372 * a default GFP_KERNEL for them. 2373 */ 2374 return GFP_KERNEL; 2375 } 2376 2377 /* 2378 * Notify the address space that the page is about to become writable so that 2379 * it can prohibit this or wait for the page to get into an appropriate state. 2380 * 2381 * We do this without the lock held, so that it can sleep if it needs to. 2382 */ 2383 static vm_fault_t do_page_mkwrite(struct vm_fault *vmf) 2384 { 2385 vm_fault_t ret; 2386 struct page *page = vmf->page; 2387 unsigned int old_flags = vmf->flags; 2388 2389 vmf->flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE; 2390 2391 if (vmf->vma->vm_file && 2392 IS_SWAPFILE(vmf->vma->vm_file->f_mapping->host)) 2393 return VM_FAULT_SIGBUS; 2394 2395 ret = vmf->vma->vm_ops->page_mkwrite(vmf); 2396 /* Restore original flags so that caller is not surprised */ 2397 vmf->flags = old_flags; 2398 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) 2399 return ret; 2400 if (unlikely(!(ret & VM_FAULT_LOCKED))) { 2401 lock_page(page); 2402 if (!page->mapping) { 2403 unlock_page(page); 2404 return 0; /* retry */ 2405 } 2406 ret |= VM_FAULT_LOCKED; 2407 } else 2408 VM_BUG_ON_PAGE(!PageLocked(page), page); 2409 return ret; 2410 } 2411 2412 /* 2413 * Handle dirtying of a page in shared file mapping on a write fault. 2414 * 2415 * The function expects the page to be locked and unlocks it. 2416 */ 2417 static vm_fault_t fault_dirty_shared_page(struct vm_fault *vmf) 2418 { 2419 struct vm_area_struct *vma = vmf->vma; 2420 struct address_space *mapping; 2421 struct page *page = vmf->page; 2422 bool dirtied; 2423 bool page_mkwrite = vma->vm_ops && vma->vm_ops->page_mkwrite; 2424 2425 dirtied = set_page_dirty(page); 2426 VM_BUG_ON_PAGE(PageAnon(page), page); 2427 /* 2428 * Take a local copy of the address_space - page.mapping may be zeroed 2429 * by truncate after unlock_page(). The address_space itself remains 2430 * pinned by vma->vm_file's reference. We rely on unlock_page()'s 2431 * release semantics to prevent the compiler from undoing this copying. 2432 */ 2433 mapping = page_rmapping(page); 2434 unlock_page(page); 2435 2436 if (!page_mkwrite) 2437 file_update_time(vma->vm_file); 2438 2439 /* 2440 * Throttle page dirtying rate down to writeback speed. 2441 * 2442 * mapping may be NULL here because some device drivers do not 2443 * set page.mapping but still dirty their pages 2444 * 2445 * Drop the mmap_sem before waiting on IO, if we can. The file 2446 * is pinning the mapping, as per above. 2447 */ 2448 if ((dirtied || page_mkwrite) && mapping) { 2449 struct file *fpin; 2450 2451 fpin = maybe_unlock_mmap_for_io(vmf, NULL); 2452 balance_dirty_pages_ratelimited(mapping); 2453 if (fpin) { 2454 fput(fpin); 2455 return VM_FAULT_RETRY; 2456 } 2457 } 2458 2459 return 0; 2460 } 2461 2462 /* 2463 * Handle write page faults for pages that can be reused in the current vma 2464 * 2465 * This can happen either due to the mapping being with the VM_SHARED flag, 2466 * or due to us being the last reference standing to the page. In either 2467 * case, all we need to do here is to mark the page as writable and update 2468 * any related book-keeping. 2469 */ 2470 static inline void wp_page_reuse(struct vm_fault *vmf) 2471 __releases(vmf->ptl) 2472 { 2473 struct vm_area_struct *vma = vmf->vma; 2474 struct page *page = vmf->page; 2475 pte_t entry; 2476 /* 2477 * Clear the pages cpupid information as the existing 2478 * information potentially belongs to a now completely 2479 * unrelated process. 2480 */ 2481 if (page) 2482 page_cpupid_xchg_last(page, (1 << LAST_CPUPID_SHIFT) - 1); 2483 2484 flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte)); 2485 entry = pte_mkyoung(vmf->orig_pte); 2486 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 2487 if (ptep_set_access_flags(vma, vmf->address, vmf->pte, entry, 1)) 2488 update_mmu_cache(vma, vmf->address, vmf->pte); 2489 pte_unmap_unlock(vmf->pte, vmf->ptl); 2490 } 2491 2492 /* 2493 * Handle the case of a page which we actually need to copy to a new page. 2494 * 2495 * Called with mmap_sem locked and the old page referenced, but 2496 * without the ptl held. 2497 * 2498 * High level logic flow: 2499 * 2500 * - Allocate a page, copy the content of the old page to the new one. 2501 * - Handle book keeping and accounting - cgroups, mmu-notifiers, etc. 2502 * - Take the PTL. If the pte changed, bail out and release the allocated page 2503 * - If the pte is still the way we remember it, update the page table and all 2504 * relevant references. This includes dropping the reference the page-table 2505 * held to the old page, as well as updating the rmap. 2506 * - In any case, unlock the PTL and drop the reference we took to the old page. 2507 */ 2508 static vm_fault_t wp_page_copy(struct vm_fault *vmf) 2509 { 2510 struct vm_area_struct *vma = vmf->vma; 2511 struct mm_struct *mm = vma->vm_mm; 2512 struct page *old_page = vmf->page; 2513 struct page *new_page = NULL; 2514 pte_t entry; 2515 int page_copied = 0; 2516 struct mem_cgroup *memcg; 2517 struct mmu_notifier_range range; 2518 2519 if (unlikely(anon_vma_prepare(vma))) 2520 goto oom; 2521 2522 if (is_zero_pfn(pte_pfn(vmf->orig_pte))) { 2523 new_page = alloc_zeroed_user_highpage_movable(vma, 2524 vmf->address); 2525 if (!new_page) 2526 goto oom; 2527 } else { 2528 new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, 2529 vmf->address); 2530 if (!new_page) 2531 goto oom; 2532 2533 if (!cow_user_page(new_page, old_page, vmf)) { 2534 /* 2535 * COW failed, if the fault was solved by other, 2536 * it's fine. If not, userspace would re-fault on 2537 * the same address and we will handle the fault 2538 * from the second attempt. 2539 */ 2540 put_page(new_page); 2541 if (old_page) 2542 put_page(old_page); 2543 return 0; 2544 } 2545 } 2546 2547 if (mem_cgroup_try_charge_delay(new_page, mm, GFP_KERNEL, &memcg, false)) 2548 goto oom_free_new; 2549 2550 __SetPageUptodate(new_page); 2551 2552 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, 2553 vmf->address & PAGE_MASK, 2554 (vmf->address & PAGE_MASK) + PAGE_SIZE); 2555 mmu_notifier_invalidate_range_start(&range); 2556 2557 /* 2558 * Re-check the pte - we dropped the lock 2559 */ 2560 vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); 2561 if (likely(pte_same(*vmf->pte, vmf->orig_pte))) { 2562 if (old_page) { 2563 if (!PageAnon(old_page)) { 2564 dec_mm_counter_fast(mm, 2565 mm_counter_file(old_page)); 2566 inc_mm_counter_fast(mm, MM_ANONPAGES); 2567 } 2568 } else { 2569 inc_mm_counter_fast(mm, MM_ANONPAGES); 2570 } 2571 flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte)); 2572 entry = mk_pte(new_page, vma->vm_page_prot); 2573 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 2574 /* 2575 * Clear the pte entry and flush it first, before updating the 2576 * pte with the new entry. This will avoid a race condition 2577 * seen in the presence of one thread doing SMC and another 2578 * thread doing COW. 2579 */ 2580 ptep_clear_flush_notify(vma, vmf->address, vmf->pte); 2581 page_add_new_anon_rmap(new_page, vma, vmf->address, false); 2582 mem_cgroup_commit_charge(new_page, memcg, false, false); 2583 lru_cache_add_active_or_unevictable(new_page, vma); 2584 /* 2585 * We call the notify macro here because, when using secondary 2586 * mmu page tables (such as kvm shadow page tables), we want the 2587 * new page to be mapped directly into the secondary page table. 2588 */ 2589 set_pte_at_notify(mm, vmf->address, vmf->pte, entry); 2590 update_mmu_cache(vma, vmf->address, vmf->pte); 2591 if (old_page) { 2592 /* 2593 * Only after switching the pte to the new page may 2594 * we remove the mapcount here. Otherwise another 2595 * process may come and find the rmap count decremented 2596 * before the pte is switched to the new page, and 2597 * "reuse" the old page writing into it while our pte 2598 * here still points into it and can be read by other 2599 * threads. 2600 * 2601 * The critical issue is to order this 2602 * page_remove_rmap with the ptp_clear_flush above. 2603 * Those stores are ordered by (if nothing else,) 2604 * the barrier present in the atomic_add_negative 2605 * in page_remove_rmap. 2606 * 2607 * Then the TLB flush in ptep_clear_flush ensures that 2608 * no process can access the old page before the 2609 * decremented mapcount is visible. And the old page 2610 * cannot be reused until after the decremented 2611 * mapcount is visible. So transitively, TLBs to 2612 * old page will be flushed before it can be reused. 2613 */ 2614 page_remove_rmap(old_page, false); 2615 } 2616 2617 /* Free the old page.. */ 2618 new_page = old_page; 2619 page_copied = 1; 2620 } else { 2621 mem_cgroup_cancel_charge(new_page, memcg, false); 2622 } 2623 2624 if (new_page) 2625 put_page(new_page); 2626 2627 pte_unmap_unlock(vmf->pte, vmf->ptl); 2628 /* 2629 * No need to double call mmu_notifier->invalidate_range() callback as 2630 * the above ptep_clear_flush_notify() did already call it. 2631 */ 2632 mmu_notifier_invalidate_range_only_end(&range); 2633 if (old_page) { 2634 /* 2635 * Don't let another task, with possibly unlocked vma, 2636 * keep the mlocked page. 2637 */ 2638 if (page_copied && (vma->vm_flags & VM_LOCKED)) { 2639 lock_page(old_page); /* LRU manipulation */ 2640 if (PageMlocked(old_page)) 2641 munlock_vma_page(old_page); 2642 unlock_page(old_page); 2643 } 2644 put_page(old_page); 2645 } 2646 return page_copied ? VM_FAULT_WRITE : 0; 2647 oom_free_new: 2648 put_page(new_page); 2649 oom: 2650 if (old_page) 2651 put_page(old_page); 2652 return VM_FAULT_OOM; 2653 } 2654 2655 /** 2656 * finish_mkwrite_fault - finish page fault for a shared mapping, making PTE 2657 * writeable once the page is prepared 2658 * 2659 * @vmf: structure describing the fault 2660 * 2661 * This function handles all that is needed to finish a write page fault in a 2662 * shared mapping due to PTE being read-only once the mapped page is prepared. 2663 * It handles locking of PTE and modifying it. 2664 * 2665 * The function expects the page to be locked or other protection against 2666 * concurrent faults / writeback (such as DAX radix tree locks). 2667 * 2668 * Return: %VM_FAULT_WRITE on success, %0 when PTE got changed before 2669 * we acquired PTE lock. 2670 */ 2671 vm_fault_t finish_mkwrite_fault(struct vm_fault *vmf) 2672 { 2673 WARN_ON_ONCE(!(vmf->vma->vm_flags & VM_SHARED)); 2674 vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, vmf->address, 2675 &vmf->ptl); 2676 /* 2677 * We might have raced with another page fault while we released the 2678 * pte_offset_map_lock. 2679 */ 2680 if (!pte_same(*vmf->pte, vmf->orig_pte)) { 2681 pte_unmap_unlock(vmf->pte, vmf->ptl); 2682 return VM_FAULT_NOPAGE; 2683 } 2684 wp_page_reuse(vmf); 2685 return 0; 2686 } 2687 2688 /* 2689 * Handle write page faults for VM_MIXEDMAP or VM_PFNMAP for a VM_SHARED 2690 * mapping 2691 */ 2692 static vm_fault_t wp_pfn_shared(struct vm_fault *vmf) 2693 { 2694 struct vm_area_struct *vma = vmf->vma; 2695 2696 if (vma->vm_ops && vma->vm_ops->pfn_mkwrite) { 2697 vm_fault_t ret; 2698 2699 pte_unmap_unlock(vmf->pte, vmf->ptl); 2700 vmf->flags |= FAULT_FLAG_MKWRITE; 2701 ret = vma->vm_ops->pfn_mkwrite(vmf); 2702 if (ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)) 2703 return ret; 2704 return finish_mkwrite_fault(vmf); 2705 } 2706 wp_page_reuse(vmf); 2707 return VM_FAULT_WRITE; 2708 } 2709 2710 static vm_fault_t wp_page_shared(struct vm_fault *vmf) 2711 __releases(vmf->ptl) 2712 { 2713 struct vm_area_struct *vma = vmf->vma; 2714 vm_fault_t ret = VM_FAULT_WRITE; 2715 2716 get_page(vmf->page); 2717 2718 if (vma->vm_ops && vma->vm_ops->page_mkwrite) { 2719 vm_fault_t tmp; 2720 2721 pte_unmap_unlock(vmf->pte, vmf->ptl); 2722 tmp = do_page_mkwrite(vmf); 2723 if (unlikely(!tmp || (tmp & 2724 (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) { 2725 put_page(vmf->page); 2726 return tmp; 2727 } 2728 tmp = finish_mkwrite_fault(vmf); 2729 if (unlikely(tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) { 2730 unlock_page(vmf->page); 2731 put_page(vmf->page); 2732 return tmp; 2733 } 2734 } else { 2735 wp_page_reuse(vmf); 2736 lock_page(vmf->page); 2737 } 2738 ret |= fault_dirty_shared_page(vmf); 2739 put_page(vmf->page); 2740 2741 return ret; 2742 } 2743 2744 /* 2745 * This routine handles present pages, when users try to write 2746 * to a shared page. It is done by copying the page to a new address 2747 * and decrementing the shared-page counter for the old page. 2748 * 2749 * Note that this routine assumes that the protection checks have been 2750 * done by the caller (the low-level page fault routine in most cases). 2751 * Thus we can safely just mark it writable once we've done any necessary 2752 * COW. 2753 * 2754 * We also mark the page dirty at this point even though the page will 2755 * change only once the write actually happens. This avoids a few races, 2756 * and potentially makes it more efficient. 2757 * 2758 * We enter with non-exclusive mmap_sem (to exclude vma changes, 2759 * but allow concurrent faults), with pte both mapped and locked. 2760 * We return with mmap_sem still held, but pte unmapped and unlocked. 2761 */ 2762 static vm_fault_t do_wp_page(struct vm_fault *vmf) 2763 __releases(vmf->ptl) 2764 { 2765 struct vm_area_struct *vma = vmf->vma; 2766 2767 if (userfaultfd_pte_wp(vma, *vmf->pte)) { 2768 pte_unmap_unlock(vmf->pte, vmf->ptl); 2769 return handle_userfault(vmf, VM_UFFD_WP); 2770 } 2771 2772 vmf->page = vm_normal_page(vma, vmf->address, vmf->orig_pte); 2773 if (!vmf->page) { 2774 /* 2775 * VM_MIXEDMAP !pfn_valid() case, or VM_SOFTDIRTY clear on a 2776 * VM_PFNMAP VMA. 2777 * 2778 * We should not cow pages in a shared writeable mapping. 2779 * Just mark the pages writable and/or call ops->pfn_mkwrite. 2780 */ 2781 if ((vma->vm_flags & (VM_WRITE|VM_SHARED)) == 2782 (VM_WRITE|VM_SHARED)) 2783 return wp_pfn_shared(vmf); 2784 2785 pte_unmap_unlock(vmf->pte, vmf->ptl); 2786 return wp_page_copy(vmf); 2787 } 2788 2789 /* 2790 * Take out anonymous pages first, anonymous shared vmas are 2791 * not dirty accountable. 2792 */ 2793 if (PageAnon(vmf->page)) { 2794 int total_map_swapcount; 2795 if (PageKsm(vmf->page) && (PageSwapCache(vmf->page) || 2796 page_count(vmf->page) != 1)) 2797 goto copy; 2798 if (!trylock_page(vmf->page)) { 2799 get_page(vmf->page); 2800 pte_unmap_unlock(vmf->pte, vmf->ptl); 2801 lock_page(vmf->page); 2802 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, 2803 vmf->address, &vmf->ptl); 2804 if (!pte_same(*vmf->pte, vmf->orig_pte)) { 2805 unlock_page(vmf->page); 2806 pte_unmap_unlock(vmf->pte, vmf->ptl); 2807 put_page(vmf->page); 2808 return 0; 2809 } 2810 put_page(vmf->page); 2811 } 2812 if (PageKsm(vmf->page)) { 2813 bool reused = reuse_ksm_page(vmf->page, vmf->vma, 2814 vmf->address); 2815 unlock_page(vmf->page); 2816 if (!reused) 2817 goto copy; 2818 wp_page_reuse(vmf); 2819 return VM_FAULT_WRITE; 2820 } 2821 if (reuse_swap_page(vmf->page, &total_map_swapcount)) { 2822 if (total_map_swapcount == 1) { 2823 /* 2824 * The page is all ours. Move it to 2825 * our anon_vma so the rmap code will 2826 * not search our parent or siblings. 2827 * Protected against the rmap code by 2828 * the page lock. 2829 */ 2830 page_move_anon_rmap(vmf->page, vma); 2831 } 2832 unlock_page(vmf->page); 2833 wp_page_reuse(vmf); 2834 return VM_FAULT_WRITE; 2835 } 2836 unlock_page(vmf->page); 2837 } else if (unlikely((vma->vm_flags & (VM_WRITE|VM_SHARED)) == 2838 (VM_WRITE|VM_SHARED))) { 2839 return wp_page_shared(vmf); 2840 } 2841 copy: 2842 /* 2843 * Ok, we need to copy. Oh, well.. 2844 */ 2845 get_page(vmf->page); 2846 2847 pte_unmap_unlock(vmf->pte, vmf->ptl); 2848 return wp_page_copy(vmf); 2849 } 2850 2851 static void unmap_mapping_range_vma(struct vm_area_struct *vma, 2852 unsigned long start_addr, unsigned long end_addr, 2853 struct zap_details *details) 2854 { 2855 zap_page_range_single(vma, start_addr, end_addr - start_addr, details); 2856 } 2857 2858 static inline void unmap_mapping_range_tree(struct rb_root_cached *root, 2859 struct zap_details *details) 2860 { 2861 struct vm_area_struct *vma; 2862 pgoff_t vba, vea, zba, zea; 2863 2864 vma_interval_tree_foreach(vma, root, 2865 details->first_index, details->last_index) { 2866 2867 vba = vma->vm_pgoff; 2868 vea = vba + vma_pages(vma) - 1; 2869 zba = details->first_index; 2870 if (zba < vba) 2871 zba = vba; 2872 zea = details->last_index; 2873 if (zea > vea) 2874 zea = vea; 2875 2876 unmap_mapping_range_vma(vma, 2877 ((zba - vba) << PAGE_SHIFT) + vma->vm_start, 2878 ((zea - vba + 1) << PAGE_SHIFT) + vma->vm_start, 2879 details); 2880 } 2881 } 2882 2883 /** 2884 * unmap_mapping_pages() - Unmap pages from processes. 2885 * @mapping: The address space containing pages to be unmapped. 2886 * @start: Index of first page to be unmapped. 2887 * @nr: Number of pages to be unmapped. 0 to unmap to end of file. 2888 * @even_cows: Whether to unmap even private COWed pages. 2889 * 2890 * Unmap the pages in this address space from any userspace process which 2891 * has them mmaped. Generally, you want to remove COWed pages as well when 2892 * a file is being truncated, but not when invalidating pages from the page 2893 * cache. 2894 */ 2895 void unmap_mapping_pages(struct address_space *mapping, pgoff_t start, 2896 pgoff_t nr, bool even_cows) 2897 { 2898 struct zap_details details = { }; 2899 2900 details.check_mapping = even_cows ? NULL : mapping; 2901 details.first_index = start; 2902 details.last_index = start + nr - 1; 2903 if (details.last_index < details.first_index) 2904 details.last_index = ULONG_MAX; 2905 2906 i_mmap_lock_write(mapping); 2907 if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root))) 2908 unmap_mapping_range_tree(&mapping->i_mmap, &details); 2909 i_mmap_unlock_write(mapping); 2910 } 2911 2912 /** 2913 * unmap_mapping_range - unmap the portion of all mmaps in the specified 2914 * address_space corresponding to the specified byte range in the underlying 2915 * file. 2916 * 2917 * @mapping: the address space containing mmaps to be unmapped. 2918 * @holebegin: byte in first page to unmap, relative to the start of 2919 * the underlying file. This will be rounded down to a PAGE_SIZE 2920 * boundary. Note that this is different from truncate_pagecache(), which 2921 * must keep the partial page. In contrast, we must get rid of 2922 * partial pages. 2923 * @holelen: size of prospective hole in bytes. This will be rounded 2924 * up to a PAGE_SIZE boundary. A holelen of zero truncates to the 2925 * end of the file. 2926 * @even_cows: 1 when truncating a file, unmap even private COWed pages; 2927 * but 0 when invalidating pagecache, don't throw away private data. 2928 */ 2929 void unmap_mapping_range(struct address_space *mapping, 2930 loff_t const holebegin, loff_t const holelen, int even_cows) 2931 { 2932 pgoff_t hba = holebegin >> PAGE_SHIFT; 2933 pgoff_t hlen = (holelen + PAGE_SIZE - 1) >> PAGE_SHIFT; 2934 2935 /* Check for overflow. */ 2936 if (sizeof(holelen) > sizeof(hlen)) { 2937 long long holeend = 2938 (holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT; 2939 if (holeend & ~(long long)ULONG_MAX) 2940 hlen = ULONG_MAX - hba + 1; 2941 } 2942 2943 unmap_mapping_pages(mapping, hba, hlen, even_cows); 2944 } 2945 EXPORT_SYMBOL(unmap_mapping_range); 2946 2947 /* 2948 * We enter with non-exclusive mmap_sem (to exclude vma changes, 2949 * but allow concurrent faults), and pte mapped but not yet locked. 2950 * We return with pte unmapped and unlocked. 2951 * 2952 * We return with the mmap_sem locked or unlocked in the same cases 2953 * as does filemap_fault(). 2954 */ 2955 vm_fault_t do_swap_page(struct vm_fault *vmf) 2956 { 2957 struct vm_area_struct *vma = vmf->vma; 2958 struct page *page = NULL, *swapcache; 2959 struct mem_cgroup *memcg; 2960 swp_entry_t entry; 2961 pte_t pte; 2962 int locked; 2963 int exclusive = 0; 2964 vm_fault_t ret = 0; 2965 2966 if (!pte_unmap_same(vma->vm_mm, vmf->pmd, vmf->pte, vmf->orig_pte)) 2967 goto out; 2968 2969 entry = pte_to_swp_entry(vmf->orig_pte); 2970 if (unlikely(non_swap_entry(entry))) { 2971 if (is_migration_entry(entry)) { 2972 migration_entry_wait(vma->vm_mm, vmf->pmd, 2973 vmf->address); 2974 } else if (is_device_private_entry(entry)) { 2975 vmf->page = device_private_entry_to_page(entry); 2976 ret = vmf->page->pgmap->ops->migrate_to_ram(vmf); 2977 } else if (is_hwpoison_entry(entry)) { 2978 ret = VM_FAULT_HWPOISON; 2979 } else { 2980 print_bad_pte(vma, vmf->address, vmf->orig_pte, NULL); 2981 ret = VM_FAULT_SIGBUS; 2982 } 2983 goto out; 2984 } 2985 2986 2987 delayacct_set_flag(DELAYACCT_PF_SWAPIN); 2988 page = lookup_swap_cache(entry, vma, vmf->address); 2989 swapcache = page; 2990 2991 if (!page) { 2992 struct swap_info_struct *si = swp_swap_info(entry); 2993 2994 if (si->flags & SWP_SYNCHRONOUS_IO && 2995 __swap_count(entry) == 1) { 2996 /* skip swapcache */ 2997 page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, 2998 vmf->address); 2999 if (page) { 3000 __SetPageLocked(page); 3001 __SetPageSwapBacked(page); 3002 set_page_private(page, entry.val); 3003 lru_cache_add_anon(page); 3004 swap_readpage(page, true); 3005 } 3006 } else { 3007 page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE, 3008 vmf); 3009 swapcache = page; 3010 } 3011 3012 if (!page) { 3013 /* 3014 * Back out if somebody else faulted in this pte 3015 * while we released the pte lock. 3016 */ 3017 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, 3018 vmf->address, &vmf->ptl); 3019 if (likely(pte_same(*vmf->pte, vmf->orig_pte))) 3020 ret = VM_FAULT_OOM; 3021 delayacct_clear_flag(DELAYACCT_PF_SWAPIN); 3022 goto unlock; 3023 } 3024 3025 /* Had to read the page from swap area: Major fault */ 3026 ret = VM_FAULT_MAJOR; 3027 count_vm_event(PGMAJFAULT); 3028 count_memcg_event_mm(vma->vm_mm, PGMAJFAULT); 3029 } else if (PageHWPoison(page)) { 3030 /* 3031 * hwpoisoned dirty swapcache pages are kept for killing 3032 * owner processes (which may be unknown at hwpoison time) 3033 */ 3034 ret = VM_FAULT_HWPOISON; 3035 delayacct_clear_flag(DELAYACCT_PF_SWAPIN); 3036 goto out_release; 3037 } 3038 3039 locked = lock_page_or_retry(page, vma->vm_mm, vmf->flags); 3040 3041 delayacct_clear_flag(DELAYACCT_PF_SWAPIN); 3042 if (!locked) { 3043 ret |= VM_FAULT_RETRY; 3044 goto out_release; 3045 } 3046 3047 /* 3048 * Make sure try_to_free_swap or reuse_swap_page or swapoff did not 3049 * release the swapcache from under us. The page pin, and pte_same 3050 * test below, are not enough to exclude that. Even if it is still 3051 * swapcache, we need to check that the page's swap has not changed. 3052 */ 3053 if (unlikely((!PageSwapCache(page) || 3054 page_private(page) != entry.val)) && swapcache) 3055 goto out_page; 3056 3057 page = ksm_might_need_to_copy(page, vma, vmf->address); 3058 if (unlikely(!page)) { 3059 ret = VM_FAULT_OOM; 3060 page = swapcache; 3061 goto out_page; 3062 } 3063 3064 if (mem_cgroup_try_charge_delay(page, vma->vm_mm, GFP_KERNEL, 3065 &memcg, false)) { 3066 ret = VM_FAULT_OOM; 3067 goto out_page; 3068 } 3069 3070 /* 3071 * Back out if somebody else already faulted in this pte. 3072 */ 3073 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, 3074 &vmf->ptl); 3075 if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte))) 3076 goto out_nomap; 3077 3078 if (unlikely(!PageUptodate(page))) { 3079 ret = VM_FAULT_SIGBUS; 3080 goto out_nomap; 3081 } 3082 3083 /* 3084 * The page isn't present yet, go ahead with the fault. 3085 * 3086 * Be careful about the sequence of operations here. 3087 * To get its accounting right, reuse_swap_page() must be called 3088 * while the page is counted on swap but not yet in mapcount i.e. 3089 * before page_add_anon_rmap() and swap_free(); try_to_free_swap() 3090 * must be called after the swap_free(), or it will never succeed. 3091 */ 3092 3093 inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES); 3094 dec_mm_counter_fast(vma->vm_mm, MM_SWAPENTS); 3095 pte = mk_pte(page, vma->vm_page_prot); 3096 if ((vmf->flags & FAULT_FLAG_WRITE) && reuse_swap_page(page, NULL)) { 3097 pte = maybe_mkwrite(pte_mkdirty(pte), vma); 3098 vmf->flags &= ~FAULT_FLAG_WRITE; 3099 ret |= VM_FAULT_WRITE; 3100 exclusive = RMAP_EXCLUSIVE; 3101 } 3102 flush_icache_page(vma, page); 3103 if (pte_swp_soft_dirty(vmf->orig_pte)) 3104 pte = pte_mksoft_dirty(pte); 3105 if (pte_swp_uffd_wp(vmf->orig_pte)) { 3106 pte = pte_mkuffd_wp(pte); 3107 pte = pte_wrprotect(pte); 3108 } 3109 set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte); 3110 arch_do_swap_page(vma->vm_mm, vma, vmf->address, pte, vmf->orig_pte); 3111 vmf->orig_pte = pte; 3112 3113 /* ksm created a completely new copy */ 3114 if (unlikely(page != swapcache && swapcache)) { 3115 page_add_new_anon_rmap(page, vma, vmf->address, false); 3116 mem_cgroup_commit_charge(page, memcg, false, false); 3117 lru_cache_add_active_or_unevictable(page, vma); 3118 } else { 3119 do_page_add_anon_rmap(page, vma, vmf->address, exclusive); 3120 mem_cgroup_commit_charge(page, memcg, true, false); 3121 activate_page(page); 3122 } 3123 3124 swap_free(entry); 3125 if (mem_cgroup_swap_full(page) || 3126 (vma->vm_flags & VM_LOCKED) || PageMlocked(page)) 3127 try_to_free_swap(page); 3128 unlock_page(page); 3129 if (page != swapcache && swapcache) { 3130 /* 3131 * Hold the lock to avoid the swap entry to be reused 3132 * until we take the PT lock for the pte_same() check 3133 * (to avoid false positives from pte_same). For 3134 * further safety release the lock after the swap_free 3135 * so that the swap count won't change under a 3136 * parallel locked swapcache. 3137 */ 3138 unlock_page(swapcache); 3139 put_page(swapcache); 3140 } 3141 3142 if (vmf->flags & FAULT_FLAG_WRITE) { 3143 ret |= do_wp_page(vmf); 3144 if (ret & VM_FAULT_ERROR) 3145 ret &= VM_FAULT_ERROR; 3146 goto out; 3147 } 3148 3149 /* No need to invalidate - it was non-present before */ 3150 update_mmu_cache(vma, vmf->address, vmf->pte); 3151 unlock: 3152 pte_unmap_unlock(vmf->pte, vmf->ptl); 3153 out: 3154 return ret; 3155 out_nomap: 3156 mem_cgroup_cancel_charge(page, memcg, false); 3157 pte_unmap_unlock(vmf->pte, vmf->ptl); 3158 out_page: 3159 unlock_page(page); 3160 out_release: 3161 put_page(page); 3162 if (page != swapcache && swapcache) { 3163 unlock_page(swapcache); 3164 put_page(swapcache); 3165 } 3166 return ret; 3167 } 3168 3169 /* 3170 * We enter with non-exclusive mmap_sem (to exclude vma changes, 3171 * but allow concurrent faults), and pte mapped but not yet locked. 3172 * We return with mmap_sem still held, but pte unmapped and unlocked. 3173 */ 3174 static vm_fault_t do_anonymous_page(struct vm_fault *vmf) 3175 { 3176 struct vm_area_struct *vma = vmf->vma; 3177 struct mem_cgroup *memcg; 3178 struct page *page; 3179 vm_fault_t ret = 0; 3180 pte_t entry; 3181 3182 /* File mapping without ->vm_ops ? */ 3183 if (vma->vm_flags & VM_SHARED) 3184 return VM_FAULT_SIGBUS; 3185 3186 /* 3187 * Use pte_alloc() instead of pte_alloc_map(). We can't run 3188 * pte_offset_map() on pmds where a huge pmd might be created 3189 * from a different thread. 3190 * 3191 * pte_alloc_map() is safe to use under down_write(mmap_sem) or when 3192 * parallel threads are excluded by other means. 3193 * 3194 * Here we only have down_read(mmap_sem). 3195 */ 3196 if (pte_alloc(vma->vm_mm, vmf->pmd)) 3197 return VM_FAULT_OOM; 3198 3199 /* See the comment in pte_alloc_one_map() */ 3200 if (unlikely(pmd_trans_unstable(vmf->pmd))) 3201 return 0; 3202 3203 /* Use the zero-page for reads */ 3204 if (!(vmf->flags & FAULT_FLAG_WRITE) && 3205 !mm_forbids_zeropage(vma->vm_mm)) { 3206 entry = pte_mkspecial(pfn_pte(my_zero_pfn(vmf->address), 3207 vma->vm_page_prot)); 3208 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, 3209 vmf->address, &vmf->ptl); 3210 if (!pte_none(*vmf->pte)) 3211 goto unlock; 3212 ret = check_stable_address_space(vma->vm_mm); 3213 if (ret) 3214 goto unlock; 3215 /* Deliver the page fault to userland, check inside PT lock */ 3216 if (userfaultfd_missing(vma)) { 3217 pte_unmap_unlock(vmf->pte, vmf->ptl); 3218 return handle_userfault(vmf, VM_UFFD_MISSING); 3219 } 3220 goto setpte; 3221 } 3222 3223 /* Allocate our own private page. */ 3224 if (unlikely(anon_vma_prepare(vma))) 3225 goto oom; 3226 page = alloc_zeroed_user_highpage_movable(vma, vmf->address); 3227 if (!page) 3228 goto oom; 3229 3230 if (mem_cgroup_try_charge_delay(page, vma->vm_mm, GFP_KERNEL, &memcg, 3231 false)) 3232 goto oom_free_page; 3233 3234 /* 3235 * The memory barrier inside __SetPageUptodate makes sure that 3236 * preceding stores to the page contents become visible before 3237 * the set_pte_at() write. 3238 */ 3239 __SetPageUptodate(page); 3240 3241 entry = mk_pte(page, vma->vm_page_prot); 3242 if (vma->vm_flags & VM_WRITE) 3243 entry = pte_mkwrite(pte_mkdirty(entry)); 3244 3245 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, 3246 &vmf->ptl); 3247 if (!pte_none(*vmf->pte)) 3248 goto release; 3249 3250 ret = check_stable_address_space(vma->vm_mm); 3251 if (ret) 3252 goto release; 3253 3254 /* Deliver the page fault to userland, check inside PT lock */ 3255 if (userfaultfd_missing(vma)) { 3256 pte_unmap_unlock(vmf->pte, vmf->ptl); 3257 mem_cgroup_cancel_charge(page, memcg, false); 3258 put_page(page); 3259 return handle_userfault(vmf, VM_UFFD_MISSING); 3260 } 3261 3262 inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES); 3263 page_add_new_anon_rmap(page, vma, vmf->address, false); 3264 mem_cgroup_commit_charge(page, memcg, false, false); 3265 lru_cache_add_active_or_unevictable(page, vma); 3266 setpte: 3267 set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry); 3268 3269 /* No need to invalidate - it was non-present before */ 3270 update_mmu_cache(vma, vmf->address, vmf->pte); 3271 unlock: 3272 pte_unmap_unlock(vmf->pte, vmf->ptl); 3273 return ret; 3274 release: 3275 mem_cgroup_cancel_charge(page, memcg, false); 3276 put_page(page); 3277 goto unlock; 3278 oom_free_page: 3279 put_page(page); 3280 oom: 3281 return VM_FAULT_OOM; 3282 } 3283 3284 /* 3285 * The mmap_sem must have been held on entry, and may have been 3286 * released depending on flags and vma->vm_ops->fault() return value. 3287 * See filemap_fault() and __lock_page_retry(). 3288 */ 3289 static vm_fault_t __do_fault(struct vm_fault *vmf) 3290 { 3291 struct vm_area_struct *vma = vmf->vma; 3292 vm_fault_t ret; 3293 3294 /* 3295 * Preallocate pte before we take page_lock because this might lead to 3296 * deadlocks for memcg reclaim which waits for pages under writeback: 3297 * lock_page(A) 3298 * SetPageWriteback(A) 3299 * unlock_page(A) 3300 * lock_page(B) 3301 * lock_page(B) 3302 * pte_alloc_pne 3303 * shrink_page_list 3304 * wait_on_page_writeback(A) 3305 * SetPageWriteback(B) 3306 * unlock_page(B) 3307 * # flush A, B to clear the writeback 3308 */ 3309 if (pmd_none(*vmf->pmd) && !vmf->prealloc_pte) { 3310 vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm); 3311 if (!vmf->prealloc_pte) 3312 return VM_FAULT_OOM; 3313 smp_wmb(); /* See comment in __pte_alloc() */ 3314 } 3315 3316 ret = vma->vm_ops->fault(vmf); 3317 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY | 3318 VM_FAULT_DONE_COW))) 3319 return ret; 3320 3321 if (unlikely(PageHWPoison(vmf->page))) { 3322 if (ret & VM_FAULT_LOCKED) 3323 unlock_page(vmf->page); 3324 put_page(vmf->page); 3325 vmf->page = NULL; 3326 return VM_FAULT_HWPOISON; 3327 } 3328 3329 if (unlikely(!(ret & VM_FAULT_LOCKED))) 3330 lock_page(vmf->page); 3331 else 3332 VM_BUG_ON_PAGE(!PageLocked(vmf->page), vmf->page); 3333 3334 return ret; 3335 } 3336 3337 /* 3338 * The ordering of these checks is important for pmds with _PAGE_DEVMAP set. 3339 * If we check pmd_trans_unstable() first we will trip the bad_pmd() check 3340 * inside of pmd_none_or_trans_huge_or_clear_bad(). This will end up correctly 3341 * returning 1 but not before it spams dmesg with the pmd_clear_bad() output. 3342 */ 3343 static int pmd_devmap_trans_unstable(pmd_t *pmd) 3344 { 3345 return pmd_devmap(*pmd) || pmd_trans_unstable(pmd); 3346 } 3347 3348 static vm_fault_t pte_alloc_one_map(struct vm_fault *vmf) 3349 { 3350 struct vm_area_struct *vma = vmf->vma; 3351 3352 if (!pmd_none(*vmf->pmd)) 3353 goto map_pte; 3354 if (vmf->prealloc_pte) { 3355 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 3356 if (unlikely(!pmd_none(*vmf->pmd))) { 3357 spin_unlock(vmf->ptl); 3358 goto map_pte; 3359 } 3360 3361 mm_inc_nr_ptes(vma->vm_mm); 3362 pmd_populate(vma->vm_mm, vmf->pmd, vmf->prealloc_pte); 3363 spin_unlock(vmf->ptl); 3364 vmf->prealloc_pte = NULL; 3365 } else if (unlikely(pte_alloc(vma->vm_mm, vmf->pmd))) { 3366 return VM_FAULT_OOM; 3367 } 3368 map_pte: 3369 /* 3370 * If a huge pmd materialized under us just retry later. Use 3371 * pmd_trans_unstable() via pmd_devmap_trans_unstable() instead of 3372 * pmd_trans_huge() to ensure the pmd didn't become pmd_trans_huge 3373 * under us and then back to pmd_none, as a result of MADV_DONTNEED 3374 * running immediately after a huge pmd fault in a different thread of 3375 * this mm, in turn leading to a misleading pmd_trans_huge() retval. 3376 * All we have to ensure is that it is a regular pmd that we can walk 3377 * with pte_offset_map() and we can do that through an atomic read in 3378 * C, which is what pmd_trans_unstable() provides. 3379 */ 3380 if (pmd_devmap_trans_unstable(vmf->pmd)) 3381 return VM_FAULT_NOPAGE; 3382 3383 /* 3384 * At this point we know that our vmf->pmd points to a page of ptes 3385 * and it cannot become pmd_none(), pmd_devmap() or pmd_trans_huge() 3386 * for the duration of the fault. If a racing MADV_DONTNEED runs and 3387 * we zap the ptes pointed to by our vmf->pmd, the vmf->ptl will still 3388 * be valid and we will re-check to make sure the vmf->pte isn't 3389 * pte_none() under vmf->ptl protection when we return to 3390 * alloc_set_pte(). 3391 */ 3392 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, 3393 &vmf->ptl); 3394 return 0; 3395 } 3396 3397 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3398 static void deposit_prealloc_pte(struct vm_fault *vmf) 3399 { 3400 struct vm_area_struct *vma = vmf->vma; 3401 3402 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, vmf->prealloc_pte); 3403 /* 3404 * We are going to consume the prealloc table, 3405 * count that as nr_ptes. 3406 */ 3407 mm_inc_nr_ptes(vma->vm_mm); 3408 vmf->prealloc_pte = NULL; 3409 } 3410 3411 static vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page) 3412 { 3413 struct vm_area_struct *vma = vmf->vma; 3414 bool write = vmf->flags & FAULT_FLAG_WRITE; 3415 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 3416 pmd_t entry; 3417 int i; 3418 vm_fault_t ret; 3419 3420 if (!transhuge_vma_suitable(vma, haddr)) 3421 return VM_FAULT_FALLBACK; 3422 3423 ret = VM_FAULT_FALLBACK; 3424 page = compound_head(page); 3425 3426 /* 3427 * Archs like ppc64 need additonal space to store information 3428 * related to pte entry. Use the preallocated table for that. 3429 */ 3430 if (arch_needs_pgtable_deposit() && !vmf->prealloc_pte) { 3431 vmf->prealloc_pte = pte_alloc_one(vma->vm_mm); 3432 if (!vmf->prealloc_pte) 3433 return VM_FAULT_OOM; 3434 smp_wmb(); /* See comment in __pte_alloc() */ 3435 } 3436 3437 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 3438 if (unlikely(!pmd_none(*vmf->pmd))) 3439 goto out; 3440 3441 for (i = 0; i < HPAGE_PMD_NR; i++) 3442 flush_icache_page(vma, page + i); 3443 3444 entry = mk_huge_pmd(page, vma->vm_page_prot); 3445 if (write) 3446 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 3447 3448 add_mm_counter(vma->vm_mm, mm_counter_file(page), HPAGE_PMD_NR); 3449 page_add_file_rmap(page, true); 3450 /* 3451 * deposit and withdraw with pmd lock held 3452 */ 3453 if (arch_needs_pgtable_deposit()) 3454 deposit_prealloc_pte(vmf); 3455 3456 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry); 3457 3458 update_mmu_cache_pmd(vma, haddr, vmf->pmd); 3459 3460 /* fault is handled */ 3461 ret = 0; 3462 count_vm_event(THP_FILE_MAPPED); 3463 out: 3464 spin_unlock(vmf->ptl); 3465 return ret; 3466 } 3467 #else 3468 static vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page) 3469 { 3470 BUILD_BUG(); 3471 return 0; 3472 } 3473 #endif 3474 3475 /** 3476 * alloc_set_pte - setup new PTE entry for given page and add reverse page 3477 * mapping. If needed, the fucntion allocates page table or use pre-allocated. 3478 * 3479 * @vmf: fault environment 3480 * @memcg: memcg to charge page (only for private mappings) 3481 * @page: page to map 3482 * 3483 * Caller must take care of unlocking vmf->ptl, if vmf->pte is non-NULL on 3484 * return. 3485 * 3486 * Target users are page handler itself and implementations of 3487 * vm_ops->map_pages. 3488 * 3489 * Return: %0 on success, %VM_FAULT_ code in case of error. 3490 */ 3491 vm_fault_t alloc_set_pte(struct vm_fault *vmf, struct mem_cgroup *memcg, 3492 struct page *page) 3493 { 3494 struct vm_area_struct *vma = vmf->vma; 3495 bool write = vmf->flags & FAULT_FLAG_WRITE; 3496 pte_t entry; 3497 vm_fault_t ret; 3498 3499 if (pmd_none(*vmf->pmd) && PageTransCompound(page)) { 3500 /* THP on COW? */ 3501 VM_BUG_ON_PAGE(memcg, page); 3502 3503 ret = do_set_pmd(vmf, page); 3504 if (ret != VM_FAULT_FALLBACK) 3505 return ret; 3506 } 3507 3508 if (!vmf->pte) { 3509 ret = pte_alloc_one_map(vmf); 3510 if (ret) 3511 return ret; 3512 } 3513 3514 /* Re-check under ptl */ 3515 if (unlikely(!pte_none(*vmf->pte))) 3516 return VM_FAULT_NOPAGE; 3517 3518 flush_icache_page(vma, page); 3519 entry = mk_pte(page, vma->vm_page_prot); 3520 if (write) 3521 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 3522 /* copy-on-write page */ 3523 if (write && !(vma->vm_flags & VM_SHARED)) { 3524 inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES); 3525 page_add_new_anon_rmap(page, vma, vmf->address, false); 3526 mem_cgroup_commit_charge(page, memcg, false, false); 3527 lru_cache_add_active_or_unevictable(page, vma); 3528 } else { 3529 inc_mm_counter_fast(vma->vm_mm, mm_counter_file(page)); 3530 page_add_file_rmap(page, false); 3531 } 3532 set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry); 3533 3534 /* no need to invalidate: a not-present page won't be cached */ 3535 update_mmu_cache(vma, vmf->address, vmf->pte); 3536 3537 return 0; 3538 } 3539 3540 3541 /** 3542 * finish_fault - finish page fault once we have prepared the page to fault 3543 * 3544 * @vmf: structure describing the fault 3545 * 3546 * This function handles all that is needed to finish a page fault once the 3547 * page to fault in is prepared. It handles locking of PTEs, inserts PTE for 3548 * given page, adds reverse page mapping, handles memcg charges and LRU 3549 * addition. 3550 * 3551 * The function expects the page to be locked and on success it consumes a 3552 * reference of a page being mapped (for the PTE which maps it). 3553 * 3554 * Return: %0 on success, %VM_FAULT_ code in case of error. 3555 */ 3556 vm_fault_t finish_fault(struct vm_fault *vmf) 3557 { 3558 struct page *page; 3559 vm_fault_t ret = 0; 3560 3561 /* Did we COW the page? */ 3562 if ((vmf->flags & FAULT_FLAG_WRITE) && 3563 !(vmf->vma->vm_flags & VM_SHARED)) 3564 page = vmf->cow_page; 3565 else 3566 page = vmf->page; 3567 3568 /* 3569 * check even for read faults because we might have lost our CoWed 3570 * page 3571 */ 3572 if (!(vmf->vma->vm_flags & VM_SHARED)) 3573 ret = check_stable_address_space(vmf->vma->vm_mm); 3574 if (!ret) 3575 ret = alloc_set_pte(vmf, vmf->memcg, page); 3576 if (vmf->pte) 3577 pte_unmap_unlock(vmf->pte, vmf->ptl); 3578 return ret; 3579 } 3580 3581 static unsigned long fault_around_bytes __read_mostly = 3582 rounddown_pow_of_two(65536); 3583 3584 #ifdef CONFIG_DEBUG_FS 3585 static int fault_around_bytes_get(void *data, u64 *val) 3586 { 3587 *val = fault_around_bytes; 3588 return 0; 3589 } 3590 3591 /* 3592 * fault_around_bytes must be rounded down to the nearest page order as it's 3593 * what do_fault_around() expects to see. 3594 */ 3595 static int fault_around_bytes_set(void *data, u64 val) 3596 { 3597 if (val / PAGE_SIZE > PTRS_PER_PTE) 3598 return -EINVAL; 3599 if (val > PAGE_SIZE) 3600 fault_around_bytes = rounddown_pow_of_two(val); 3601 else 3602 fault_around_bytes = PAGE_SIZE; /* rounddown_pow_of_two(0) is undefined */ 3603 return 0; 3604 } 3605 DEFINE_DEBUGFS_ATTRIBUTE(fault_around_bytes_fops, 3606 fault_around_bytes_get, fault_around_bytes_set, "%llu\n"); 3607 3608 static int __init fault_around_debugfs(void) 3609 { 3610 debugfs_create_file_unsafe("fault_around_bytes", 0644, NULL, NULL, 3611 &fault_around_bytes_fops); 3612 return 0; 3613 } 3614 late_initcall(fault_around_debugfs); 3615 #endif 3616 3617 /* 3618 * do_fault_around() tries to map few pages around the fault address. The hope 3619 * is that the pages will be needed soon and this will lower the number of 3620 * faults to handle. 3621 * 3622 * It uses vm_ops->map_pages() to map the pages, which skips the page if it's 3623 * not ready to be mapped: not up-to-date, locked, etc. 3624 * 3625 * This function is called with the page table lock taken. In the split ptlock 3626 * case the page table lock only protects only those entries which belong to 3627 * the page table corresponding to the fault address. 3628 * 3629 * This function doesn't cross the VMA boundaries, in order to call map_pages() 3630 * only once. 3631 * 3632 * fault_around_bytes defines how many bytes we'll try to map. 3633 * do_fault_around() expects it to be set to a power of two less than or equal 3634 * to PTRS_PER_PTE. 3635 * 3636 * The virtual address of the area that we map is naturally aligned to 3637 * fault_around_bytes rounded down to the machine page size 3638 * (and therefore to page order). This way it's easier to guarantee 3639 * that we don't cross page table boundaries. 3640 */ 3641 static vm_fault_t do_fault_around(struct vm_fault *vmf) 3642 { 3643 unsigned long address = vmf->address, nr_pages, mask; 3644 pgoff_t start_pgoff = vmf->pgoff; 3645 pgoff_t end_pgoff; 3646 int off; 3647 vm_fault_t ret = 0; 3648 3649 nr_pages = READ_ONCE(fault_around_bytes) >> PAGE_SHIFT; 3650 mask = ~(nr_pages * PAGE_SIZE - 1) & PAGE_MASK; 3651 3652 vmf->address = max(address & mask, vmf->vma->vm_start); 3653 off = ((address - vmf->address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1); 3654 start_pgoff -= off; 3655 3656 /* 3657 * end_pgoff is either the end of the page table, the end of 3658 * the vma or nr_pages from start_pgoff, depending what is nearest. 3659 */ 3660 end_pgoff = start_pgoff - 3661 ((vmf->address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) + 3662 PTRS_PER_PTE - 1; 3663 end_pgoff = min3(end_pgoff, vma_pages(vmf->vma) + vmf->vma->vm_pgoff - 1, 3664 start_pgoff + nr_pages - 1); 3665 3666 if (pmd_none(*vmf->pmd)) { 3667 vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm); 3668 if (!vmf->prealloc_pte) 3669 goto out; 3670 smp_wmb(); /* See comment in __pte_alloc() */ 3671 } 3672 3673 vmf->vma->vm_ops->map_pages(vmf, start_pgoff, end_pgoff); 3674 3675 /* Huge page is mapped? Page fault is solved */ 3676 if (pmd_trans_huge(*vmf->pmd)) { 3677 ret = VM_FAULT_NOPAGE; 3678 goto out; 3679 } 3680 3681 /* ->map_pages() haven't done anything useful. Cold page cache? */ 3682 if (!vmf->pte) 3683 goto out; 3684 3685 /* check if the page fault is solved */ 3686 vmf->pte -= (vmf->address >> PAGE_SHIFT) - (address >> PAGE_SHIFT); 3687 if (!pte_none(*vmf->pte)) 3688 ret = VM_FAULT_NOPAGE; 3689 pte_unmap_unlock(vmf->pte, vmf->ptl); 3690 out: 3691 vmf->address = address; 3692 vmf->pte = NULL; 3693 return ret; 3694 } 3695 3696 static vm_fault_t do_read_fault(struct vm_fault *vmf) 3697 { 3698 struct vm_area_struct *vma = vmf->vma; 3699 vm_fault_t ret = 0; 3700 3701 /* 3702 * Let's call ->map_pages() first and use ->fault() as fallback 3703 * if page by the offset is not ready to be mapped (cold cache or 3704 * something). 3705 */ 3706 if (vma->vm_ops->map_pages && fault_around_bytes >> PAGE_SHIFT > 1) { 3707 ret = do_fault_around(vmf); 3708 if (ret) 3709 return ret; 3710 } 3711 3712 ret = __do_fault(vmf); 3713 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY))) 3714 return ret; 3715 3716 ret |= finish_fault(vmf); 3717 unlock_page(vmf->page); 3718 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY))) 3719 put_page(vmf->page); 3720 return ret; 3721 } 3722 3723 static vm_fault_t do_cow_fault(struct vm_fault *vmf) 3724 { 3725 struct vm_area_struct *vma = vmf->vma; 3726 vm_fault_t ret; 3727 3728 if (unlikely(anon_vma_prepare(vma))) 3729 return VM_FAULT_OOM; 3730 3731 vmf->cow_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vmf->address); 3732 if (!vmf->cow_page) 3733 return VM_FAULT_OOM; 3734 3735 if (mem_cgroup_try_charge_delay(vmf->cow_page, vma->vm_mm, GFP_KERNEL, 3736 &vmf->memcg, false)) { 3737 put_page(vmf->cow_page); 3738 return VM_FAULT_OOM; 3739 } 3740 3741 ret = __do_fault(vmf); 3742 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY))) 3743 goto uncharge_out; 3744 if (ret & VM_FAULT_DONE_COW) 3745 return ret; 3746 3747 copy_user_highpage(vmf->cow_page, vmf->page, vmf->address, vma); 3748 __SetPageUptodate(vmf->cow_page); 3749 3750 ret |= finish_fault(vmf); 3751 unlock_page(vmf->page); 3752 put_page(vmf->page); 3753 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY))) 3754 goto uncharge_out; 3755 return ret; 3756 uncharge_out: 3757 mem_cgroup_cancel_charge(vmf->cow_page, vmf->memcg, false); 3758 put_page(vmf->cow_page); 3759 return ret; 3760 } 3761 3762 static vm_fault_t do_shared_fault(struct vm_fault *vmf) 3763 { 3764 struct vm_area_struct *vma = vmf->vma; 3765 vm_fault_t ret, tmp; 3766 3767 ret = __do_fault(vmf); 3768 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY))) 3769 return ret; 3770 3771 /* 3772 * Check if the backing address space wants to know that the page is 3773 * about to become writable 3774 */ 3775 if (vma->vm_ops->page_mkwrite) { 3776 unlock_page(vmf->page); 3777 tmp = do_page_mkwrite(vmf); 3778 if (unlikely(!tmp || 3779 (tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) { 3780 put_page(vmf->page); 3781 return tmp; 3782 } 3783 } 3784 3785 ret |= finish_fault(vmf); 3786 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | 3787 VM_FAULT_RETRY))) { 3788 unlock_page(vmf->page); 3789 put_page(vmf->page); 3790 return ret; 3791 } 3792 3793 ret |= fault_dirty_shared_page(vmf); 3794 return ret; 3795 } 3796 3797 /* 3798 * We enter with non-exclusive mmap_sem (to exclude vma changes, 3799 * but allow concurrent faults). 3800 * The mmap_sem may have been released depending on flags and our 3801 * return value. See filemap_fault() and __lock_page_or_retry(). 3802 * If mmap_sem is released, vma may become invalid (for example 3803 * by other thread calling munmap()). 3804 */ 3805 static vm_fault_t do_fault(struct vm_fault *vmf) 3806 { 3807 struct vm_area_struct *vma = vmf->vma; 3808 struct mm_struct *vm_mm = vma->vm_mm; 3809 vm_fault_t ret; 3810 3811 /* 3812 * The VMA was not fully populated on mmap() or missing VM_DONTEXPAND 3813 */ 3814 if (!vma->vm_ops->fault) { 3815 /* 3816 * If we find a migration pmd entry or a none pmd entry, which 3817 * should never happen, return SIGBUS 3818 */ 3819 if (unlikely(!pmd_present(*vmf->pmd))) 3820 ret = VM_FAULT_SIGBUS; 3821 else { 3822 vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, 3823 vmf->pmd, 3824 vmf->address, 3825 &vmf->ptl); 3826 /* 3827 * Make sure this is not a temporary clearing of pte 3828 * by holding ptl and checking again. A R/M/W update 3829 * of pte involves: take ptl, clearing the pte so that 3830 * we don't have concurrent modification by hardware 3831 * followed by an update. 3832 */ 3833 if (unlikely(pte_none(*vmf->pte))) 3834 ret = VM_FAULT_SIGBUS; 3835 else 3836 ret = VM_FAULT_NOPAGE; 3837 3838 pte_unmap_unlock(vmf->pte, vmf->ptl); 3839 } 3840 } else if (!(vmf->flags & FAULT_FLAG_WRITE)) 3841 ret = do_read_fault(vmf); 3842 else if (!(vma->vm_flags & VM_SHARED)) 3843 ret = do_cow_fault(vmf); 3844 else 3845 ret = do_shared_fault(vmf); 3846 3847 /* preallocated pagetable is unused: free it */ 3848 if (vmf->prealloc_pte) { 3849 pte_free(vm_mm, vmf->prealloc_pte); 3850 vmf->prealloc_pte = NULL; 3851 } 3852 return ret; 3853 } 3854 3855 static int numa_migrate_prep(struct page *page, struct vm_area_struct *vma, 3856 unsigned long addr, int page_nid, 3857 int *flags) 3858 { 3859 get_page(page); 3860 3861 count_vm_numa_event(NUMA_HINT_FAULTS); 3862 if (page_nid == numa_node_id()) { 3863 count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL); 3864 *flags |= TNF_FAULT_LOCAL; 3865 } 3866 3867 return mpol_misplaced(page, vma, addr); 3868 } 3869 3870 static vm_fault_t do_numa_page(struct vm_fault *vmf) 3871 { 3872 struct vm_area_struct *vma = vmf->vma; 3873 struct page *page = NULL; 3874 int page_nid = NUMA_NO_NODE; 3875 int last_cpupid; 3876 int target_nid; 3877 bool migrated = false; 3878 pte_t pte, old_pte; 3879 bool was_writable = pte_savedwrite(vmf->orig_pte); 3880 int flags = 0; 3881 3882 /* 3883 * The "pte" at this point cannot be used safely without 3884 * validation through pte_unmap_same(). It's of NUMA type but 3885 * the pfn may be screwed if the read is non atomic. 3886 */ 3887 vmf->ptl = pte_lockptr(vma->vm_mm, vmf->pmd); 3888 spin_lock(vmf->ptl); 3889 if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte))) { 3890 pte_unmap_unlock(vmf->pte, vmf->ptl); 3891 goto out; 3892 } 3893 3894 /* 3895 * Make it present again, Depending on how arch implementes non 3896 * accessible ptes, some can allow access by kernel mode. 3897 */ 3898 old_pte = ptep_modify_prot_start(vma, vmf->address, vmf->pte); 3899 pte = pte_modify(old_pte, vma->vm_page_prot); 3900 pte = pte_mkyoung(pte); 3901 if (was_writable) 3902 pte = pte_mkwrite(pte); 3903 ptep_modify_prot_commit(vma, vmf->address, vmf->pte, old_pte, pte); 3904 update_mmu_cache(vma, vmf->address, vmf->pte); 3905 3906 page = vm_normal_page(vma, vmf->address, pte); 3907 if (!page) { 3908 pte_unmap_unlock(vmf->pte, vmf->ptl); 3909 return 0; 3910 } 3911 3912 /* TODO: handle PTE-mapped THP */ 3913 if (PageCompound(page)) { 3914 pte_unmap_unlock(vmf->pte, vmf->ptl); 3915 return 0; 3916 } 3917 3918 /* 3919 * Avoid grouping on RO pages in general. RO pages shouldn't hurt as 3920 * much anyway since they can be in shared cache state. This misses 3921 * the case where a mapping is writable but the process never writes 3922 * to it but pte_write gets cleared during protection updates and 3923 * pte_dirty has unpredictable behaviour between PTE scan updates, 3924 * background writeback, dirty balancing and application behaviour. 3925 */ 3926 if (!pte_write(pte)) 3927 flags |= TNF_NO_GROUP; 3928 3929 /* 3930 * Flag if the page is shared between multiple address spaces. This 3931 * is later used when determining whether to group tasks together 3932 */ 3933 if (page_mapcount(page) > 1 && (vma->vm_flags & VM_SHARED)) 3934 flags |= TNF_SHARED; 3935 3936 last_cpupid = page_cpupid_last(page); 3937 page_nid = page_to_nid(page); 3938 target_nid = numa_migrate_prep(page, vma, vmf->address, page_nid, 3939 &flags); 3940 pte_unmap_unlock(vmf->pte, vmf->ptl); 3941 if (target_nid == NUMA_NO_NODE) { 3942 put_page(page); 3943 goto out; 3944 } 3945 3946 /* Migrate to the requested node */ 3947 migrated = migrate_misplaced_page(page, vma, target_nid); 3948 if (migrated) { 3949 page_nid = target_nid; 3950 flags |= TNF_MIGRATED; 3951 } else 3952 flags |= TNF_MIGRATE_FAIL; 3953 3954 out: 3955 if (page_nid != NUMA_NO_NODE) 3956 task_numa_fault(last_cpupid, page_nid, 1, flags); 3957 return 0; 3958 } 3959 3960 static inline vm_fault_t create_huge_pmd(struct vm_fault *vmf) 3961 { 3962 if (vma_is_anonymous(vmf->vma)) 3963 return do_huge_pmd_anonymous_page(vmf); 3964 if (vmf->vma->vm_ops->huge_fault) 3965 return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PMD); 3966 return VM_FAULT_FALLBACK; 3967 } 3968 3969 /* `inline' is required to avoid gcc 4.1.2 build error */ 3970 static inline vm_fault_t wp_huge_pmd(struct vm_fault *vmf, pmd_t orig_pmd) 3971 { 3972 if (vma_is_anonymous(vmf->vma)) { 3973 if (userfaultfd_huge_pmd_wp(vmf->vma, orig_pmd)) 3974 return handle_userfault(vmf, VM_UFFD_WP); 3975 return do_huge_pmd_wp_page(vmf, orig_pmd); 3976 } 3977 if (vmf->vma->vm_ops->huge_fault) { 3978 vm_fault_t ret = vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PMD); 3979 3980 if (!(ret & VM_FAULT_FALLBACK)) 3981 return ret; 3982 } 3983 3984 /* COW or write-notify handled on pte level: split pmd. */ 3985 __split_huge_pmd(vmf->vma, vmf->pmd, vmf->address, false, NULL); 3986 3987 return VM_FAULT_FALLBACK; 3988 } 3989 3990 static vm_fault_t create_huge_pud(struct vm_fault *vmf) 3991 { 3992 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ 3993 defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) 3994 /* No support for anonymous transparent PUD pages yet */ 3995 if (vma_is_anonymous(vmf->vma)) 3996 goto split; 3997 if (vmf->vma->vm_ops->huge_fault) { 3998 vm_fault_t ret = vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PUD); 3999 4000 if (!(ret & VM_FAULT_FALLBACK)) 4001 return ret; 4002 } 4003 split: 4004 /* COW or write-notify not handled on PUD level: split pud.*/ 4005 __split_huge_pud(vmf->vma, vmf->pud, vmf->address); 4006 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 4007 return VM_FAULT_FALLBACK; 4008 } 4009 4010 static vm_fault_t wp_huge_pud(struct vm_fault *vmf, pud_t orig_pud) 4011 { 4012 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 4013 /* No support for anonymous transparent PUD pages yet */ 4014 if (vma_is_anonymous(vmf->vma)) 4015 return VM_FAULT_FALLBACK; 4016 if (vmf->vma->vm_ops->huge_fault) 4017 return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PUD); 4018 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 4019 return VM_FAULT_FALLBACK; 4020 } 4021 4022 /* 4023 * These routines also need to handle stuff like marking pages dirty 4024 * and/or accessed for architectures that don't do it in hardware (most 4025 * RISC architectures). The early dirtying is also good on the i386. 4026 * 4027 * There is also a hook called "update_mmu_cache()" that architectures 4028 * with external mmu caches can use to update those (ie the Sparc or 4029 * PowerPC hashed page tables that act as extended TLBs). 4030 * 4031 * We enter with non-exclusive mmap_sem (to exclude vma changes, but allow 4032 * concurrent faults). 4033 * 4034 * The mmap_sem may have been released depending on flags and our return value. 4035 * See filemap_fault() and __lock_page_or_retry(). 4036 */ 4037 static vm_fault_t handle_pte_fault(struct vm_fault *vmf) 4038 { 4039 pte_t entry; 4040 4041 if (unlikely(pmd_none(*vmf->pmd))) { 4042 /* 4043 * Leave __pte_alloc() until later: because vm_ops->fault may 4044 * want to allocate huge page, and if we expose page table 4045 * for an instant, it will be difficult to retract from 4046 * concurrent faults and from rmap lookups. 4047 */ 4048 vmf->pte = NULL; 4049 } else { 4050 /* See comment in pte_alloc_one_map() */ 4051 if (pmd_devmap_trans_unstable(vmf->pmd)) 4052 return 0; 4053 /* 4054 * A regular pmd is established and it can't morph into a huge 4055 * pmd from under us anymore at this point because we hold the 4056 * mmap_sem read mode and khugepaged takes it in write mode. 4057 * So now it's safe to run pte_offset_map(). 4058 */ 4059 vmf->pte = pte_offset_map(vmf->pmd, vmf->address); 4060 vmf->orig_pte = *vmf->pte; 4061 4062 /* 4063 * some architectures can have larger ptes than wordsize, 4064 * e.g.ppc44x-defconfig has CONFIG_PTE_64BIT=y and 4065 * CONFIG_32BIT=y, so READ_ONCE cannot guarantee atomic 4066 * accesses. The code below just needs a consistent view 4067 * for the ifs and we later double check anyway with the 4068 * ptl lock held. So here a barrier will do. 4069 */ 4070 barrier(); 4071 if (pte_none(vmf->orig_pte)) { 4072 pte_unmap(vmf->pte); 4073 vmf->pte = NULL; 4074 } 4075 } 4076 4077 if (!vmf->pte) { 4078 if (vma_is_anonymous(vmf->vma)) 4079 return do_anonymous_page(vmf); 4080 else 4081 return do_fault(vmf); 4082 } 4083 4084 if (!pte_present(vmf->orig_pte)) 4085 return do_swap_page(vmf); 4086 4087 if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma)) 4088 return do_numa_page(vmf); 4089 4090 vmf->ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd); 4091 spin_lock(vmf->ptl); 4092 entry = vmf->orig_pte; 4093 if (unlikely(!pte_same(*vmf->pte, entry))) 4094 goto unlock; 4095 if (vmf->flags & FAULT_FLAG_WRITE) { 4096 if (!pte_write(entry)) 4097 return do_wp_page(vmf); 4098 entry = pte_mkdirty(entry); 4099 } 4100 entry = pte_mkyoung(entry); 4101 if (ptep_set_access_flags(vmf->vma, vmf->address, vmf->pte, entry, 4102 vmf->flags & FAULT_FLAG_WRITE)) { 4103 update_mmu_cache(vmf->vma, vmf->address, vmf->pte); 4104 } else { 4105 /* 4106 * This is needed only for protection faults but the arch code 4107 * is not yet telling us if this is a protection fault or not. 4108 * This still avoids useless tlb flushes for .text page faults 4109 * with threads. 4110 */ 4111 if (vmf->flags & FAULT_FLAG_WRITE) 4112 flush_tlb_fix_spurious_fault(vmf->vma, vmf->address); 4113 } 4114 unlock: 4115 pte_unmap_unlock(vmf->pte, vmf->ptl); 4116 return 0; 4117 } 4118 4119 /* 4120 * By the time we get here, we already hold the mm semaphore 4121 * 4122 * The mmap_sem may have been released depending on flags and our 4123 * return value. See filemap_fault() and __lock_page_or_retry(). 4124 */ 4125 static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, 4126 unsigned long address, unsigned int flags) 4127 { 4128 struct vm_fault vmf = { 4129 .vma = vma, 4130 .address = address & PAGE_MASK, 4131 .flags = flags, 4132 .pgoff = linear_page_index(vma, address), 4133 .gfp_mask = __get_fault_gfp_mask(vma), 4134 }; 4135 unsigned int dirty = flags & FAULT_FLAG_WRITE; 4136 struct mm_struct *mm = vma->vm_mm; 4137 pgd_t *pgd; 4138 p4d_t *p4d; 4139 vm_fault_t ret; 4140 4141 pgd = pgd_offset(mm, address); 4142 p4d = p4d_alloc(mm, pgd, address); 4143 if (!p4d) 4144 return VM_FAULT_OOM; 4145 4146 vmf.pud = pud_alloc(mm, p4d, address); 4147 if (!vmf.pud) 4148 return VM_FAULT_OOM; 4149 retry_pud: 4150 if (pud_none(*vmf.pud) && __transparent_hugepage_enabled(vma)) { 4151 ret = create_huge_pud(&vmf); 4152 if (!(ret & VM_FAULT_FALLBACK)) 4153 return ret; 4154 } else { 4155 pud_t orig_pud = *vmf.pud; 4156 4157 barrier(); 4158 if (pud_trans_huge(orig_pud) || pud_devmap(orig_pud)) { 4159 4160 /* NUMA case for anonymous PUDs would go here */ 4161 4162 if (dirty && !pud_write(orig_pud)) { 4163 ret = wp_huge_pud(&vmf, orig_pud); 4164 if (!(ret & VM_FAULT_FALLBACK)) 4165 return ret; 4166 } else { 4167 huge_pud_set_accessed(&vmf, orig_pud); 4168 return 0; 4169 } 4170 } 4171 } 4172 4173 vmf.pmd = pmd_alloc(mm, vmf.pud, address); 4174 if (!vmf.pmd) 4175 return VM_FAULT_OOM; 4176 4177 /* Huge pud page fault raced with pmd_alloc? */ 4178 if (pud_trans_unstable(vmf.pud)) 4179 goto retry_pud; 4180 4181 if (pmd_none(*vmf.pmd) && __transparent_hugepage_enabled(vma)) { 4182 ret = create_huge_pmd(&vmf); 4183 if (!(ret & VM_FAULT_FALLBACK)) 4184 return ret; 4185 } else { 4186 pmd_t orig_pmd = *vmf.pmd; 4187 4188 barrier(); 4189 if (unlikely(is_swap_pmd(orig_pmd))) { 4190 VM_BUG_ON(thp_migration_supported() && 4191 !is_pmd_migration_entry(orig_pmd)); 4192 if (is_pmd_migration_entry(orig_pmd)) 4193 pmd_migration_entry_wait(mm, vmf.pmd); 4194 return 0; 4195 } 4196 if (pmd_trans_huge(orig_pmd) || pmd_devmap(orig_pmd)) { 4197 if (pmd_protnone(orig_pmd) && vma_is_accessible(vma)) 4198 return do_huge_pmd_numa_page(&vmf, orig_pmd); 4199 4200 if (dirty && !pmd_write(orig_pmd)) { 4201 ret = wp_huge_pmd(&vmf, orig_pmd); 4202 if (!(ret & VM_FAULT_FALLBACK)) 4203 return ret; 4204 } else { 4205 huge_pmd_set_accessed(&vmf, orig_pmd); 4206 return 0; 4207 } 4208 } 4209 } 4210 4211 return handle_pte_fault(&vmf); 4212 } 4213 4214 /* 4215 * By the time we get here, we already hold the mm semaphore 4216 * 4217 * The mmap_sem may have been released depending on flags and our 4218 * return value. See filemap_fault() and __lock_page_or_retry(). 4219 */ 4220 vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, 4221 unsigned int flags) 4222 { 4223 vm_fault_t ret; 4224 4225 __set_current_state(TASK_RUNNING); 4226 4227 count_vm_event(PGFAULT); 4228 count_memcg_event_mm(vma->vm_mm, PGFAULT); 4229 4230 /* do counter updates before entering really critical section. */ 4231 check_sync_rss_stat(current); 4232 4233 if (!arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE, 4234 flags & FAULT_FLAG_INSTRUCTION, 4235 flags & FAULT_FLAG_REMOTE)) 4236 return VM_FAULT_SIGSEGV; 4237 4238 /* 4239 * Enable the memcg OOM handling for faults triggered in user 4240 * space. Kernel faults are handled more gracefully. 4241 */ 4242 if (flags & FAULT_FLAG_USER) 4243 mem_cgroup_enter_user_fault(); 4244 4245 if (unlikely(is_vm_hugetlb_page(vma))) 4246 ret = hugetlb_fault(vma->vm_mm, vma, address, flags); 4247 else 4248 ret = __handle_mm_fault(vma, address, flags); 4249 4250 if (flags & FAULT_FLAG_USER) { 4251 mem_cgroup_exit_user_fault(); 4252 /* 4253 * The task may have entered a memcg OOM situation but 4254 * if the allocation error was handled gracefully (no 4255 * VM_FAULT_OOM), there is no need to kill anything. 4256 * Just clean up the OOM state peacefully. 4257 */ 4258 if (task_in_memcg_oom(current) && !(ret & VM_FAULT_OOM)) 4259 mem_cgroup_oom_synchronize(false); 4260 } 4261 4262 return ret; 4263 } 4264 EXPORT_SYMBOL_GPL(handle_mm_fault); 4265 4266 #ifndef __PAGETABLE_P4D_FOLDED 4267 /* 4268 * Allocate p4d page table. 4269 * We've already handled the fast-path in-line. 4270 */ 4271 int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address) 4272 { 4273 p4d_t *new = p4d_alloc_one(mm, address); 4274 if (!new) 4275 return -ENOMEM; 4276 4277 smp_wmb(); /* See comment in __pte_alloc */ 4278 4279 spin_lock(&mm->page_table_lock); 4280 if (pgd_present(*pgd)) /* Another has populated it */ 4281 p4d_free(mm, new); 4282 else 4283 pgd_populate(mm, pgd, new); 4284 spin_unlock(&mm->page_table_lock); 4285 return 0; 4286 } 4287 #endif /* __PAGETABLE_P4D_FOLDED */ 4288 4289 #ifndef __PAGETABLE_PUD_FOLDED 4290 /* 4291 * Allocate page upper directory. 4292 * We've already handled the fast-path in-line. 4293 */ 4294 int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address) 4295 { 4296 pud_t *new = pud_alloc_one(mm, address); 4297 if (!new) 4298 return -ENOMEM; 4299 4300 smp_wmb(); /* See comment in __pte_alloc */ 4301 4302 spin_lock(&mm->page_table_lock); 4303 #ifndef __ARCH_HAS_5LEVEL_HACK 4304 if (!p4d_present(*p4d)) { 4305 mm_inc_nr_puds(mm); 4306 p4d_populate(mm, p4d, new); 4307 } else /* Another has populated it */ 4308 pud_free(mm, new); 4309 #else 4310 if (!pgd_present(*p4d)) { 4311 mm_inc_nr_puds(mm); 4312 pgd_populate(mm, p4d, new); 4313 } else /* Another has populated it */ 4314 pud_free(mm, new); 4315 #endif /* __ARCH_HAS_5LEVEL_HACK */ 4316 spin_unlock(&mm->page_table_lock); 4317 return 0; 4318 } 4319 #endif /* __PAGETABLE_PUD_FOLDED */ 4320 4321 #ifndef __PAGETABLE_PMD_FOLDED 4322 /* 4323 * Allocate page middle directory. 4324 * We've already handled the fast-path in-line. 4325 */ 4326 int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) 4327 { 4328 spinlock_t *ptl; 4329 pmd_t *new = pmd_alloc_one(mm, address); 4330 if (!new) 4331 return -ENOMEM; 4332 4333 smp_wmb(); /* See comment in __pte_alloc */ 4334 4335 ptl = pud_lock(mm, pud); 4336 if (!pud_present(*pud)) { 4337 mm_inc_nr_pmds(mm); 4338 pud_populate(mm, pud, new); 4339 } else /* Another has populated it */ 4340 pmd_free(mm, new); 4341 spin_unlock(ptl); 4342 return 0; 4343 } 4344 #endif /* __PAGETABLE_PMD_FOLDED */ 4345 4346 static int __follow_pte_pmd(struct mm_struct *mm, unsigned long address, 4347 struct mmu_notifier_range *range, 4348 pte_t **ptepp, pmd_t **pmdpp, spinlock_t **ptlp) 4349 { 4350 pgd_t *pgd; 4351 p4d_t *p4d; 4352 pud_t *pud; 4353 pmd_t *pmd; 4354 pte_t *ptep; 4355 4356 pgd = pgd_offset(mm, address); 4357 if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) 4358 goto out; 4359 4360 p4d = p4d_offset(pgd, address); 4361 if (p4d_none(*p4d) || unlikely(p4d_bad(*p4d))) 4362 goto out; 4363 4364 pud = pud_offset(p4d, address); 4365 if (pud_none(*pud) || unlikely(pud_bad(*pud))) 4366 goto out; 4367 4368 pmd = pmd_offset(pud, address); 4369 VM_BUG_ON(pmd_trans_huge(*pmd)); 4370 4371 if (pmd_huge(*pmd)) { 4372 if (!pmdpp) 4373 goto out; 4374 4375 if (range) { 4376 mmu_notifier_range_init(range, MMU_NOTIFY_CLEAR, 0, 4377 NULL, mm, address & PMD_MASK, 4378 (address & PMD_MASK) + PMD_SIZE); 4379 mmu_notifier_invalidate_range_start(range); 4380 } 4381 *ptlp = pmd_lock(mm, pmd); 4382 if (pmd_huge(*pmd)) { 4383 *pmdpp = pmd; 4384 return 0; 4385 } 4386 spin_unlock(*ptlp); 4387 if (range) 4388 mmu_notifier_invalidate_range_end(range); 4389 } 4390 4391 if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd))) 4392 goto out; 4393 4394 if (range) { 4395 mmu_notifier_range_init(range, MMU_NOTIFY_CLEAR, 0, NULL, mm, 4396 address & PAGE_MASK, 4397 (address & PAGE_MASK) + PAGE_SIZE); 4398 mmu_notifier_invalidate_range_start(range); 4399 } 4400 ptep = pte_offset_map_lock(mm, pmd, address, ptlp); 4401 if (!pte_present(*ptep)) 4402 goto unlock; 4403 *ptepp = ptep; 4404 return 0; 4405 unlock: 4406 pte_unmap_unlock(ptep, *ptlp); 4407 if (range) 4408 mmu_notifier_invalidate_range_end(range); 4409 out: 4410 return -EINVAL; 4411 } 4412 4413 static inline int follow_pte(struct mm_struct *mm, unsigned long address, 4414 pte_t **ptepp, spinlock_t **ptlp) 4415 { 4416 int res; 4417 4418 /* (void) is needed to make gcc happy */ 4419 (void) __cond_lock(*ptlp, 4420 !(res = __follow_pte_pmd(mm, address, NULL, 4421 ptepp, NULL, ptlp))); 4422 return res; 4423 } 4424 4425 int follow_pte_pmd(struct mm_struct *mm, unsigned long address, 4426 struct mmu_notifier_range *range, 4427 pte_t **ptepp, pmd_t **pmdpp, spinlock_t **ptlp) 4428 { 4429 int res; 4430 4431 /* (void) is needed to make gcc happy */ 4432 (void) __cond_lock(*ptlp, 4433 !(res = __follow_pte_pmd(mm, address, range, 4434 ptepp, pmdpp, ptlp))); 4435 return res; 4436 } 4437 EXPORT_SYMBOL(follow_pte_pmd); 4438 4439 /** 4440 * follow_pfn - look up PFN at a user virtual address 4441 * @vma: memory mapping 4442 * @address: user virtual address 4443 * @pfn: location to store found PFN 4444 * 4445 * Only IO mappings and raw PFN mappings are allowed. 4446 * 4447 * Return: zero and the pfn at @pfn on success, -ve otherwise. 4448 */ 4449 int follow_pfn(struct vm_area_struct *vma, unsigned long address, 4450 unsigned long *pfn) 4451 { 4452 int ret = -EINVAL; 4453 spinlock_t *ptl; 4454 pte_t *ptep; 4455 4456 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) 4457 return ret; 4458 4459 ret = follow_pte(vma->vm_mm, address, &ptep, &ptl); 4460 if (ret) 4461 return ret; 4462 *pfn = pte_pfn(*ptep); 4463 pte_unmap_unlock(ptep, ptl); 4464 return 0; 4465 } 4466 EXPORT_SYMBOL(follow_pfn); 4467 4468 #ifdef CONFIG_HAVE_IOREMAP_PROT 4469 int follow_phys(struct vm_area_struct *vma, 4470 unsigned long address, unsigned int flags, 4471 unsigned long *prot, resource_size_t *phys) 4472 { 4473 int ret = -EINVAL; 4474 pte_t *ptep, pte; 4475 spinlock_t *ptl; 4476 4477 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) 4478 goto out; 4479 4480 if (follow_pte(vma->vm_mm, address, &ptep, &ptl)) 4481 goto out; 4482 pte = *ptep; 4483 4484 if ((flags & FOLL_WRITE) && !pte_write(pte)) 4485 goto unlock; 4486 4487 *prot = pgprot_val(pte_pgprot(pte)); 4488 *phys = (resource_size_t)pte_pfn(pte) << PAGE_SHIFT; 4489 4490 ret = 0; 4491 unlock: 4492 pte_unmap_unlock(ptep, ptl); 4493 out: 4494 return ret; 4495 } 4496 4497 int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, 4498 void *buf, int len, int write) 4499 { 4500 resource_size_t phys_addr; 4501 unsigned long prot = 0; 4502 void __iomem *maddr; 4503 int offset = addr & (PAGE_SIZE-1); 4504 4505 if (follow_phys(vma, addr, write, &prot, &phys_addr)) 4506 return -EINVAL; 4507 4508 maddr = ioremap_prot(phys_addr, PAGE_ALIGN(len + offset), prot); 4509 if (!maddr) 4510 return -ENOMEM; 4511 4512 if (write) 4513 memcpy_toio(maddr + offset, buf, len); 4514 else 4515 memcpy_fromio(buf, maddr + offset, len); 4516 iounmap(maddr); 4517 4518 return len; 4519 } 4520 EXPORT_SYMBOL_GPL(generic_access_phys); 4521 #endif 4522 4523 /* 4524 * Access another process' address space as given in mm. If non-NULL, use the 4525 * given task for page fault accounting. 4526 */ 4527 int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm, 4528 unsigned long addr, void *buf, int len, unsigned int gup_flags) 4529 { 4530 struct vm_area_struct *vma; 4531 void *old_buf = buf; 4532 int write = gup_flags & FOLL_WRITE; 4533 4534 if (down_read_killable(&mm->mmap_sem)) 4535 return 0; 4536 4537 /* ignore errors, just check how much was successfully transferred */ 4538 while (len) { 4539 int bytes, ret, offset; 4540 void *maddr; 4541 struct page *page = NULL; 4542 4543 ret = get_user_pages_remote(tsk, mm, addr, 1, 4544 gup_flags, &page, &vma, NULL); 4545 if (ret <= 0) { 4546 #ifndef CONFIG_HAVE_IOREMAP_PROT 4547 break; 4548 #else 4549 /* 4550 * Check if this is a VM_IO | VM_PFNMAP VMA, which 4551 * we can access using slightly different code. 4552 */ 4553 vma = find_vma(mm, addr); 4554 if (!vma || vma->vm_start > addr) 4555 break; 4556 if (vma->vm_ops && vma->vm_ops->access) 4557 ret = vma->vm_ops->access(vma, addr, buf, 4558 len, write); 4559 if (ret <= 0) 4560 break; 4561 bytes = ret; 4562 #endif 4563 } else { 4564 bytes = len; 4565 offset = addr & (PAGE_SIZE-1); 4566 if (bytes > PAGE_SIZE-offset) 4567 bytes = PAGE_SIZE-offset; 4568 4569 maddr = kmap(page); 4570 if (write) { 4571 copy_to_user_page(vma, page, addr, 4572 maddr + offset, buf, bytes); 4573 set_page_dirty_lock(page); 4574 } else { 4575 copy_from_user_page(vma, page, addr, 4576 buf, maddr + offset, bytes); 4577 } 4578 kunmap(page); 4579 put_page(page); 4580 } 4581 len -= bytes; 4582 buf += bytes; 4583 addr += bytes; 4584 } 4585 up_read(&mm->mmap_sem); 4586 4587 return buf - old_buf; 4588 } 4589 4590 /** 4591 * access_remote_vm - access another process' address space 4592 * @mm: the mm_struct of the target address space 4593 * @addr: start address to access 4594 * @buf: source or destination buffer 4595 * @len: number of bytes to transfer 4596 * @gup_flags: flags modifying lookup behaviour 4597 * 4598 * The caller must hold a reference on @mm. 4599 * 4600 * Return: number of bytes copied from source to destination. 4601 */ 4602 int access_remote_vm(struct mm_struct *mm, unsigned long addr, 4603 void *buf, int len, unsigned int gup_flags) 4604 { 4605 return __access_remote_vm(NULL, mm, addr, buf, len, gup_flags); 4606 } 4607 4608 /* 4609 * Access another process' address space. 4610 * Source/target buffer must be kernel space, 4611 * Do not walk the page table directly, use get_user_pages 4612 */ 4613 int access_process_vm(struct task_struct *tsk, unsigned long addr, 4614 void *buf, int len, unsigned int gup_flags) 4615 { 4616 struct mm_struct *mm; 4617 int ret; 4618 4619 mm = get_task_mm(tsk); 4620 if (!mm) 4621 return 0; 4622 4623 ret = __access_remote_vm(tsk, mm, addr, buf, len, gup_flags); 4624 4625 mmput(mm); 4626 4627 return ret; 4628 } 4629 EXPORT_SYMBOL_GPL(access_process_vm); 4630 4631 /* 4632 * Print the name of a VMA. 4633 */ 4634 void print_vma_addr(char *prefix, unsigned long ip) 4635 { 4636 struct mm_struct *mm = current->mm; 4637 struct vm_area_struct *vma; 4638 4639 /* 4640 * we might be running from an atomic context so we cannot sleep 4641 */ 4642 if (!down_read_trylock(&mm->mmap_sem)) 4643 return; 4644 4645 vma = find_vma(mm, ip); 4646 if (vma && vma->vm_file) { 4647 struct file *f = vma->vm_file; 4648 char *buf = (char *)__get_free_page(GFP_NOWAIT); 4649 if (buf) { 4650 char *p; 4651 4652 p = file_path(f, buf, PAGE_SIZE); 4653 if (IS_ERR(p)) 4654 p = "?"; 4655 printk("%s%s[%lx+%lx]", prefix, kbasename(p), 4656 vma->vm_start, 4657 vma->vm_end - vma->vm_start); 4658 free_page((unsigned long)buf); 4659 } 4660 } 4661 up_read(&mm->mmap_sem); 4662 } 4663 4664 #if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP) 4665 void __might_fault(const char *file, int line) 4666 { 4667 /* 4668 * Some code (nfs/sunrpc) uses socket ops on kernel memory while 4669 * holding the mmap_sem, this is safe because kernel memory doesn't 4670 * get paged out, therefore we'll never actually fault, and the 4671 * below annotations will generate false positives. 4672 */ 4673 if (uaccess_kernel()) 4674 return; 4675 if (pagefault_disabled()) 4676 return; 4677 __might_sleep(file, line, 0); 4678 #if defined(CONFIG_DEBUG_ATOMIC_SLEEP) 4679 if (current->mm) 4680 might_lock_read(¤t->mm->mmap_sem); 4681 #endif 4682 } 4683 EXPORT_SYMBOL(__might_fault); 4684 #endif 4685 4686 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS) 4687 /* 4688 * Process all subpages of the specified huge page with the specified 4689 * operation. The target subpage will be processed last to keep its 4690 * cache lines hot. 4691 */ 4692 static inline void process_huge_page( 4693 unsigned long addr_hint, unsigned int pages_per_huge_page, 4694 void (*process_subpage)(unsigned long addr, int idx, void *arg), 4695 void *arg) 4696 { 4697 int i, n, base, l; 4698 unsigned long addr = addr_hint & 4699 ~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1); 4700 4701 /* Process target subpage last to keep its cache lines hot */ 4702 might_sleep(); 4703 n = (addr_hint - addr) / PAGE_SIZE; 4704 if (2 * n <= pages_per_huge_page) { 4705 /* If target subpage in first half of huge page */ 4706 base = 0; 4707 l = n; 4708 /* Process subpages at the end of huge page */ 4709 for (i = pages_per_huge_page - 1; i >= 2 * n; i--) { 4710 cond_resched(); 4711 process_subpage(addr + i * PAGE_SIZE, i, arg); 4712 } 4713 } else { 4714 /* If target subpage in second half of huge page */ 4715 base = pages_per_huge_page - 2 * (pages_per_huge_page - n); 4716 l = pages_per_huge_page - n; 4717 /* Process subpages at the begin of huge page */ 4718 for (i = 0; i < base; i++) { 4719 cond_resched(); 4720 process_subpage(addr + i * PAGE_SIZE, i, arg); 4721 } 4722 } 4723 /* 4724 * Process remaining subpages in left-right-left-right pattern 4725 * towards the target subpage 4726 */ 4727 for (i = 0; i < l; i++) { 4728 int left_idx = base + i; 4729 int right_idx = base + 2 * l - 1 - i; 4730 4731 cond_resched(); 4732 process_subpage(addr + left_idx * PAGE_SIZE, left_idx, arg); 4733 cond_resched(); 4734 process_subpage(addr + right_idx * PAGE_SIZE, right_idx, arg); 4735 } 4736 } 4737 4738 static void clear_gigantic_page(struct page *page, 4739 unsigned long addr, 4740 unsigned int pages_per_huge_page) 4741 { 4742 int i; 4743 struct page *p = page; 4744 4745 might_sleep(); 4746 for (i = 0; i < pages_per_huge_page; 4747 i++, p = mem_map_next(p, page, i)) { 4748 cond_resched(); 4749 clear_user_highpage(p, addr + i * PAGE_SIZE); 4750 } 4751 } 4752 4753 static void clear_subpage(unsigned long addr, int idx, void *arg) 4754 { 4755 struct page *page = arg; 4756 4757 clear_user_highpage(page + idx, addr); 4758 } 4759 4760 void clear_huge_page(struct page *page, 4761 unsigned long addr_hint, unsigned int pages_per_huge_page) 4762 { 4763 unsigned long addr = addr_hint & 4764 ~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1); 4765 4766 if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) { 4767 clear_gigantic_page(page, addr, pages_per_huge_page); 4768 return; 4769 } 4770 4771 process_huge_page(addr_hint, pages_per_huge_page, clear_subpage, page); 4772 } 4773 4774 static void copy_user_gigantic_page(struct page *dst, struct page *src, 4775 unsigned long addr, 4776 struct vm_area_struct *vma, 4777 unsigned int pages_per_huge_page) 4778 { 4779 int i; 4780 struct page *dst_base = dst; 4781 struct page *src_base = src; 4782 4783 for (i = 0; i < pages_per_huge_page; ) { 4784 cond_resched(); 4785 copy_user_highpage(dst, src, addr + i*PAGE_SIZE, vma); 4786 4787 i++; 4788 dst = mem_map_next(dst, dst_base, i); 4789 src = mem_map_next(src, src_base, i); 4790 } 4791 } 4792 4793 struct copy_subpage_arg { 4794 struct page *dst; 4795 struct page *src; 4796 struct vm_area_struct *vma; 4797 }; 4798 4799 static void copy_subpage(unsigned long addr, int idx, void *arg) 4800 { 4801 struct copy_subpage_arg *copy_arg = arg; 4802 4803 copy_user_highpage(copy_arg->dst + idx, copy_arg->src + idx, 4804 addr, copy_arg->vma); 4805 } 4806 4807 void copy_user_huge_page(struct page *dst, struct page *src, 4808 unsigned long addr_hint, struct vm_area_struct *vma, 4809 unsigned int pages_per_huge_page) 4810 { 4811 unsigned long addr = addr_hint & 4812 ~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1); 4813 struct copy_subpage_arg arg = { 4814 .dst = dst, 4815 .src = src, 4816 .vma = vma, 4817 }; 4818 4819 if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) { 4820 copy_user_gigantic_page(dst, src, addr, vma, 4821 pages_per_huge_page); 4822 return; 4823 } 4824 4825 process_huge_page(addr_hint, pages_per_huge_page, copy_subpage, &arg); 4826 } 4827 4828 long copy_huge_page_from_user(struct page *dst_page, 4829 const void __user *usr_src, 4830 unsigned int pages_per_huge_page, 4831 bool allow_pagefault) 4832 { 4833 void *src = (void *)usr_src; 4834 void *page_kaddr; 4835 unsigned long i, rc = 0; 4836 unsigned long ret_val = pages_per_huge_page * PAGE_SIZE; 4837 4838 for (i = 0; i < pages_per_huge_page; i++) { 4839 if (allow_pagefault) 4840 page_kaddr = kmap(dst_page + i); 4841 else 4842 page_kaddr = kmap_atomic(dst_page + i); 4843 rc = copy_from_user(page_kaddr, 4844 (const void __user *)(src + i * PAGE_SIZE), 4845 PAGE_SIZE); 4846 if (allow_pagefault) 4847 kunmap(dst_page + i); 4848 else 4849 kunmap_atomic(page_kaddr); 4850 4851 ret_val -= (PAGE_SIZE - rc); 4852 if (rc) 4853 break; 4854 4855 cond_resched(); 4856 } 4857 return ret_val; 4858 } 4859 #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */ 4860 4861 #if USE_SPLIT_PTE_PTLOCKS && ALLOC_SPLIT_PTLOCKS 4862 4863 static struct kmem_cache *page_ptl_cachep; 4864 4865 void __init ptlock_cache_init(void) 4866 { 4867 page_ptl_cachep = kmem_cache_create("page->ptl", sizeof(spinlock_t), 0, 4868 SLAB_PANIC, NULL); 4869 } 4870 4871 bool ptlock_alloc(struct page *page) 4872 { 4873 spinlock_t *ptl; 4874 4875 ptl = kmem_cache_alloc(page_ptl_cachep, GFP_KERNEL); 4876 if (!ptl) 4877 return false; 4878 page->ptl = ptl; 4879 return true; 4880 } 4881 4882 void ptlock_free(struct page *page) 4883 { 4884 kmem_cache_free(page_ptl_cachep, page->ptl); 4885 } 4886 #endif 4887