1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * linux/mm/memory.c
4 *
5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
6 */
7
8 /*
9 * demand-loading started 01.12.91 - seems it is high on the list of
10 * things wanted, and it should be easy to implement. - Linus
11 */
12
13 /*
14 * Ok, demand-loading was easy, shared pages a little bit tricker. Shared
15 * pages started 02.12.91, seems to work. - Linus.
16 *
17 * Tested sharing by executing about 30 /bin/sh: under the old kernel it
18 * would have taken more than the 6M I have free, but it worked well as
19 * far as I could see.
20 *
21 * Also corrected some "invalidate()"s - I wasn't doing enough of them.
22 */
23
24 /*
25 * Real VM (paging to/from disk) started 18.12.91. Much more work and
26 * thought has to go into this. Oh, well..
27 * 19.12.91 - works, somewhat. Sometimes I get faults, don't know why.
28 * Found it. Everything seems to work now.
29 * 20.12.91 - Ok, making the swap-device changeable like the root.
30 */
31
32 /*
33 * 05.04.94 - Multi-page memory management added for v1.1.
34 * Idea by Alex Bligh (alex@cconcepts.co.uk)
35 *
36 * 16.07.99 - Support of BIGMEM added by Gerhard Wichert, Siemens AG
37 * (Gerhard.Wichert@pdb.siemens.de)
38 *
39 * Aug/Sep 2004 Changed to four level page tables (Andi Kleen)
40 */
41
42 #include <linux/kernel_stat.h>
43 #include <linux/mm.h>
44 #include <linux/mm_inline.h>
45 #include <linux/sched/mm.h>
46 #include <linux/sched/coredump.h>
47 #include <linux/sched/numa_balancing.h>
48 #include <linux/sched/task.h>
49 #include <linux/hugetlb.h>
50 #include <linux/mman.h>
51 #include <linux/swap.h>
52 #include <linux/highmem.h>
53 #include <linux/pagemap.h>
54 #include <linux/memremap.h>
55 #include <linux/kmsan.h>
56 #include <linux/ksm.h>
57 #include <linux/rmap.h>
58 #include <linux/export.h>
59 #include <linux/delayacct.h>
60 #include <linux/init.h>
61 #include <linux/pfn_t.h>
62 #include <linux/writeback.h>
63 #include <linux/memcontrol.h>
64 #include <linux/mmu_notifier.h>
65 #include <linux/swapops.h>
66 #include <linux/elf.h>
67 #include <linux/gfp.h>
68 #include <linux/migrate.h>
69 #include <linux/string.h>
70 #include <linux/memory-tiers.h>
71 #include <linux/debugfs.h>
72 #include <linux/userfaultfd_k.h>
73 #include <linux/dax.h>
74 #include <linux/oom.h>
75 #include <linux/numa.h>
76 #include <linux/perf_event.h>
77 #include <linux/ptrace.h>
78 #include <linux/vmalloc.h>
79 #include <linux/sched/sysctl.h>
80
81 #include <trace/events/kmem.h>
82
83 #include <asm/io.h>
84 #include <asm/mmu_context.h>
85 #include <asm/pgalloc.h>
86 #include <linux/uaccess.h>
87 #include <asm/tlb.h>
88 #include <asm/tlbflush.h>
89
90 #include "pgalloc-track.h"
91 #include "internal.h"
92 #include "swap.h"
93
94 #if defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) && !defined(CONFIG_COMPILE_TEST)
95 #warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.
96 #endif
97
98 #ifndef CONFIG_NUMA
99 unsigned long max_mapnr;
100 EXPORT_SYMBOL(max_mapnr);
101
102 struct page *mem_map;
103 EXPORT_SYMBOL(mem_map);
104 #endif
105
106 static vm_fault_t do_fault(struct vm_fault *vmf);
107 static vm_fault_t do_anonymous_page(struct vm_fault *vmf);
108 static bool vmf_pte_changed(struct vm_fault *vmf);
109
110 /*
111 * Return true if the original pte was a uffd-wp pte marker (so the pte was
112 * wr-protected).
113 */
vmf_orig_pte_uffd_wp(struct vm_fault * vmf)114 static bool vmf_orig_pte_uffd_wp(struct vm_fault *vmf)
115 {
116 if (!(vmf->flags & FAULT_FLAG_ORIG_PTE_VALID))
117 return false;
118
119 return pte_marker_uffd_wp(vmf->orig_pte);
120 }
121
122 /*
123 * A number of key systems in x86 including ioremap() rely on the assumption
124 * that high_memory defines the upper bound on direct map memory, then end
125 * of ZONE_NORMAL. Under CONFIG_DISCONTIG this means that max_low_pfn and
126 * highstart_pfn must be the same; there must be no gap between ZONE_NORMAL
127 * and ZONE_HIGHMEM.
128 */
129 void *high_memory;
130 EXPORT_SYMBOL(high_memory);
131
132 /*
133 * Randomize the address space (stacks, mmaps, brk, etc.).
134 *
135 * ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization,
136 * as ancient (libc5 based) binaries can segfault. )
137 */
138 int randomize_va_space __read_mostly =
139 #ifdef CONFIG_COMPAT_BRK
140 1;
141 #else
142 2;
143 #endif
144
145 #ifndef arch_wants_old_prefaulted_pte
arch_wants_old_prefaulted_pte(void)146 static inline bool arch_wants_old_prefaulted_pte(void)
147 {
148 /*
149 * Transitioning a PTE from 'old' to 'young' can be expensive on
150 * some architectures, even if it's performed in hardware. By
151 * default, "false" means prefaulted entries will be 'young'.
152 */
153 return false;
154 }
155 #endif
156
disable_randmaps(char * s)157 static int __init disable_randmaps(char *s)
158 {
159 randomize_va_space = 0;
160 return 1;
161 }
162 __setup("norandmaps", disable_randmaps);
163
164 unsigned long zero_pfn __read_mostly;
165 EXPORT_SYMBOL(zero_pfn);
166
167 unsigned long highest_memmap_pfn __read_mostly;
168
169 /*
170 * CONFIG_MMU architectures set up ZERO_PAGE in their paging_init()
171 */
init_zero_pfn(void)172 static int __init init_zero_pfn(void)
173 {
174 zero_pfn = page_to_pfn(ZERO_PAGE(0));
175 return 0;
176 }
177 early_initcall(init_zero_pfn);
178
mm_trace_rss_stat(struct mm_struct * mm,int member)179 void mm_trace_rss_stat(struct mm_struct *mm, int member)
180 {
181 trace_rss_stat(mm, member);
182 }
183
184 /*
185 * Note: this doesn't free the actual pages themselves. That
186 * has been handled earlier when unmapping all the memory regions.
187 */
free_pte_range(struct mmu_gather * tlb,pmd_t * pmd,unsigned long addr)188 static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
189 unsigned long addr)
190 {
191 pgtable_t token = pmd_pgtable(*pmd);
192 pmd_clear(pmd);
193 pte_free_tlb(tlb, token, addr);
194 mm_dec_nr_ptes(tlb->mm);
195 }
196
free_pmd_range(struct mmu_gather * tlb,pud_t * pud,unsigned long addr,unsigned long end,unsigned long floor,unsigned long ceiling)197 static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
198 unsigned long addr, unsigned long end,
199 unsigned long floor, unsigned long ceiling)
200 {
201 pmd_t *pmd;
202 unsigned long next;
203 unsigned long start;
204
205 start = addr;
206 pmd = pmd_offset(pud, addr);
207 do {
208 next = pmd_addr_end(addr, end);
209 if (pmd_none_or_clear_bad(pmd))
210 continue;
211 free_pte_range(tlb, pmd, addr);
212 } while (pmd++, addr = next, addr != end);
213
214 start &= PUD_MASK;
215 if (start < floor)
216 return;
217 if (ceiling) {
218 ceiling &= PUD_MASK;
219 if (!ceiling)
220 return;
221 }
222 if (end - 1 > ceiling - 1)
223 return;
224
225 pmd = pmd_offset(pud, start);
226 pud_clear(pud);
227 pmd_free_tlb(tlb, pmd, start);
228 mm_dec_nr_pmds(tlb->mm);
229 }
230
free_pud_range(struct mmu_gather * tlb,p4d_t * p4d,unsigned long addr,unsigned long end,unsigned long floor,unsigned long ceiling)231 static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,
232 unsigned long addr, unsigned long end,
233 unsigned long floor, unsigned long ceiling)
234 {
235 pud_t *pud;
236 unsigned long next;
237 unsigned long start;
238
239 start = addr;
240 pud = pud_offset(p4d, addr);
241 do {
242 next = pud_addr_end(addr, end);
243 if (pud_none_or_clear_bad(pud))
244 continue;
245 free_pmd_range(tlb, pud, addr, next, floor, ceiling);
246 } while (pud++, addr = next, addr != end);
247
248 start &= P4D_MASK;
249 if (start < floor)
250 return;
251 if (ceiling) {
252 ceiling &= P4D_MASK;
253 if (!ceiling)
254 return;
255 }
256 if (end - 1 > ceiling - 1)
257 return;
258
259 pud = pud_offset(p4d, start);
260 p4d_clear(p4d);
261 pud_free_tlb(tlb, pud, start);
262 mm_dec_nr_puds(tlb->mm);
263 }
264
free_p4d_range(struct mmu_gather * tlb,pgd_t * pgd,unsigned long addr,unsigned long end,unsigned long floor,unsigned long ceiling)265 static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd,
266 unsigned long addr, unsigned long end,
267 unsigned long floor, unsigned long ceiling)
268 {
269 p4d_t *p4d;
270 unsigned long next;
271 unsigned long start;
272
273 start = addr;
274 p4d = p4d_offset(pgd, addr);
275 do {
276 next = p4d_addr_end(addr, end);
277 if (p4d_none_or_clear_bad(p4d))
278 continue;
279 free_pud_range(tlb, p4d, addr, next, floor, ceiling);
280 } while (p4d++, addr = next, addr != end);
281
282 start &= PGDIR_MASK;
283 if (start < floor)
284 return;
285 if (ceiling) {
286 ceiling &= PGDIR_MASK;
287 if (!ceiling)
288 return;
289 }
290 if (end - 1 > ceiling - 1)
291 return;
292
293 p4d = p4d_offset(pgd, start);
294 pgd_clear(pgd);
295 p4d_free_tlb(tlb, p4d, start);
296 }
297
298 /*
299 * This function frees user-level page tables of a process.
300 */
free_pgd_range(struct mmu_gather * tlb,unsigned long addr,unsigned long end,unsigned long floor,unsigned long ceiling)301 void free_pgd_range(struct mmu_gather *tlb,
302 unsigned long addr, unsigned long end,
303 unsigned long floor, unsigned long ceiling)
304 {
305 pgd_t *pgd;
306 unsigned long next;
307
308 /*
309 * The next few lines have given us lots of grief...
310 *
311 * Why are we testing PMD* at this top level? Because often
312 * there will be no work to do at all, and we'd prefer not to
313 * go all the way down to the bottom just to discover that.
314 *
315 * Why all these "- 1"s? Because 0 represents both the bottom
316 * of the address space and the top of it (using -1 for the
317 * top wouldn't help much: the masks would do the wrong thing).
318 * The rule is that addr 0 and floor 0 refer to the bottom of
319 * the address space, but end 0 and ceiling 0 refer to the top
320 * Comparisons need to use "end - 1" and "ceiling - 1" (though
321 * that end 0 case should be mythical).
322 *
323 * Wherever addr is brought up or ceiling brought down, we must
324 * be careful to reject "the opposite 0" before it confuses the
325 * subsequent tests. But what about where end is brought down
326 * by PMD_SIZE below? no, end can't go down to 0 there.
327 *
328 * Whereas we round start (addr) and ceiling down, by different
329 * masks at different levels, in order to test whether a table
330 * now has no other vmas using it, so can be freed, we don't
331 * bother to round floor or end up - the tests don't need that.
332 */
333
334 addr &= PMD_MASK;
335 if (addr < floor) {
336 addr += PMD_SIZE;
337 if (!addr)
338 return;
339 }
340 if (ceiling) {
341 ceiling &= PMD_MASK;
342 if (!ceiling)
343 return;
344 }
345 if (end - 1 > ceiling - 1)
346 end -= PMD_SIZE;
347 if (addr > end - 1)
348 return;
349 /*
350 * We add page table cache pages with PAGE_SIZE,
351 * (see pte_free_tlb()), flush the tlb if we need
352 */
353 tlb_change_page_size(tlb, PAGE_SIZE);
354 pgd = pgd_offset(tlb->mm, addr);
355 do {
356 next = pgd_addr_end(addr, end);
357 if (pgd_none_or_clear_bad(pgd))
358 continue;
359 free_p4d_range(tlb, pgd, addr, next, floor, ceiling);
360 } while (pgd++, addr = next, addr != end);
361 }
362
free_pgtables(struct mmu_gather * tlb,struct ma_state * mas,struct vm_area_struct * vma,unsigned long floor,unsigned long ceiling,bool mm_wr_locked)363 void free_pgtables(struct mmu_gather *tlb, struct ma_state *mas,
364 struct vm_area_struct *vma, unsigned long floor,
365 unsigned long ceiling, bool mm_wr_locked)
366 {
367 do {
368 unsigned long addr = vma->vm_start;
369 struct vm_area_struct *next;
370
371 /*
372 * Note: USER_PGTABLES_CEILING may be passed as ceiling and may
373 * be 0. This will underflow and is okay.
374 */
375 next = mas_find(mas, ceiling - 1);
376
377 /*
378 * Hide vma from rmap and truncate_pagecache before freeing
379 * pgtables
380 */
381 if (mm_wr_locked)
382 vma_start_write(vma);
383 unlink_anon_vmas(vma);
384 unlink_file_vma(vma);
385
386 if (is_vm_hugetlb_page(vma)) {
387 hugetlb_free_pgd_range(tlb, addr, vma->vm_end,
388 floor, next ? next->vm_start : ceiling);
389 } else {
390 /*
391 * Optimization: gather nearby vmas into one call down
392 */
393 while (next && next->vm_start <= vma->vm_end + PMD_SIZE
394 && !is_vm_hugetlb_page(next)) {
395 vma = next;
396 next = mas_find(mas, ceiling - 1);
397 if (mm_wr_locked)
398 vma_start_write(vma);
399 unlink_anon_vmas(vma);
400 unlink_file_vma(vma);
401 }
402 free_pgd_range(tlb, addr, vma->vm_end,
403 floor, next ? next->vm_start : ceiling);
404 }
405 vma = next;
406 } while (vma);
407 }
408
pmd_install(struct mm_struct * mm,pmd_t * pmd,pgtable_t * pte)409 void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte)
410 {
411 spinlock_t *ptl = pmd_lock(mm, pmd);
412
413 if (likely(pmd_none(*pmd))) { /* Has another populated it ? */
414 mm_inc_nr_ptes(mm);
415 /*
416 * Ensure all pte setup (eg. pte page lock and page clearing) are
417 * visible before the pte is made visible to other CPUs by being
418 * put into page tables.
419 *
420 * The other side of the story is the pointer chasing in the page
421 * table walking code (when walking the page table without locking;
422 * ie. most of the time). Fortunately, these data accesses consist
423 * of a chain of data-dependent loads, meaning most CPUs (alpha
424 * being the notable exception) will already guarantee loads are
425 * seen in-order. See the alpha page table accessors for the
426 * smp_rmb() barriers in page table walking code.
427 */
428 smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */
429 pmd_populate(mm, pmd, *pte);
430 *pte = NULL;
431 }
432 spin_unlock(ptl);
433 }
434
__pte_alloc(struct mm_struct * mm,pmd_t * pmd)435 int __pte_alloc(struct mm_struct *mm, pmd_t *pmd)
436 {
437 pgtable_t new = pte_alloc_one(mm);
438 if (!new)
439 return -ENOMEM;
440
441 pmd_install(mm, pmd, &new);
442 if (new)
443 pte_free(mm, new);
444 return 0;
445 }
446
__pte_alloc_kernel(pmd_t * pmd)447 int __pte_alloc_kernel(pmd_t *pmd)
448 {
449 pte_t *new = pte_alloc_one_kernel(&init_mm);
450 if (!new)
451 return -ENOMEM;
452
453 spin_lock(&init_mm.page_table_lock);
454 if (likely(pmd_none(*pmd))) { /* Has another populated it ? */
455 smp_wmb(); /* See comment in pmd_install() */
456 pmd_populate_kernel(&init_mm, pmd, new);
457 new = NULL;
458 }
459 spin_unlock(&init_mm.page_table_lock);
460 if (new)
461 pte_free_kernel(&init_mm, new);
462 return 0;
463 }
464
init_rss_vec(int * rss)465 static inline void init_rss_vec(int *rss)
466 {
467 memset(rss, 0, sizeof(int) * NR_MM_COUNTERS);
468 }
469
add_mm_rss_vec(struct mm_struct * mm,int * rss)470 static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss)
471 {
472 int i;
473
474 if (current->mm == mm)
475 sync_mm_rss(mm);
476 for (i = 0; i < NR_MM_COUNTERS; i++)
477 if (rss[i])
478 add_mm_counter(mm, i, rss[i]);
479 }
480
481 /*
482 * This function is called to print an error when a bad pte
483 * is found. For example, we might have a PFN-mapped pte in
484 * a region that doesn't allow it.
485 *
486 * The calling function must still handle the error.
487 */
print_bad_pte(struct vm_area_struct * vma,unsigned long addr,pte_t pte,struct page * page)488 static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr,
489 pte_t pte, struct page *page)
490 {
491 pgd_t *pgd = pgd_offset(vma->vm_mm, addr);
492 p4d_t *p4d = p4d_offset(pgd, addr);
493 pud_t *pud = pud_offset(p4d, addr);
494 pmd_t *pmd = pmd_offset(pud, addr);
495 struct address_space *mapping;
496 pgoff_t index;
497 static unsigned long resume;
498 static unsigned long nr_shown;
499 static unsigned long nr_unshown;
500
501 /*
502 * Allow a burst of 60 reports, then keep quiet for that minute;
503 * or allow a steady drip of one report per second.
504 */
505 if (nr_shown == 60) {
506 if (time_before(jiffies, resume)) {
507 nr_unshown++;
508 return;
509 }
510 if (nr_unshown) {
511 pr_alert("BUG: Bad page map: %lu messages suppressed\n",
512 nr_unshown);
513 nr_unshown = 0;
514 }
515 nr_shown = 0;
516 }
517 if (nr_shown++ == 0)
518 resume = jiffies + 60 * HZ;
519
520 mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL;
521 index = linear_page_index(vma, addr);
522
523 pr_alert("BUG: Bad page map in process %s pte:%08llx pmd:%08llx\n",
524 current->comm,
525 (long long)pte_val(pte), (long long)pmd_val(*pmd));
526 if (page)
527 dump_page(page, "bad pte");
528 pr_alert("addr:%px vm_flags:%08lx anon_vma:%px mapping:%px index:%lx\n",
529 (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index);
530 pr_alert("file:%pD fault:%ps mmap:%ps read_folio:%ps\n",
531 vma->vm_file,
532 vma->vm_ops ? vma->vm_ops->fault : NULL,
533 vma->vm_file ? vma->vm_file->f_op->mmap : NULL,
534 mapping ? mapping->a_ops->read_folio : NULL);
535 dump_stack();
536 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
537 }
538
539 /*
540 * vm_normal_page -- This function gets the "struct page" associated with a pte.
541 *
542 * "Special" mappings do not wish to be associated with a "struct page" (either
543 * it doesn't exist, or it exists but they don't want to touch it). In this
544 * case, NULL is returned here. "Normal" mappings do have a struct page.
545 *
546 * There are 2 broad cases. Firstly, an architecture may define a pte_special()
547 * pte bit, in which case this function is trivial. Secondly, an architecture
548 * may not have a spare pte bit, which requires a more complicated scheme,
549 * described below.
550 *
551 * A raw VM_PFNMAP mapping (ie. one that is not COWed) is always considered a
552 * special mapping (even if there are underlying and valid "struct pages").
553 * COWed pages of a VM_PFNMAP are always normal.
554 *
555 * The way we recognize COWed pages within VM_PFNMAP mappings is through the
556 * rules set up by "remap_pfn_range()": the vma will have the VM_PFNMAP bit
557 * set, and the vm_pgoff will point to the first PFN mapped: thus every special
558 * mapping will always honor the rule
559 *
560 * pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT)
561 *
562 * And for normal mappings this is false.
563 *
564 * This restricts such mappings to be a linear translation from virtual address
565 * to pfn. To get around this restriction, we allow arbitrary mappings so long
566 * as the vma is not a COW mapping; in that case, we know that all ptes are
567 * special (because none can have been COWed).
568 *
569 *
570 * In order to support COW of arbitrary special mappings, we have VM_MIXEDMAP.
571 *
572 * VM_MIXEDMAP mappings can likewise contain memory with or without "struct
573 * page" backing, however the difference is that _all_ pages with a struct
574 * page (that is, those where pfn_valid is true) are refcounted and considered
575 * normal pages by the VM. The disadvantage is that pages are refcounted
576 * (which can be slower and simply not an option for some PFNMAP users). The
577 * advantage is that we don't have to follow the strict linearity rule of
578 * PFNMAP mappings in order to support COWable mappings.
579 *
580 */
vm_normal_page(struct vm_area_struct * vma,unsigned long addr,pte_t pte)581 struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
582 pte_t pte)
583 {
584 unsigned long pfn = pte_pfn(pte);
585
586 if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL)) {
587 if (likely(!pte_special(pte)))
588 goto check_pfn;
589 if (vma->vm_ops && vma->vm_ops->find_special_page)
590 return vma->vm_ops->find_special_page(vma, addr);
591 if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
592 return NULL;
593 if (is_zero_pfn(pfn))
594 return NULL;
595 if (pte_devmap(pte))
596 /*
597 * NOTE: New users of ZONE_DEVICE will not set pte_devmap()
598 * and will have refcounts incremented on their struct pages
599 * when they are inserted into PTEs, thus they are safe to
600 * return here. Legacy ZONE_DEVICE pages that set pte_devmap()
601 * do not have refcounts. Example of legacy ZONE_DEVICE is
602 * MEMORY_DEVICE_FS_DAX type in pmem or virtio_fs drivers.
603 */
604 return NULL;
605
606 print_bad_pte(vma, addr, pte, NULL);
607 return NULL;
608 }
609
610 /* !CONFIG_ARCH_HAS_PTE_SPECIAL case follows: */
611
612 if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
613 if (vma->vm_flags & VM_MIXEDMAP) {
614 if (!pfn_valid(pfn))
615 return NULL;
616 goto out;
617 } else {
618 unsigned long off;
619 off = (addr - vma->vm_start) >> PAGE_SHIFT;
620 if (pfn == vma->vm_pgoff + off)
621 return NULL;
622 if (!is_cow_mapping(vma->vm_flags))
623 return NULL;
624 }
625 }
626
627 if (is_zero_pfn(pfn))
628 return NULL;
629
630 check_pfn:
631 if (unlikely(pfn > highest_memmap_pfn)) {
632 print_bad_pte(vma, addr, pte, NULL);
633 return NULL;
634 }
635
636 /*
637 * NOTE! We still have PageReserved() pages in the page tables.
638 * eg. VDSO mappings can cause them to exist.
639 */
640 out:
641 return pfn_to_page(pfn);
642 }
643
vm_normal_folio(struct vm_area_struct * vma,unsigned long addr,pte_t pte)644 struct folio *vm_normal_folio(struct vm_area_struct *vma, unsigned long addr,
645 pte_t pte)
646 {
647 struct page *page = vm_normal_page(vma, addr, pte);
648
649 if (page)
650 return page_folio(page);
651 return NULL;
652 }
653
654 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
vm_normal_page_pmd(struct vm_area_struct * vma,unsigned long addr,pmd_t pmd)655 struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
656 pmd_t pmd)
657 {
658 unsigned long pfn = pmd_pfn(pmd);
659
660 /*
661 * There is no pmd_special() but there may be special pmds, e.g.
662 * in a direct-access (dax) mapping, so let's just replicate the
663 * !CONFIG_ARCH_HAS_PTE_SPECIAL case from vm_normal_page() here.
664 */
665 if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
666 if (vma->vm_flags & VM_MIXEDMAP) {
667 if (!pfn_valid(pfn))
668 return NULL;
669 goto out;
670 } else {
671 unsigned long off;
672 off = (addr - vma->vm_start) >> PAGE_SHIFT;
673 if (pfn == vma->vm_pgoff + off)
674 return NULL;
675 if (!is_cow_mapping(vma->vm_flags))
676 return NULL;
677 }
678 }
679
680 if (pmd_devmap(pmd))
681 return NULL;
682 if (is_huge_zero_pmd(pmd))
683 return NULL;
684 if (unlikely(pfn > highest_memmap_pfn))
685 return NULL;
686
687 /*
688 * NOTE! We still have PageReserved() pages in the page tables.
689 * eg. VDSO mappings can cause them to exist.
690 */
691 out:
692 return pfn_to_page(pfn);
693 }
694 #endif
695
restore_exclusive_pte(struct vm_area_struct * vma,struct page * page,unsigned long address,pte_t * ptep)696 static void restore_exclusive_pte(struct vm_area_struct *vma,
697 struct page *page, unsigned long address,
698 pte_t *ptep)
699 {
700 pte_t orig_pte;
701 pte_t pte;
702 swp_entry_t entry;
703
704 orig_pte = ptep_get(ptep);
705 pte = pte_mkold(mk_pte(page, READ_ONCE(vma->vm_page_prot)));
706 if (pte_swp_soft_dirty(orig_pte))
707 pte = pte_mksoft_dirty(pte);
708
709 entry = pte_to_swp_entry(orig_pte);
710 if (pte_swp_uffd_wp(orig_pte))
711 pte = pte_mkuffd_wp(pte);
712 else if (is_writable_device_exclusive_entry(entry))
713 pte = maybe_mkwrite(pte_mkdirty(pte), vma);
714
715 VM_BUG_ON(pte_write(pte) && !(PageAnon(page) && PageAnonExclusive(page)));
716
717 /*
718 * No need to take a page reference as one was already
719 * created when the swap entry was made.
720 */
721 if (PageAnon(page))
722 page_add_anon_rmap(page, vma, address, RMAP_NONE);
723 else
724 /*
725 * Currently device exclusive access only supports anonymous
726 * memory so the entry shouldn't point to a filebacked page.
727 */
728 WARN_ON_ONCE(1);
729
730 set_pte_at(vma->vm_mm, address, ptep, pte);
731
732 /*
733 * No need to invalidate - it was non-present before. However
734 * secondary CPUs may have mappings that need invalidating.
735 */
736 update_mmu_cache(vma, address, ptep);
737 }
738
739 /*
740 * Tries to restore an exclusive pte if the page lock can be acquired without
741 * sleeping.
742 */
743 static int
try_restore_exclusive_pte(pte_t * src_pte,struct vm_area_struct * vma,unsigned long addr)744 try_restore_exclusive_pte(pte_t *src_pte, struct vm_area_struct *vma,
745 unsigned long addr)
746 {
747 swp_entry_t entry = pte_to_swp_entry(ptep_get(src_pte));
748 struct page *page = pfn_swap_entry_to_page(entry);
749
750 if (trylock_page(page)) {
751 restore_exclusive_pte(vma, page, addr, src_pte);
752 unlock_page(page);
753 return 0;
754 }
755
756 return -EBUSY;
757 }
758
759 /*
760 * copy one vm_area from one task to the other. Assumes the page tables
761 * already present in the new task to be cleared in the whole range
762 * covered by this vma.
763 */
764
765 static unsigned long
copy_nonpresent_pte(struct mm_struct * dst_mm,struct mm_struct * src_mm,pte_t * dst_pte,pte_t * src_pte,struct vm_area_struct * dst_vma,struct vm_area_struct * src_vma,unsigned long addr,int * rss)766 copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
767 pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *dst_vma,
768 struct vm_area_struct *src_vma, unsigned long addr, int *rss)
769 {
770 unsigned long vm_flags = dst_vma->vm_flags;
771 pte_t orig_pte = ptep_get(src_pte);
772 pte_t pte = orig_pte;
773 struct page *page;
774 swp_entry_t entry = pte_to_swp_entry(orig_pte);
775
776 if (likely(!non_swap_entry(entry))) {
777 if (swap_duplicate(entry) < 0)
778 return -EIO;
779
780 /* make sure dst_mm is on swapoff's mmlist. */
781 if (unlikely(list_empty(&dst_mm->mmlist))) {
782 spin_lock(&mmlist_lock);
783 if (list_empty(&dst_mm->mmlist))
784 list_add(&dst_mm->mmlist,
785 &src_mm->mmlist);
786 spin_unlock(&mmlist_lock);
787 }
788 /* Mark the swap entry as shared. */
789 if (pte_swp_exclusive(orig_pte)) {
790 pte = pte_swp_clear_exclusive(orig_pte);
791 set_pte_at(src_mm, addr, src_pte, pte);
792 }
793 rss[MM_SWAPENTS]++;
794 } else if (is_migration_entry(entry)) {
795 page = pfn_swap_entry_to_page(entry);
796
797 rss[mm_counter(page)]++;
798
799 if (!is_readable_migration_entry(entry) &&
800 is_cow_mapping(vm_flags)) {
801 /*
802 * COW mappings require pages in both parent and child
803 * to be set to read. A previously exclusive entry is
804 * now shared.
805 */
806 entry = make_readable_migration_entry(
807 swp_offset(entry));
808 pte = swp_entry_to_pte(entry);
809 if (pte_swp_soft_dirty(orig_pte))
810 pte = pte_swp_mksoft_dirty(pte);
811 if (pte_swp_uffd_wp(orig_pte))
812 pte = pte_swp_mkuffd_wp(pte);
813 set_pte_at(src_mm, addr, src_pte, pte);
814 }
815 } else if (is_device_private_entry(entry)) {
816 page = pfn_swap_entry_to_page(entry);
817
818 /*
819 * Update rss count even for unaddressable pages, as
820 * they should treated just like normal pages in this
821 * respect.
822 *
823 * We will likely want to have some new rss counters
824 * for unaddressable pages, at some point. But for now
825 * keep things as they are.
826 */
827 get_page(page);
828 rss[mm_counter(page)]++;
829 /* Cannot fail as these pages cannot get pinned. */
830 BUG_ON(page_try_dup_anon_rmap(page, false, src_vma));
831
832 /*
833 * We do not preserve soft-dirty information, because so
834 * far, checkpoint/restore is the only feature that
835 * requires that. And checkpoint/restore does not work
836 * when a device driver is involved (you cannot easily
837 * save and restore device driver state).
838 */
839 if (is_writable_device_private_entry(entry) &&
840 is_cow_mapping(vm_flags)) {
841 entry = make_readable_device_private_entry(
842 swp_offset(entry));
843 pte = swp_entry_to_pte(entry);
844 if (pte_swp_uffd_wp(orig_pte))
845 pte = pte_swp_mkuffd_wp(pte);
846 set_pte_at(src_mm, addr, src_pte, pte);
847 }
848 } else if (is_device_exclusive_entry(entry)) {
849 /*
850 * Make device exclusive entries present by restoring the
851 * original entry then copying as for a present pte. Device
852 * exclusive entries currently only support private writable
853 * (ie. COW) mappings.
854 */
855 VM_BUG_ON(!is_cow_mapping(src_vma->vm_flags));
856 if (try_restore_exclusive_pte(src_pte, src_vma, addr))
857 return -EBUSY;
858 return -ENOENT;
859 } else if (is_pte_marker_entry(entry)) {
860 pte_marker marker = copy_pte_marker(entry, dst_vma);
861
862 if (marker)
863 set_pte_at(dst_mm, addr, dst_pte,
864 make_pte_marker(marker));
865 return 0;
866 }
867 if (!userfaultfd_wp(dst_vma))
868 pte = pte_swp_clear_uffd_wp(pte);
869 set_pte_at(dst_mm, addr, dst_pte, pte);
870 return 0;
871 }
872
873 /*
874 * Copy a present and normal page.
875 *
876 * NOTE! The usual case is that this isn't required;
877 * instead, the caller can just increase the page refcount
878 * and re-use the pte the traditional way.
879 *
880 * And if we need a pre-allocated page but don't yet have
881 * one, return a negative error to let the preallocation
882 * code know so that it can do so outside the page table
883 * lock.
884 */
885 static inline int
copy_present_page(struct vm_area_struct * dst_vma,struct vm_area_struct * src_vma,pte_t * dst_pte,pte_t * src_pte,unsigned long addr,int * rss,struct folio ** prealloc,struct page * page)886 copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
887 pte_t *dst_pte, pte_t *src_pte, unsigned long addr, int *rss,
888 struct folio **prealloc, struct page *page)
889 {
890 struct folio *new_folio;
891 pte_t pte;
892
893 new_folio = *prealloc;
894 if (!new_folio)
895 return -EAGAIN;
896
897 /*
898 * We have a prealloc page, all good! Take it
899 * over and copy the page & arm it.
900 */
901 *prealloc = NULL;
902 copy_user_highpage(&new_folio->page, page, addr, src_vma);
903 __folio_mark_uptodate(new_folio);
904 folio_add_new_anon_rmap(new_folio, dst_vma, addr);
905 folio_add_lru_vma(new_folio, dst_vma);
906 rss[MM_ANONPAGES]++;
907
908 /* All done, just insert the new page copy in the child */
909 pte = mk_pte(&new_folio->page, dst_vma->vm_page_prot);
910 pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma);
911 if (userfaultfd_pte_wp(dst_vma, ptep_get(src_pte)))
912 /* Uffd-wp needs to be delivered to dest pte as well */
913 pte = pte_mkuffd_wp(pte);
914 set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte);
915 return 0;
916 }
917
918 /*
919 * Copy one pte. Returns 0 if succeeded, or -EAGAIN if one preallocated page
920 * is required to copy this pte.
921 */
922 static inline int
copy_present_pte(struct vm_area_struct * dst_vma,struct vm_area_struct * src_vma,pte_t * dst_pte,pte_t * src_pte,unsigned long addr,int * rss,struct folio ** prealloc)923 copy_present_pte(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
924 pte_t *dst_pte, pte_t *src_pte, unsigned long addr, int *rss,
925 struct folio **prealloc)
926 {
927 struct mm_struct *src_mm = src_vma->vm_mm;
928 unsigned long vm_flags = src_vma->vm_flags;
929 pte_t pte = ptep_get(src_pte);
930 struct page *page;
931 struct folio *folio;
932
933 page = vm_normal_page(src_vma, addr, pte);
934 if (page)
935 folio = page_folio(page);
936 if (page && folio_test_anon(folio)) {
937 /*
938 * If this page may have been pinned by the parent process,
939 * copy the page immediately for the child so that we'll always
940 * guarantee the pinned page won't be randomly replaced in the
941 * future.
942 */
943 folio_get(folio);
944 if (unlikely(page_try_dup_anon_rmap(page, false, src_vma))) {
945 /* Page may be pinned, we have to copy. */
946 folio_put(folio);
947 return copy_present_page(dst_vma, src_vma, dst_pte, src_pte,
948 addr, rss, prealloc, page);
949 }
950 rss[MM_ANONPAGES]++;
951 } else if (page) {
952 folio_get(folio);
953 page_dup_file_rmap(page, false);
954 rss[mm_counter_file(page)]++;
955 }
956
957 /*
958 * If it's a COW mapping, write protect it both
959 * in the parent and the child
960 */
961 if (is_cow_mapping(vm_flags) && pte_write(pte)) {
962 ptep_set_wrprotect(src_mm, addr, src_pte);
963 pte = pte_wrprotect(pte);
964 }
965 VM_BUG_ON(page && folio_test_anon(folio) && PageAnonExclusive(page));
966
967 /*
968 * If it's a shared mapping, mark it clean in
969 * the child
970 */
971 if (vm_flags & VM_SHARED)
972 pte = pte_mkclean(pte);
973 pte = pte_mkold(pte);
974
975 if (!userfaultfd_wp(dst_vma))
976 pte = pte_clear_uffd_wp(pte);
977
978 set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte);
979 return 0;
980 }
981
page_copy_prealloc(struct mm_struct * src_mm,struct vm_area_struct * vma,unsigned long addr)982 static inline struct folio *page_copy_prealloc(struct mm_struct *src_mm,
983 struct vm_area_struct *vma, unsigned long addr)
984 {
985 struct folio *new_folio;
986
987 new_folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma, addr, false);
988 if (!new_folio)
989 return NULL;
990
991 if (mem_cgroup_charge(new_folio, src_mm, GFP_KERNEL)) {
992 folio_put(new_folio);
993 return NULL;
994 }
995 folio_throttle_swaprate(new_folio, GFP_KERNEL);
996
997 return new_folio;
998 }
999
1000 static int
copy_pte_range(struct vm_area_struct * dst_vma,struct vm_area_struct * src_vma,pmd_t * dst_pmd,pmd_t * src_pmd,unsigned long addr,unsigned long end)1001 copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1002 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
1003 unsigned long end)
1004 {
1005 struct mm_struct *dst_mm = dst_vma->vm_mm;
1006 struct mm_struct *src_mm = src_vma->vm_mm;
1007 pte_t *orig_src_pte, *orig_dst_pte;
1008 pte_t *src_pte, *dst_pte;
1009 pte_t ptent;
1010 spinlock_t *src_ptl, *dst_ptl;
1011 int progress, ret = 0;
1012 int rss[NR_MM_COUNTERS];
1013 swp_entry_t entry = (swp_entry_t){0};
1014 struct folio *prealloc = NULL;
1015
1016 again:
1017 progress = 0;
1018 init_rss_vec(rss);
1019
1020 /*
1021 * copy_pmd_range()'s prior pmd_none_or_clear_bad(src_pmd), and the
1022 * error handling here, assume that exclusive mmap_lock on dst and src
1023 * protects anon from unexpected THP transitions; with shmem and file
1024 * protected by mmap_lock-less collapse skipping areas with anon_vma
1025 * (whereas vma_needs_copy() skips areas without anon_vma). A rework
1026 * can remove such assumptions later, but this is good enough for now.
1027 */
1028 dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl);
1029 if (!dst_pte) {
1030 ret = -ENOMEM;
1031 goto out;
1032 }
1033 src_pte = pte_offset_map_nolock(src_mm, src_pmd, addr, &src_ptl);
1034 if (!src_pte) {
1035 pte_unmap_unlock(dst_pte, dst_ptl);
1036 /* ret == 0 */
1037 goto out;
1038 }
1039 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
1040 orig_src_pte = src_pte;
1041 orig_dst_pte = dst_pte;
1042 arch_enter_lazy_mmu_mode();
1043
1044 do {
1045 /*
1046 * We are holding two locks at this point - either of them
1047 * could generate latencies in another task on another CPU.
1048 */
1049 if (progress >= 32) {
1050 progress = 0;
1051 if (need_resched() ||
1052 spin_needbreak(src_ptl) || spin_needbreak(dst_ptl))
1053 break;
1054 }
1055 ptent = ptep_get(src_pte);
1056 if (pte_none(ptent)) {
1057 progress++;
1058 continue;
1059 }
1060 if (unlikely(!pte_present(ptent))) {
1061 ret = copy_nonpresent_pte(dst_mm, src_mm,
1062 dst_pte, src_pte,
1063 dst_vma, src_vma,
1064 addr, rss);
1065 if (ret == -EIO) {
1066 entry = pte_to_swp_entry(ptep_get(src_pte));
1067 break;
1068 } else if (ret == -EBUSY) {
1069 break;
1070 } else if (!ret) {
1071 progress += 8;
1072 continue;
1073 }
1074
1075 /*
1076 * Device exclusive entry restored, continue by copying
1077 * the now present pte.
1078 */
1079 WARN_ON_ONCE(ret != -ENOENT);
1080 }
1081 /* copy_present_pte() will clear `*prealloc' if consumed */
1082 ret = copy_present_pte(dst_vma, src_vma, dst_pte, src_pte,
1083 addr, rss, &prealloc);
1084 /*
1085 * If we need a pre-allocated page for this pte, drop the
1086 * locks, allocate, and try again.
1087 */
1088 if (unlikely(ret == -EAGAIN))
1089 break;
1090 if (unlikely(prealloc)) {
1091 /*
1092 * pre-alloc page cannot be reused by next time so as
1093 * to strictly follow mempolicy (e.g., alloc_page_vma()
1094 * will allocate page according to address). This
1095 * could only happen if one pinned pte changed.
1096 */
1097 folio_put(prealloc);
1098 prealloc = NULL;
1099 }
1100 progress += 8;
1101 } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end);
1102
1103 arch_leave_lazy_mmu_mode();
1104 pte_unmap_unlock(orig_src_pte, src_ptl);
1105 add_mm_rss_vec(dst_mm, rss);
1106 pte_unmap_unlock(orig_dst_pte, dst_ptl);
1107 cond_resched();
1108
1109 if (ret == -EIO) {
1110 VM_WARN_ON_ONCE(!entry.val);
1111 if (add_swap_count_continuation(entry, GFP_KERNEL) < 0) {
1112 ret = -ENOMEM;
1113 goto out;
1114 }
1115 entry.val = 0;
1116 } else if (ret == -EBUSY) {
1117 goto out;
1118 } else if (ret == -EAGAIN) {
1119 prealloc = page_copy_prealloc(src_mm, src_vma, addr);
1120 if (!prealloc)
1121 return -ENOMEM;
1122 } else if (ret) {
1123 VM_WARN_ON_ONCE(1);
1124 }
1125
1126 /* We've captured and resolved the error. Reset, try again. */
1127 ret = 0;
1128
1129 if (addr != end)
1130 goto again;
1131 out:
1132 if (unlikely(prealloc))
1133 folio_put(prealloc);
1134 return ret;
1135 }
1136
1137 static inline int
copy_pmd_range(struct vm_area_struct * dst_vma,struct vm_area_struct * src_vma,pud_t * dst_pud,pud_t * src_pud,unsigned long addr,unsigned long end)1138 copy_pmd_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1139 pud_t *dst_pud, pud_t *src_pud, unsigned long addr,
1140 unsigned long end)
1141 {
1142 struct mm_struct *dst_mm = dst_vma->vm_mm;
1143 struct mm_struct *src_mm = src_vma->vm_mm;
1144 pmd_t *src_pmd, *dst_pmd;
1145 unsigned long next;
1146
1147 dst_pmd = pmd_alloc(dst_mm, dst_pud, addr);
1148 if (!dst_pmd)
1149 return -ENOMEM;
1150 src_pmd = pmd_offset(src_pud, addr);
1151 do {
1152 next = pmd_addr_end(addr, end);
1153 if (is_swap_pmd(*src_pmd) || pmd_trans_huge(*src_pmd)
1154 || pmd_devmap(*src_pmd)) {
1155 int err;
1156 VM_BUG_ON_VMA(next-addr != HPAGE_PMD_SIZE, src_vma);
1157 err = copy_huge_pmd(dst_mm, src_mm, dst_pmd, src_pmd,
1158 addr, dst_vma, src_vma);
1159 if (err == -ENOMEM)
1160 return -ENOMEM;
1161 if (!err)
1162 continue;
1163 /* fall through */
1164 }
1165 if (pmd_none_or_clear_bad(src_pmd))
1166 continue;
1167 if (copy_pte_range(dst_vma, src_vma, dst_pmd, src_pmd,
1168 addr, next))
1169 return -ENOMEM;
1170 } while (dst_pmd++, src_pmd++, addr = next, addr != end);
1171 return 0;
1172 }
1173
1174 static inline int
copy_pud_range(struct vm_area_struct * dst_vma,struct vm_area_struct * src_vma,p4d_t * dst_p4d,p4d_t * src_p4d,unsigned long addr,unsigned long end)1175 copy_pud_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1176 p4d_t *dst_p4d, p4d_t *src_p4d, unsigned long addr,
1177 unsigned long end)
1178 {
1179 struct mm_struct *dst_mm = dst_vma->vm_mm;
1180 struct mm_struct *src_mm = src_vma->vm_mm;
1181 pud_t *src_pud, *dst_pud;
1182 unsigned long next;
1183
1184 dst_pud = pud_alloc(dst_mm, dst_p4d, addr);
1185 if (!dst_pud)
1186 return -ENOMEM;
1187 src_pud = pud_offset(src_p4d, addr);
1188 do {
1189 next = pud_addr_end(addr, end);
1190 if (pud_trans_huge(*src_pud) || pud_devmap(*src_pud)) {
1191 int err;
1192
1193 VM_BUG_ON_VMA(next-addr != HPAGE_PUD_SIZE, src_vma);
1194 err = copy_huge_pud(dst_mm, src_mm,
1195 dst_pud, src_pud, addr, src_vma);
1196 if (err == -ENOMEM)
1197 return -ENOMEM;
1198 if (!err)
1199 continue;
1200 /* fall through */
1201 }
1202 if (pud_none_or_clear_bad(src_pud))
1203 continue;
1204 if (copy_pmd_range(dst_vma, src_vma, dst_pud, src_pud,
1205 addr, next))
1206 return -ENOMEM;
1207 } while (dst_pud++, src_pud++, addr = next, addr != end);
1208 return 0;
1209 }
1210
1211 static inline int
copy_p4d_range(struct vm_area_struct * dst_vma,struct vm_area_struct * src_vma,pgd_t * dst_pgd,pgd_t * src_pgd,unsigned long addr,unsigned long end)1212 copy_p4d_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1213 pgd_t *dst_pgd, pgd_t *src_pgd, unsigned long addr,
1214 unsigned long end)
1215 {
1216 struct mm_struct *dst_mm = dst_vma->vm_mm;
1217 p4d_t *src_p4d, *dst_p4d;
1218 unsigned long next;
1219
1220 dst_p4d = p4d_alloc(dst_mm, dst_pgd, addr);
1221 if (!dst_p4d)
1222 return -ENOMEM;
1223 src_p4d = p4d_offset(src_pgd, addr);
1224 do {
1225 next = p4d_addr_end(addr, end);
1226 if (p4d_none_or_clear_bad(src_p4d))
1227 continue;
1228 if (copy_pud_range(dst_vma, src_vma, dst_p4d, src_p4d,
1229 addr, next))
1230 return -ENOMEM;
1231 } while (dst_p4d++, src_p4d++, addr = next, addr != end);
1232 return 0;
1233 }
1234
1235 /*
1236 * Return true if the vma needs to copy the pgtable during this fork(). Return
1237 * false when we can speed up fork() by allowing lazy page faults later until
1238 * when the child accesses the memory range.
1239 */
1240 static bool
vma_needs_copy(struct vm_area_struct * dst_vma,struct vm_area_struct * src_vma)1241 vma_needs_copy(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
1242 {
1243 /*
1244 * Always copy pgtables when dst_vma has uffd-wp enabled even if it's
1245 * file-backed (e.g. shmem). Because when uffd-wp is enabled, pgtable
1246 * contains uffd-wp protection information, that's something we can't
1247 * retrieve from page cache, and skip copying will lose those info.
1248 */
1249 if (userfaultfd_wp(dst_vma))
1250 return true;
1251
1252 if (src_vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
1253 return true;
1254
1255 if (src_vma->anon_vma)
1256 return true;
1257
1258 /*
1259 * Don't copy ptes where a page fault will fill them correctly. Fork
1260 * becomes much lighter when there are big shared or private readonly
1261 * mappings. The tradeoff is that copy_page_range is more efficient
1262 * than faulting.
1263 */
1264 return false;
1265 }
1266
1267 int
copy_page_range(struct vm_area_struct * dst_vma,struct vm_area_struct * src_vma)1268 copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
1269 {
1270 pgd_t *src_pgd, *dst_pgd;
1271 unsigned long next;
1272 unsigned long addr = src_vma->vm_start;
1273 unsigned long end = src_vma->vm_end;
1274 struct mm_struct *dst_mm = dst_vma->vm_mm;
1275 struct mm_struct *src_mm = src_vma->vm_mm;
1276 struct mmu_notifier_range range;
1277 bool is_cow;
1278 int ret;
1279
1280 if (!vma_needs_copy(dst_vma, src_vma))
1281 return 0;
1282
1283 if (is_vm_hugetlb_page(src_vma))
1284 return copy_hugetlb_page_range(dst_mm, src_mm, dst_vma, src_vma);
1285
1286 if (unlikely(src_vma->vm_flags & VM_PFNMAP)) {
1287 /*
1288 * We do not free on error cases below as remove_vma
1289 * gets called on error from higher level routine
1290 */
1291 ret = track_pfn_copy(src_vma);
1292 if (ret)
1293 return ret;
1294 }
1295
1296 /*
1297 * We need to invalidate the secondary MMU mappings only when
1298 * there could be a permission downgrade on the ptes of the
1299 * parent mm. And a permission downgrade will only happen if
1300 * is_cow_mapping() returns true.
1301 */
1302 is_cow = is_cow_mapping(src_vma->vm_flags);
1303
1304 if (is_cow) {
1305 mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE,
1306 0, src_mm, addr, end);
1307 mmu_notifier_invalidate_range_start(&range);
1308 /*
1309 * Disabling preemption is not needed for the write side, as
1310 * the read side doesn't spin, but goes to the mmap_lock.
1311 *
1312 * Use the raw variant of the seqcount_t write API to avoid
1313 * lockdep complaining about preemptibility.
1314 */
1315 vma_assert_write_locked(src_vma);
1316 raw_write_seqcount_begin(&src_mm->write_protect_seq);
1317 }
1318
1319 ret = 0;
1320 dst_pgd = pgd_offset(dst_mm, addr);
1321 src_pgd = pgd_offset(src_mm, addr);
1322 do {
1323 next = pgd_addr_end(addr, end);
1324 if (pgd_none_or_clear_bad(src_pgd))
1325 continue;
1326 if (unlikely(copy_p4d_range(dst_vma, src_vma, dst_pgd, src_pgd,
1327 addr, next))) {
1328 untrack_pfn_clear(dst_vma);
1329 ret = -ENOMEM;
1330 break;
1331 }
1332 } while (dst_pgd++, src_pgd++, addr = next, addr != end);
1333
1334 if (is_cow) {
1335 raw_write_seqcount_end(&src_mm->write_protect_seq);
1336 mmu_notifier_invalidate_range_end(&range);
1337 }
1338 return ret;
1339 }
1340
1341 /* Whether we should zap all COWed (private) pages too */
should_zap_cows(struct zap_details * details)1342 static inline bool should_zap_cows(struct zap_details *details)
1343 {
1344 /* By default, zap all pages */
1345 if (!details)
1346 return true;
1347
1348 /* Or, we zap COWed pages only if the caller wants to */
1349 return details->even_cows;
1350 }
1351
1352 /* Decides whether we should zap this page with the page pointer specified */
should_zap_page(struct zap_details * details,struct page * page)1353 static inline bool should_zap_page(struct zap_details *details, struct page *page)
1354 {
1355 /* If we can make a decision without *page.. */
1356 if (should_zap_cows(details))
1357 return true;
1358
1359 /* E.g. the caller passes NULL for the case of a zero page */
1360 if (!page)
1361 return true;
1362
1363 /* Otherwise we should only zap non-anon pages */
1364 return !PageAnon(page);
1365 }
1366
zap_drop_file_uffd_wp(struct zap_details * details)1367 static inline bool zap_drop_file_uffd_wp(struct zap_details *details)
1368 {
1369 if (!details)
1370 return false;
1371
1372 return details->zap_flags & ZAP_FLAG_DROP_MARKER;
1373 }
1374
1375 /*
1376 * This function makes sure that we'll replace the none pte with an uffd-wp
1377 * swap special pte marker when necessary. Must be with the pgtable lock held.
1378 */
1379 static inline void
zap_install_uffd_wp_if_needed(struct vm_area_struct * vma,unsigned long addr,pte_t * pte,struct zap_details * details,pte_t pteval)1380 zap_install_uffd_wp_if_needed(struct vm_area_struct *vma,
1381 unsigned long addr, pte_t *pte,
1382 struct zap_details *details, pte_t pteval)
1383 {
1384 /* Zap on anonymous always means dropping everything */
1385 if (vma_is_anonymous(vma))
1386 return;
1387
1388 if (zap_drop_file_uffd_wp(details))
1389 return;
1390
1391 pte_install_uffd_wp_if_needed(vma, addr, pte, pteval);
1392 }
1393
zap_pte_range(struct mmu_gather * tlb,struct vm_area_struct * vma,pmd_t * pmd,unsigned long addr,unsigned long end,struct zap_details * details)1394 static unsigned long zap_pte_range(struct mmu_gather *tlb,
1395 struct vm_area_struct *vma, pmd_t *pmd,
1396 unsigned long addr, unsigned long end,
1397 struct zap_details *details)
1398 {
1399 struct mm_struct *mm = tlb->mm;
1400 int force_flush = 0;
1401 int rss[NR_MM_COUNTERS];
1402 spinlock_t *ptl;
1403 pte_t *start_pte;
1404 pte_t *pte;
1405 swp_entry_t entry;
1406
1407 tlb_change_page_size(tlb, PAGE_SIZE);
1408 init_rss_vec(rss);
1409 start_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
1410 if (!pte)
1411 return addr;
1412
1413 flush_tlb_batched_pending(mm);
1414 arch_enter_lazy_mmu_mode();
1415 do {
1416 pte_t ptent = ptep_get(pte);
1417 struct page *page;
1418
1419 if (pte_none(ptent))
1420 continue;
1421
1422 if (need_resched())
1423 break;
1424
1425 if (pte_present(ptent)) {
1426 unsigned int delay_rmap;
1427
1428 page = vm_normal_page(vma, addr, ptent);
1429 if (unlikely(!should_zap_page(details, page)))
1430 continue;
1431 ptent = ptep_get_and_clear_full(mm, addr, pte,
1432 tlb->fullmm);
1433 arch_check_zapped_pte(vma, ptent);
1434 tlb_remove_tlb_entry(tlb, pte, addr);
1435 zap_install_uffd_wp_if_needed(vma, addr, pte, details,
1436 ptent);
1437 if (unlikely(!page)) {
1438 ksm_might_unmap_zero_page(mm, ptent);
1439 continue;
1440 }
1441
1442 delay_rmap = 0;
1443 if (!PageAnon(page)) {
1444 if (pte_dirty(ptent)) {
1445 set_page_dirty(page);
1446 if (tlb_delay_rmap(tlb)) {
1447 delay_rmap = 1;
1448 force_flush = 1;
1449 }
1450 }
1451 if (pte_young(ptent) && likely(vma_has_recency(vma)))
1452 mark_page_accessed(page);
1453 }
1454 rss[mm_counter(page)]--;
1455 if (!delay_rmap) {
1456 page_remove_rmap(page, vma, false);
1457 if (unlikely(page_mapcount(page) < 0))
1458 print_bad_pte(vma, addr, ptent, page);
1459 }
1460 if (unlikely(__tlb_remove_page(tlb, page, delay_rmap))) {
1461 force_flush = 1;
1462 addr += PAGE_SIZE;
1463 break;
1464 }
1465 continue;
1466 }
1467
1468 entry = pte_to_swp_entry(ptent);
1469 if (is_device_private_entry(entry) ||
1470 is_device_exclusive_entry(entry)) {
1471 page = pfn_swap_entry_to_page(entry);
1472 if (unlikely(!should_zap_page(details, page)))
1473 continue;
1474 /*
1475 * Both device private/exclusive mappings should only
1476 * work with anonymous page so far, so we don't need to
1477 * consider uffd-wp bit when zap. For more information,
1478 * see zap_install_uffd_wp_if_needed().
1479 */
1480 WARN_ON_ONCE(!vma_is_anonymous(vma));
1481 rss[mm_counter(page)]--;
1482 if (is_device_private_entry(entry))
1483 page_remove_rmap(page, vma, false);
1484 put_page(page);
1485 } else if (!non_swap_entry(entry)) {
1486 /* Genuine swap entry, hence a private anon page */
1487 if (!should_zap_cows(details))
1488 continue;
1489 rss[MM_SWAPENTS]--;
1490 if (unlikely(!free_swap_and_cache(entry)))
1491 print_bad_pte(vma, addr, ptent, NULL);
1492 } else if (is_migration_entry(entry)) {
1493 page = pfn_swap_entry_to_page(entry);
1494 if (!should_zap_page(details, page))
1495 continue;
1496 rss[mm_counter(page)]--;
1497 } else if (pte_marker_entry_uffd_wp(entry)) {
1498 /*
1499 * For anon: always drop the marker; for file: only
1500 * drop the marker if explicitly requested.
1501 */
1502 if (!vma_is_anonymous(vma) &&
1503 !zap_drop_file_uffd_wp(details))
1504 continue;
1505 } else if (is_hwpoison_entry(entry) ||
1506 is_poisoned_swp_entry(entry)) {
1507 if (!should_zap_cows(details))
1508 continue;
1509 } else {
1510 /* We should have covered all the swap entry types */
1511 WARN_ON_ONCE(1);
1512 }
1513 pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
1514 zap_install_uffd_wp_if_needed(vma, addr, pte, details, ptent);
1515 } while (pte++, addr += PAGE_SIZE, addr != end);
1516
1517 add_mm_rss_vec(mm, rss);
1518 arch_leave_lazy_mmu_mode();
1519
1520 /* Do the actual TLB flush before dropping ptl */
1521 if (force_flush) {
1522 tlb_flush_mmu_tlbonly(tlb);
1523 tlb_flush_rmaps(tlb, vma);
1524 }
1525 pte_unmap_unlock(start_pte, ptl);
1526
1527 /*
1528 * If we forced a TLB flush (either due to running out of
1529 * batch buffers or because we needed to flush dirty TLB
1530 * entries before releasing the ptl), free the batched
1531 * memory too. Come back again if we didn't do everything.
1532 */
1533 if (force_flush)
1534 tlb_flush_mmu(tlb);
1535
1536 return addr;
1537 }
1538
zap_pmd_range(struct mmu_gather * tlb,struct vm_area_struct * vma,pud_t * pud,unsigned long addr,unsigned long end,struct zap_details * details)1539 static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,
1540 struct vm_area_struct *vma, pud_t *pud,
1541 unsigned long addr, unsigned long end,
1542 struct zap_details *details)
1543 {
1544 pmd_t *pmd;
1545 unsigned long next;
1546
1547 pmd = pmd_offset(pud, addr);
1548 do {
1549 next = pmd_addr_end(addr, end);
1550 if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) {
1551 if (next - addr != HPAGE_PMD_SIZE)
1552 __split_huge_pmd(vma, pmd, addr, false, NULL);
1553 else if (zap_huge_pmd(tlb, vma, pmd, addr)) {
1554 addr = next;
1555 continue;
1556 }
1557 /* fall through */
1558 } else if (details && details->single_folio &&
1559 folio_test_pmd_mappable(details->single_folio) &&
1560 next - addr == HPAGE_PMD_SIZE && pmd_none(*pmd)) {
1561 spinlock_t *ptl = pmd_lock(tlb->mm, pmd);
1562 /*
1563 * Take and drop THP pmd lock so that we cannot return
1564 * prematurely, while zap_huge_pmd() has cleared *pmd,
1565 * but not yet decremented compound_mapcount().
1566 */
1567 spin_unlock(ptl);
1568 }
1569 if (pmd_none(*pmd)) {
1570 addr = next;
1571 continue;
1572 }
1573 addr = zap_pte_range(tlb, vma, pmd, addr, next, details);
1574 if (addr != next)
1575 pmd--;
1576 } while (pmd++, cond_resched(), addr != end);
1577
1578 return addr;
1579 }
1580
zap_pud_range(struct mmu_gather * tlb,struct vm_area_struct * vma,p4d_t * p4d,unsigned long addr,unsigned long end,struct zap_details * details)1581 static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
1582 struct vm_area_struct *vma, p4d_t *p4d,
1583 unsigned long addr, unsigned long end,
1584 struct zap_details *details)
1585 {
1586 pud_t *pud;
1587 unsigned long next;
1588
1589 pud = pud_offset(p4d, addr);
1590 do {
1591 next = pud_addr_end(addr, end);
1592 if (pud_trans_huge(*pud) || pud_devmap(*pud)) {
1593 if (next - addr != HPAGE_PUD_SIZE) {
1594 mmap_assert_locked(tlb->mm);
1595 split_huge_pud(vma, pud, addr);
1596 } else if (zap_huge_pud(tlb, vma, pud, addr))
1597 goto next;
1598 /* fall through */
1599 }
1600 if (pud_none_or_clear_bad(pud))
1601 continue;
1602 next = zap_pmd_range(tlb, vma, pud, addr, next, details);
1603 next:
1604 cond_resched();
1605 } while (pud++, addr = next, addr != end);
1606
1607 return addr;
1608 }
1609
zap_p4d_range(struct mmu_gather * tlb,struct vm_area_struct * vma,pgd_t * pgd,unsigned long addr,unsigned long end,struct zap_details * details)1610 static inline unsigned long zap_p4d_range(struct mmu_gather *tlb,
1611 struct vm_area_struct *vma, pgd_t *pgd,
1612 unsigned long addr, unsigned long end,
1613 struct zap_details *details)
1614 {
1615 p4d_t *p4d;
1616 unsigned long next;
1617
1618 p4d = p4d_offset(pgd, addr);
1619 do {
1620 next = p4d_addr_end(addr, end);
1621 if (p4d_none_or_clear_bad(p4d))
1622 continue;
1623 next = zap_pud_range(tlb, vma, p4d, addr, next, details);
1624 } while (p4d++, addr = next, addr != end);
1625
1626 return addr;
1627 }
1628
unmap_page_range(struct mmu_gather * tlb,struct vm_area_struct * vma,unsigned long addr,unsigned long end,struct zap_details * details)1629 void unmap_page_range(struct mmu_gather *tlb,
1630 struct vm_area_struct *vma,
1631 unsigned long addr, unsigned long end,
1632 struct zap_details *details)
1633 {
1634 pgd_t *pgd;
1635 unsigned long next;
1636
1637 BUG_ON(addr >= end);
1638 tlb_start_vma(tlb, vma);
1639 pgd = pgd_offset(vma->vm_mm, addr);
1640 do {
1641 next = pgd_addr_end(addr, end);
1642 if (pgd_none_or_clear_bad(pgd))
1643 continue;
1644 next = zap_p4d_range(tlb, vma, pgd, addr, next, details);
1645 } while (pgd++, addr = next, addr != end);
1646 tlb_end_vma(tlb, vma);
1647 }
1648
1649
unmap_single_vma(struct mmu_gather * tlb,struct vm_area_struct * vma,unsigned long start_addr,unsigned long end_addr,struct zap_details * details,bool mm_wr_locked)1650 static void unmap_single_vma(struct mmu_gather *tlb,
1651 struct vm_area_struct *vma, unsigned long start_addr,
1652 unsigned long end_addr,
1653 struct zap_details *details, bool mm_wr_locked)
1654 {
1655 unsigned long start = max(vma->vm_start, start_addr);
1656 unsigned long end;
1657
1658 if (start >= vma->vm_end)
1659 return;
1660 end = min(vma->vm_end, end_addr);
1661 if (end <= vma->vm_start)
1662 return;
1663
1664 if (vma->vm_file)
1665 uprobe_munmap(vma, start, end);
1666
1667 if (unlikely(vma->vm_flags & VM_PFNMAP))
1668 untrack_pfn(vma, 0, 0, mm_wr_locked);
1669
1670 if (start != end) {
1671 if (unlikely(is_vm_hugetlb_page(vma))) {
1672 /*
1673 * It is undesirable to test vma->vm_file as it
1674 * should be non-null for valid hugetlb area.
1675 * However, vm_file will be NULL in the error
1676 * cleanup path of mmap_region. When
1677 * hugetlbfs ->mmap method fails,
1678 * mmap_region() nullifies vma->vm_file
1679 * before calling this function to clean up.
1680 * Since no pte has actually been setup, it is
1681 * safe to do nothing in this case.
1682 */
1683 if (vma->vm_file) {
1684 zap_flags_t zap_flags = details ?
1685 details->zap_flags : 0;
1686 __unmap_hugepage_range(tlb, vma, start, end,
1687 NULL, zap_flags);
1688 }
1689 } else
1690 unmap_page_range(tlb, vma, start, end, details);
1691 }
1692 }
1693
1694 /**
1695 * unmap_vmas - unmap a range of memory covered by a list of vma's
1696 * @tlb: address of the caller's struct mmu_gather
1697 * @mas: the maple state
1698 * @vma: the starting vma
1699 * @start_addr: virtual address at which to start unmapping
1700 * @end_addr: virtual address at which to end unmapping
1701 * @tree_end: The maximum index to check
1702 * @mm_wr_locked: lock flag
1703 *
1704 * Unmap all pages in the vma list.
1705 *
1706 * Only addresses between `start' and `end' will be unmapped.
1707 *
1708 * The VMA list must be sorted in ascending virtual address order.
1709 *
1710 * unmap_vmas() assumes that the caller will flush the whole unmapped address
1711 * range after unmap_vmas() returns. So the only responsibility here is to
1712 * ensure that any thus-far unmapped pages are flushed before unmap_vmas()
1713 * drops the lock and schedules.
1714 */
unmap_vmas(struct mmu_gather * tlb,struct ma_state * mas,struct vm_area_struct * vma,unsigned long start_addr,unsigned long end_addr,unsigned long tree_end,bool mm_wr_locked)1715 void unmap_vmas(struct mmu_gather *tlb, struct ma_state *mas,
1716 struct vm_area_struct *vma, unsigned long start_addr,
1717 unsigned long end_addr, unsigned long tree_end,
1718 bool mm_wr_locked)
1719 {
1720 struct mmu_notifier_range range;
1721 struct zap_details details = {
1722 .zap_flags = ZAP_FLAG_DROP_MARKER | ZAP_FLAG_UNMAP,
1723 /* Careful - we need to zap private pages too! */
1724 .even_cows = true,
1725 };
1726
1727 mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma->vm_mm,
1728 start_addr, end_addr);
1729 mmu_notifier_invalidate_range_start(&range);
1730 do {
1731 unsigned long start = start_addr;
1732 unsigned long end = end_addr;
1733 hugetlb_zap_begin(vma, &start, &end);
1734 unmap_single_vma(tlb, vma, start, end, &details,
1735 mm_wr_locked);
1736 hugetlb_zap_end(vma, &details);
1737 } while ((vma = mas_find(mas, tree_end - 1)) != NULL);
1738 mmu_notifier_invalidate_range_end(&range);
1739 }
1740
1741 /**
1742 * zap_page_range_single - remove user pages in a given range
1743 * @vma: vm_area_struct holding the applicable pages
1744 * @address: starting address of pages to zap
1745 * @size: number of bytes to zap
1746 * @details: details of shared cache invalidation
1747 *
1748 * The range must fit into one VMA.
1749 */
zap_page_range_single(struct vm_area_struct * vma,unsigned long address,unsigned long size,struct zap_details * details)1750 void zap_page_range_single(struct vm_area_struct *vma, unsigned long address,
1751 unsigned long size, struct zap_details *details)
1752 {
1753 const unsigned long end = address + size;
1754 struct mmu_notifier_range range;
1755 struct mmu_gather tlb;
1756
1757 lru_add_drain();
1758 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
1759 address, end);
1760 hugetlb_zap_begin(vma, &range.start, &range.end);
1761 tlb_gather_mmu(&tlb, vma->vm_mm);
1762 update_hiwater_rss(vma->vm_mm);
1763 mmu_notifier_invalidate_range_start(&range);
1764 /*
1765 * unmap 'address-end' not 'range.start-range.end' as range
1766 * could have been expanded for hugetlb pmd sharing.
1767 */
1768 unmap_single_vma(&tlb, vma, address, end, details, false);
1769 mmu_notifier_invalidate_range_end(&range);
1770 tlb_finish_mmu(&tlb);
1771 hugetlb_zap_end(vma, details);
1772 }
1773
1774 /**
1775 * zap_vma_ptes - remove ptes mapping the vma
1776 * @vma: vm_area_struct holding ptes to be zapped
1777 * @address: starting address of pages to zap
1778 * @size: number of bytes to zap
1779 *
1780 * This function only unmaps ptes assigned to VM_PFNMAP vmas.
1781 *
1782 * The entire address range must be fully contained within the vma.
1783 *
1784 */
zap_vma_ptes(struct vm_area_struct * vma,unsigned long address,unsigned long size)1785 void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
1786 unsigned long size)
1787 {
1788 if (!range_in_vma(vma, address, address + size) ||
1789 !(vma->vm_flags & VM_PFNMAP))
1790 return;
1791
1792 zap_page_range_single(vma, address, size, NULL);
1793 }
1794 EXPORT_SYMBOL_GPL(zap_vma_ptes);
1795
walk_to_pmd(struct mm_struct * mm,unsigned long addr)1796 static pmd_t *walk_to_pmd(struct mm_struct *mm, unsigned long addr)
1797 {
1798 pgd_t *pgd;
1799 p4d_t *p4d;
1800 pud_t *pud;
1801 pmd_t *pmd;
1802
1803 pgd = pgd_offset(mm, addr);
1804 p4d = p4d_alloc(mm, pgd, addr);
1805 if (!p4d)
1806 return NULL;
1807 pud = pud_alloc(mm, p4d, addr);
1808 if (!pud)
1809 return NULL;
1810 pmd = pmd_alloc(mm, pud, addr);
1811 if (!pmd)
1812 return NULL;
1813
1814 VM_BUG_ON(pmd_trans_huge(*pmd));
1815 return pmd;
1816 }
1817
__get_locked_pte(struct mm_struct * mm,unsigned long addr,spinlock_t ** ptl)1818 pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
1819 spinlock_t **ptl)
1820 {
1821 pmd_t *pmd = walk_to_pmd(mm, addr);
1822
1823 if (!pmd)
1824 return NULL;
1825 return pte_alloc_map_lock(mm, pmd, addr, ptl);
1826 }
1827
validate_page_before_insert(struct page * page)1828 static int validate_page_before_insert(struct page *page)
1829 {
1830 if (PageAnon(page) || PageSlab(page) || page_has_type(page))
1831 return -EINVAL;
1832 flush_dcache_page(page);
1833 return 0;
1834 }
1835
insert_page_into_pte_locked(struct vm_area_struct * vma,pte_t * pte,unsigned long addr,struct page * page,pgprot_t prot)1836 static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte,
1837 unsigned long addr, struct page *page, pgprot_t prot)
1838 {
1839 if (!pte_none(ptep_get(pte)))
1840 return -EBUSY;
1841 /* Ok, finally just insert the thing.. */
1842 get_page(page);
1843 inc_mm_counter(vma->vm_mm, mm_counter_file(page));
1844 page_add_file_rmap(page, vma, false);
1845 set_pte_at(vma->vm_mm, addr, pte, mk_pte(page, prot));
1846 return 0;
1847 }
1848
1849 /*
1850 * This is the old fallback for page remapping.
1851 *
1852 * For historical reasons, it only allows reserved pages. Only
1853 * old drivers should use this, and they needed to mark their
1854 * pages reserved for the old functions anyway.
1855 */
insert_page(struct vm_area_struct * vma,unsigned long addr,struct page * page,pgprot_t prot)1856 static int insert_page(struct vm_area_struct *vma, unsigned long addr,
1857 struct page *page, pgprot_t prot)
1858 {
1859 int retval;
1860 pte_t *pte;
1861 spinlock_t *ptl;
1862
1863 retval = validate_page_before_insert(page);
1864 if (retval)
1865 goto out;
1866 retval = -ENOMEM;
1867 pte = get_locked_pte(vma->vm_mm, addr, &ptl);
1868 if (!pte)
1869 goto out;
1870 retval = insert_page_into_pte_locked(vma, pte, addr, page, prot);
1871 pte_unmap_unlock(pte, ptl);
1872 out:
1873 return retval;
1874 }
1875
insert_page_in_batch_locked(struct vm_area_struct * vma,pte_t * pte,unsigned long addr,struct page * page,pgprot_t prot)1876 static int insert_page_in_batch_locked(struct vm_area_struct *vma, pte_t *pte,
1877 unsigned long addr, struct page *page, pgprot_t prot)
1878 {
1879 int err;
1880
1881 if (!page_count(page))
1882 return -EINVAL;
1883 err = validate_page_before_insert(page);
1884 if (err)
1885 return err;
1886 return insert_page_into_pte_locked(vma, pte, addr, page, prot);
1887 }
1888
1889 /* insert_pages() amortizes the cost of spinlock operations
1890 * when inserting pages in a loop.
1891 */
insert_pages(struct vm_area_struct * vma,unsigned long addr,struct page ** pages,unsigned long * num,pgprot_t prot)1892 static int insert_pages(struct vm_area_struct *vma, unsigned long addr,
1893 struct page **pages, unsigned long *num, pgprot_t prot)
1894 {
1895 pmd_t *pmd = NULL;
1896 pte_t *start_pte, *pte;
1897 spinlock_t *pte_lock;
1898 struct mm_struct *const mm = vma->vm_mm;
1899 unsigned long curr_page_idx = 0;
1900 unsigned long remaining_pages_total = *num;
1901 unsigned long pages_to_write_in_pmd;
1902 int ret;
1903 more:
1904 ret = -EFAULT;
1905 pmd = walk_to_pmd(mm, addr);
1906 if (!pmd)
1907 goto out;
1908
1909 pages_to_write_in_pmd = min_t(unsigned long,
1910 remaining_pages_total, PTRS_PER_PTE - pte_index(addr));
1911
1912 /* Allocate the PTE if necessary; takes PMD lock once only. */
1913 ret = -ENOMEM;
1914 if (pte_alloc(mm, pmd))
1915 goto out;
1916
1917 while (pages_to_write_in_pmd) {
1918 int pte_idx = 0;
1919 const int batch_size = min_t(int, pages_to_write_in_pmd, 8);
1920
1921 start_pte = pte_offset_map_lock(mm, pmd, addr, &pte_lock);
1922 if (!start_pte) {
1923 ret = -EFAULT;
1924 goto out;
1925 }
1926 for (pte = start_pte; pte_idx < batch_size; ++pte, ++pte_idx) {
1927 int err = insert_page_in_batch_locked(vma, pte,
1928 addr, pages[curr_page_idx], prot);
1929 if (unlikely(err)) {
1930 pte_unmap_unlock(start_pte, pte_lock);
1931 ret = err;
1932 remaining_pages_total -= pte_idx;
1933 goto out;
1934 }
1935 addr += PAGE_SIZE;
1936 ++curr_page_idx;
1937 }
1938 pte_unmap_unlock(start_pte, pte_lock);
1939 pages_to_write_in_pmd -= batch_size;
1940 remaining_pages_total -= batch_size;
1941 }
1942 if (remaining_pages_total)
1943 goto more;
1944 ret = 0;
1945 out:
1946 *num = remaining_pages_total;
1947 return ret;
1948 }
1949
1950 /**
1951 * vm_insert_pages - insert multiple pages into user vma, batching the pmd lock.
1952 * @vma: user vma to map to
1953 * @addr: target start user address of these pages
1954 * @pages: source kernel pages
1955 * @num: in: number of pages to map. out: number of pages that were *not*
1956 * mapped. (0 means all pages were successfully mapped).
1957 *
1958 * Preferred over vm_insert_page() when inserting multiple pages.
1959 *
1960 * In case of error, we may have mapped a subset of the provided
1961 * pages. It is the caller's responsibility to account for this case.
1962 *
1963 * The same restrictions apply as in vm_insert_page().
1964 */
vm_insert_pages(struct vm_area_struct * vma,unsigned long addr,struct page ** pages,unsigned long * num)1965 int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr,
1966 struct page **pages, unsigned long *num)
1967 {
1968 const unsigned long end_addr = addr + (*num * PAGE_SIZE) - 1;
1969
1970 if (addr < vma->vm_start || end_addr >= vma->vm_end)
1971 return -EFAULT;
1972 if (!(vma->vm_flags & VM_MIXEDMAP)) {
1973 BUG_ON(mmap_read_trylock(vma->vm_mm));
1974 BUG_ON(vma->vm_flags & VM_PFNMAP);
1975 vm_flags_set(vma, VM_MIXEDMAP);
1976 }
1977 /* Defer page refcount checking till we're about to map that page. */
1978 return insert_pages(vma, addr, pages, num, vma->vm_page_prot);
1979 }
1980 EXPORT_SYMBOL(vm_insert_pages);
1981
1982 /**
1983 * vm_insert_page - insert single page into user vma
1984 * @vma: user vma to map to
1985 * @addr: target user address of this page
1986 * @page: source kernel page
1987 *
1988 * This allows drivers to insert individual pages they've allocated
1989 * into a user vma.
1990 *
1991 * The page has to be a nice clean _individual_ kernel allocation.
1992 * If you allocate a compound page, you need to have marked it as
1993 * such (__GFP_COMP), or manually just split the page up yourself
1994 * (see split_page()).
1995 *
1996 * NOTE! Traditionally this was done with "remap_pfn_range()" which
1997 * took an arbitrary page protection parameter. This doesn't allow
1998 * that. Your vma protection will have to be set up correctly, which
1999 * means that if you want a shared writable mapping, you'd better
2000 * ask for a shared writable mapping!
2001 *
2002 * The page does not need to be reserved.
2003 *
2004 * Usually this function is called from f_op->mmap() handler
2005 * under mm->mmap_lock write-lock, so it can change vma->vm_flags.
2006 * Caller must set VM_MIXEDMAP on vma if it wants to call this
2007 * function from other places, for example from page-fault handler.
2008 *
2009 * Return: %0 on success, negative error code otherwise.
2010 */
vm_insert_page(struct vm_area_struct * vma,unsigned long addr,struct page * page)2011 int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
2012 struct page *page)
2013 {
2014 if (addr < vma->vm_start || addr >= vma->vm_end)
2015 return -EFAULT;
2016 if (!page_count(page))
2017 return -EINVAL;
2018 if (!(vma->vm_flags & VM_MIXEDMAP)) {
2019 BUG_ON(mmap_read_trylock(vma->vm_mm));
2020 BUG_ON(vma->vm_flags & VM_PFNMAP);
2021 vm_flags_set(vma, VM_MIXEDMAP);
2022 }
2023 return insert_page(vma, addr, page, vma->vm_page_prot);
2024 }
2025 EXPORT_SYMBOL(vm_insert_page);
2026
2027 /*
2028 * __vm_map_pages - maps range of kernel pages into user vma
2029 * @vma: user vma to map to
2030 * @pages: pointer to array of source kernel pages
2031 * @num: number of pages in page array
2032 * @offset: user's requested vm_pgoff
2033 *
2034 * This allows drivers to map range of kernel pages into a user vma.
2035 *
2036 * Return: 0 on success and error code otherwise.
2037 */
__vm_map_pages(struct vm_area_struct * vma,struct page ** pages,unsigned long num,unsigned long offset)2038 static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages,
2039 unsigned long num, unsigned long offset)
2040 {
2041 unsigned long count = vma_pages(vma);
2042 unsigned long uaddr = vma->vm_start;
2043 int ret, i;
2044
2045 /* Fail if the user requested offset is beyond the end of the object */
2046 if (offset >= num)
2047 return -ENXIO;
2048
2049 /* Fail if the user requested size exceeds available object size */
2050 if (count > num - offset)
2051 return -ENXIO;
2052
2053 for (i = 0; i < count; i++) {
2054 ret = vm_insert_page(vma, uaddr, pages[offset + i]);
2055 if (ret < 0)
2056 return ret;
2057 uaddr += PAGE_SIZE;
2058 }
2059
2060 return 0;
2061 }
2062
2063 /**
2064 * vm_map_pages - maps range of kernel pages starts with non zero offset
2065 * @vma: user vma to map to
2066 * @pages: pointer to array of source kernel pages
2067 * @num: number of pages in page array
2068 *
2069 * Maps an object consisting of @num pages, catering for the user's
2070 * requested vm_pgoff
2071 *
2072 * If we fail to insert any page into the vma, the function will return
2073 * immediately leaving any previously inserted pages present. Callers
2074 * from the mmap handler may immediately return the error as their caller
2075 * will destroy the vma, removing any successfully inserted pages. Other
2076 * callers should make their own arrangements for calling unmap_region().
2077 *
2078 * Context: Process context. Called by mmap handlers.
2079 * Return: 0 on success and error code otherwise.
2080 */
vm_map_pages(struct vm_area_struct * vma,struct page ** pages,unsigned long num)2081 int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
2082 unsigned long num)
2083 {
2084 return __vm_map_pages(vma, pages, num, vma->vm_pgoff);
2085 }
2086 EXPORT_SYMBOL(vm_map_pages);
2087
2088 /**
2089 * vm_map_pages_zero - map range of kernel pages starts with zero offset
2090 * @vma: user vma to map to
2091 * @pages: pointer to array of source kernel pages
2092 * @num: number of pages in page array
2093 *
2094 * Similar to vm_map_pages(), except that it explicitly sets the offset
2095 * to 0. This function is intended for the drivers that did not consider
2096 * vm_pgoff.
2097 *
2098 * Context: Process context. Called by mmap handlers.
2099 * Return: 0 on success and error code otherwise.
2100 */
vm_map_pages_zero(struct vm_area_struct * vma,struct page ** pages,unsigned long num)2101 int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
2102 unsigned long num)
2103 {
2104 return __vm_map_pages(vma, pages, num, 0);
2105 }
2106 EXPORT_SYMBOL(vm_map_pages_zero);
2107
insert_pfn(struct vm_area_struct * vma,unsigned long addr,pfn_t pfn,pgprot_t prot,bool mkwrite)2108 static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr,
2109 pfn_t pfn, pgprot_t prot, bool mkwrite)
2110 {
2111 struct mm_struct *mm = vma->vm_mm;
2112 pte_t *pte, entry;
2113 spinlock_t *ptl;
2114
2115 pte = get_locked_pte(mm, addr, &ptl);
2116 if (!pte)
2117 return VM_FAULT_OOM;
2118 entry = ptep_get(pte);
2119 if (!pte_none(entry)) {
2120 if (mkwrite) {
2121 /*
2122 * For read faults on private mappings the PFN passed
2123 * in may not match the PFN we have mapped if the
2124 * mapped PFN is a writeable COW page. In the mkwrite
2125 * case we are creating a writable PTE for a shared
2126 * mapping and we expect the PFNs to match. If they
2127 * don't match, we are likely racing with block
2128 * allocation and mapping invalidation so just skip the
2129 * update.
2130 */
2131 if (pte_pfn(entry) != pfn_t_to_pfn(pfn)) {
2132 WARN_ON_ONCE(!is_zero_pfn(pte_pfn(entry)));
2133 goto out_unlock;
2134 }
2135 entry = pte_mkyoung(entry);
2136 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
2137 if (ptep_set_access_flags(vma, addr, pte, entry, 1))
2138 update_mmu_cache(vma, addr, pte);
2139 }
2140 goto out_unlock;
2141 }
2142
2143 /* Ok, finally just insert the thing.. */
2144 if (pfn_t_devmap(pfn))
2145 entry = pte_mkdevmap(pfn_t_pte(pfn, prot));
2146 else
2147 entry = pte_mkspecial(pfn_t_pte(pfn, prot));
2148
2149 if (mkwrite) {
2150 entry = pte_mkyoung(entry);
2151 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
2152 }
2153
2154 set_pte_at(mm, addr, pte, entry);
2155 update_mmu_cache(vma, addr, pte); /* XXX: why not for insert_page? */
2156
2157 out_unlock:
2158 pte_unmap_unlock(pte, ptl);
2159 return VM_FAULT_NOPAGE;
2160 }
2161
2162 /**
2163 * vmf_insert_pfn_prot - insert single pfn into user vma with specified pgprot
2164 * @vma: user vma to map to
2165 * @addr: target user address of this page
2166 * @pfn: source kernel pfn
2167 * @pgprot: pgprot flags for the inserted page
2168 *
2169 * This is exactly like vmf_insert_pfn(), except that it allows drivers
2170 * to override pgprot on a per-page basis.
2171 *
2172 * This only makes sense for IO mappings, and it makes no sense for
2173 * COW mappings. In general, using multiple vmas is preferable;
2174 * vmf_insert_pfn_prot should only be used if using multiple VMAs is
2175 * impractical.
2176 *
2177 * pgprot typically only differs from @vma->vm_page_prot when drivers set
2178 * caching- and encryption bits different than those of @vma->vm_page_prot,
2179 * because the caching- or encryption mode may not be known at mmap() time.
2180 *
2181 * This is ok as long as @vma->vm_page_prot is not used by the core vm
2182 * to set caching and encryption bits for those vmas (except for COW pages).
2183 * This is ensured by core vm only modifying these page table entries using
2184 * functions that don't touch caching- or encryption bits, using pte_modify()
2185 * if needed. (See for example mprotect()).
2186 *
2187 * Also when new page-table entries are created, this is only done using the
2188 * fault() callback, and never using the value of vma->vm_page_prot,
2189 * except for page-table entries that point to anonymous pages as the result
2190 * of COW.
2191 *
2192 * Context: Process context. May allocate using %GFP_KERNEL.
2193 * Return: vm_fault_t value.
2194 */
vmf_insert_pfn_prot(struct vm_area_struct * vma,unsigned long addr,unsigned long pfn,pgprot_t pgprot)2195 vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
2196 unsigned long pfn, pgprot_t pgprot)
2197 {
2198 /*
2199 * Technically, architectures with pte_special can avoid all these
2200 * restrictions (same for remap_pfn_range). However we would like
2201 * consistency in testing and feature parity among all, so we should
2202 * try to keep these invariants in place for everybody.
2203 */
2204 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
2205 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
2206 (VM_PFNMAP|VM_MIXEDMAP));
2207 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
2208 BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn));
2209
2210 if (addr < vma->vm_start || addr >= vma->vm_end)
2211 return VM_FAULT_SIGBUS;
2212
2213 if (!pfn_modify_allowed(pfn, pgprot))
2214 return VM_FAULT_SIGBUS;
2215
2216 track_pfn_insert(vma, &pgprot, __pfn_to_pfn_t(pfn, PFN_DEV));
2217
2218 return insert_pfn(vma, addr, __pfn_to_pfn_t(pfn, PFN_DEV), pgprot,
2219 false);
2220 }
2221 EXPORT_SYMBOL(vmf_insert_pfn_prot);
2222
2223 /**
2224 * vmf_insert_pfn - insert single pfn into user vma
2225 * @vma: user vma to map to
2226 * @addr: target user address of this page
2227 * @pfn: source kernel pfn
2228 *
2229 * Similar to vm_insert_page, this allows drivers to insert individual pages
2230 * they've allocated into a user vma. Same comments apply.
2231 *
2232 * This function should only be called from a vm_ops->fault handler, and
2233 * in that case the handler should return the result of this function.
2234 *
2235 * vma cannot be a COW mapping.
2236 *
2237 * As this is called only for pages that do not currently exist, we
2238 * do not need to flush old virtual caches or the TLB.
2239 *
2240 * Context: Process context. May allocate using %GFP_KERNEL.
2241 * Return: vm_fault_t value.
2242 */
vmf_insert_pfn(struct vm_area_struct * vma,unsigned long addr,unsigned long pfn)2243 vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
2244 unsigned long pfn)
2245 {
2246 return vmf_insert_pfn_prot(vma, addr, pfn, vma->vm_page_prot);
2247 }
2248 EXPORT_SYMBOL(vmf_insert_pfn);
2249
vm_mixed_ok(struct vm_area_struct * vma,pfn_t pfn)2250 static bool vm_mixed_ok(struct vm_area_struct *vma, pfn_t pfn)
2251 {
2252 /* these checks mirror the abort conditions in vm_normal_page */
2253 if (vma->vm_flags & VM_MIXEDMAP)
2254 return true;
2255 if (pfn_t_devmap(pfn))
2256 return true;
2257 if (pfn_t_special(pfn))
2258 return true;
2259 if (is_zero_pfn(pfn_t_to_pfn(pfn)))
2260 return true;
2261 return false;
2262 }
2263
__vm_insert_mixed(struct vm_area_struct * vma,unsigned long addr,pfn_t pfn,bool mkwrite)2264 static vm_fault_t __vm_insert_mixed(struct vm_area_struct *vma,
2265 unsigned long addr, pfn_t pfn, bool mkwrite)
2266 {
2267 pgprot_t pgprot = vma->vm_page_prot;
2268 int err;
2269
2270 BUG_ON(!vm_mixed_ok(vma, pfn));
2271
2272 if (addr < vma->vm_start || addr >= vma->vm_end)
2273 return VM_FAULT_SIGBUS;
2274
2275 track_pfn_insert(vma, &pgprot, pfn);
2276
2277 if (!pfn_modify_allowed(pfn_t_to_pfn(pfn), pgprot))
2278 return VM_FAULT_SIGBUS;
2279
2280 /*
2281 * If we don't have pte special, then we have to use the pfn_valid()
2282 * based VM_MIXEDMAP scheme (see vm_normal_page), and thus we *must*
2283 * refcount the page if pfn_valid is true (hence insert_page rather
2284 * than insert_pfn). If a zero_pfn were inserted into a VM_MIXEDMAP
2285 * without pte special, it would there be refcounted as a normal page.
2286 */
2287 if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL) &&
2288 !pfn_t_devmap(pfn) && pfn_t_valid(pfn)) {
2289 struct page *page;
2290
2291 /*
2292 * At this point we are committed to insert_page()
2293 * regardless of whether the caller specified flags that
2294 * result in pfn_t_has_page() == false.
2295 */
2296 page = pfn_to_page(pfn_t_to_pfn(pfn));
2297 err = insert_page(vma, addr, page, pgprot);
2298 } else {
2299 return insert_pfn(vma, addr, pfn, pgprot, mkwrite);
2300 }
2301
2302 if (err == -ENOMEM)
2303 return VM_FAULT_OOM;
2304 if (err < 0 && err != -EBUSY)
2305 return VM_FAULT_SIGBUS;
2306
2307 return VM_FAULT_NOPAGE;
2308 }
2309
vmf_insert_mixed(struct vm_area_struct * vma,unsigned long addr,pfn_t pfn)2310 vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
2311 pfn_t pfn)
2312 {
2313 return __vm_insert_mixed(vma, addr, pfn, false);
2314 }
2315 EXPORT_SYMBOL(vmf_insert_mixed);
2316
2317 /*
2318 * If the insertion of PTE failed because someone else already added a
2319 * different entry in the mean time, we treat that as success as we assume
2320 * the same entry was actually inserted.
2321 */
vmf_insert_mixed_mkwrite(struct vm_area_struct * vma,unsigned long addr,pfn_t pfn)2322 vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma,
2323 unsigned long addr, pfn_t pfn)
2324 {
2325 return __vm_insert_mixed(vma, addr, pfn, true);
2326 }
2327 EXPORT_SYMBOL(vmf_insert_mixed_mkwrite);
2328
2329 /*
2330 * maps a range of physical memory into the requested pages. the old
2331 * mappings are removed. any references to nonexistent pages results
2332 * in null mappings (currently treated as "copy-on-access")
2333 */
remap_pte_range(struct mm_struct * mm,pmd_t * pmd,unsigned long addr,unsigned long end,unsigned long pfn,pgprot_t prot)2334 static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd,
2335 unsigned long addr, unsigned long end,
2336 unsigned long pfn, pgprot_t prot)
2337 {
2338 pte_t *pte, *mapped_pte;
2339 spinlock_t *ptl;
2340 int err = 0;
2341
2342 mapped_pte = pte = pte_alloc_map_lock(mm, pmd, addr, &ptl);
2343 if (!pte)
2344 return -ENOMEM;
2345 arch_enter_lazy_mmu_mode();
2346 do {
2347 BUG_ON(!pte_none(ptep_get(pte)));
2348 if (!pfn_modify_allowed(pfn, prot)) {
2349 err = -EACCES;
2350 break;
2351 }
2352 set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot)));
2353 pfn++;
2354 } while (pte++, addr += PAGE_SIZE, addr != end);
2355 arch_leave_lazy_mmu_mode();
2356 pte_unmap_unlock(mapped_pte, ptl);
2357 return err;
2358 }
2359
remap_pmd_range(struct mm_struct * mm,pud_t * pud,unsigned long addr,unsigned long end,unsigned long pfn,pgprot_t prot)2360 static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud,
2361 unsigned long addr, unsigned long end,
2362 unsigned long pfn, pgprot_t prot)
2363 {
2364 pmd_t *pmd;
2365 unsigned long next;
2366 int err;
2367
2368 pfn -= addr >> PAGE_SHIFT;
2369 pmd = pmd_alloc(mm, pud, addr);
2370 if (!pmd)
2371 return -ENOMEM;
2372 VM_BUG_ON(pmd_trans_huge(*pmd));
2373 do {
2374 next = pmd_addr_end(addr, end);
2375 err = remap_pte_range(mm, pmd, addr, next,
2376 pfn + (addr >> PAGE_SHIFT), prot);
2377 if (err)
2378 return err;
2379 } while (pmd++, addr = next, addr != end);
2380 return 0;
2381 }
2382
remap_pud_range(struct mm_struct * mm,p4d_t * p4d,unsigned long addr,unsigned long end,unsigned long pfn,pgprot_t prot)2383 static inline int remap_pud_range(struct mm_struct *mm, p4d_t *p4d,
2384 unsigned long addr, unsigned long end,
2385 unsigned long pfn, pgprot_t prot)
2386 {
2387 pud_t *pud;
2388 unsigned long next;
2389 int err;
2390
2391 pfn -= addr >> PAGE_SHIFT;
2392 pud = pud_alloc(mm, p4d, addr);
2393 if (!pud)
2394 return -ENOMEM;
2395 do {
2396 next = pud_addr_end(addr, end);
2397 err = remap_pmd_range(mm, pud, addr, next,
2398 pfn + (addr >> PAGE_SHIFT), prot);
2399 if (err)
2400 return err;
2401 } while (pud++, addr = next, addr != end);
2402 return 0;
2403 }
2404
remap_p4d_range(struct mm_struct * mm,pgd_t * pgd,unsigned long addr,unsigned long end,unsigned long pfn,pgprot_t prot)2405 static inline int remap_p4d_range(struct mm_struct *mm, pgd_t *pgd,
2406 unsigned long addr, unsigned long end,
2407 unsigned long pfn, pgprot_t prot)
2408 {
2409 p4d_t *p4d;
2410 unsigned long next;
2411 int err;
2412
2413 pfn -= addr >> PAGE_SHIFT;
2414 p4d = p4d_alloc(mm, pgd, addr);
2415 if (!p4d)
2416 return -ENOMEM;
2417 do {
2418 next = p4d_addr_end(addr, end);
2419 err = remap_pud_range(mm, p4d, addr, next,
2420 pfn + (addr >> PAGE_SHIFT), prot);
2421 if (err)
2422 return err;
2423 } while (p4d++, addr = next, addr != end);
2424 return 0;
2425 }
2426
remap_pfn_range_internal(struct vm_area_struct * vma,unsigned long addr,unsigned long pfn,unsigned long size,pgprot_t prot)2427 static int remap_pfn_range_internal(struct vm_area_struct *vma, unsigned long addr,
2428 unsigned long pfn, unsigned long size, pgprot_t prot)
2429 {
2430 pgd_t *pgd;
2431 unsigned long next;
2432 unsigned long end = addr + PAGE_ALIGN(size);
2433 struct mm_struct *mm = vma->vm_mm;
2434 int err;
2435
2436 if (WARN_ON_ONCE(!PAGE_ALIGNED(addr)))
2437 return -EINVAL;
2438
2439 /*
2440 * Physically remapped pages are special. Tell the
2441 * rest of the world about it:
2442 * VM_IO tells people not to look at these pages
2443 * (accesses can have side effects).
2444 * VM_PFNMAP tells the core MM that the base pages are just
2445 * raw PFN mappings, and do not have a "struct page" associated
2446 * with them.
2447 * VM_DONTEXPAND
2448 * Disable vma merging and expanding with mremap().
2449 * VM_DONTDUMP
2450 * Omit vma from core dump, even when VM_IO turned off.
2451 *
2452 * There's a horrible special case to handle copy-on-write
2453 * behaviour that some programs depend on. We mark the "original"
2454 * un-COW'ed pages by matching them up with "vma->vm_pgoff".
2455 * See vm_normal_page() for details.
2456 */
2457 if (is_cow_mapping(vma->vm_flags)) {
2458 if (addr != vma->vm_start || end != vma->vm_end)
2459 return -EINVAL;
2460 vma->vm_pgoff = pfn;
2461 }
2462
2463 vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
2464
2465 BUG_ON(addr >= end);
2466 pfn -= addr >> PAGE_SHIFT;
2467 pgd = pgd_offset(mm, addr);
2468 flush_cache_range(vma, addr, end);
2469 do {
2470 next = pgd_addr_end(addr, end);
2471 err = remap_p4d_range(mm, pgd, addr, next,
2472 pfn + (addr >> PAGE_SHIFT), prot);
2473 if (err)
2474 return err;
2475 } while (pgd++, addr = next, addr != end);
2476
2477 return 0;
2478 }
2479
2480 /*
2481 * Variant of remap_pfn_range that does not call track_pfn_remap. The caller
2482 * must have pre-validated the caching bits of the pgprot_t.
2483 */
remap_pfn_range_notrack(struct vm_area_struct * vma,unsigned long addr,unsigned long pfn,unsigned long size,pgprot_t prot)2484 int remap_pfn_range_notrack(struct vm_area_struct *vma, unsigned long addr,
2485 unsigned long pfn, unsigned long size, pgprot_t prot)
2486 {
2487 int error = remap_pfn_range_internal(vma, addr, pfn, size, prot);
2488
2489 if (!error)
2490 return 0;
2491
2492 /*
2493 * A partial pfn range mapping is dangerous: it does not
2494 * maintain page reference counts, and callers may free
2495 * pages due to the error. So zap it early.
2496 */
2497 zap_page_range_single(vma, addr, size, NULL);
2498 return error;
2499 }
2500
2501 /**
2502 * remap_pfn_range - remap kernel memory to userspace
2503 * @vma: user vma to map to
2504 * @addr: target page aligned user address to start at
2505 * @pfn: page frame number of kernel physical memory address
2506 * @size: size of mapping area
2507 * @prot: page protection flags for this mapping
2508 *
2509 * Note: this is only safe if the mm semaphore is held when called.
2510 *
2511 * Return: %0 on success, negative error code otherwise.
2512 */
remap_pfn_range(struct vm_area_struct * vma,unsigned long addr,unsigned long pfn,unsigned long size,pgprot_t prot)2513 int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
2514 unsigned long pfn, unsigned long size, pgprot_t prot)
2515 {
2516 int err;
2517
2518 err = track_pfn_remap(vma, &prot, pfn, addr, PAGE_ALIGN(size));
2519 if (err)
2520 return -EINVAL;
2521
2522 err = remap_pfn_range_notrack(vma, addr, pfn, size, prot);
2523 if (err)
2524 untrack_pfn(vma, pfn, PAGE_ALIGN(size), true);
2525 return err;
2526 }
2527 EXPORT_SYMBOL(remap_pfn_range);
2528
2529 /**
2530 * vm_iomap_memory - remap memory to userspace
2531 * @vma: user vma to map to
2532 * @start: start of the physical memory to be mapped
2533 * @len: size of area
2534 *
2535 * This is a simplified io_remap_pfn_range() for common driver use. The
2536 * driver just needs to give us the physical memory range to be mapped,
2537 * we'll figure out the rest from the vma information.
2538 *
2539 * NOTE! Some drivers might want to tweak vma->vm_page_prot first to get
2540 * whatever write-combining details or similar.
2541 *
2542 * Return: %0 on success, negative error code otherwise.
2543 */
vm_iomap_memory(struct vm_area_struct * vma,phys_addr_t start,unsigned long len)2544 int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len)
2545 {
2546 unsigned long vm_len, pfn, pages;
2547
2548 /* Check that the physical memory area passed in looks valid */
2549 if (start + len < start)
2550 return -EINVAL;
2551 /*
2552 * You *really* shouldn't map things that aren't page-aligned,
2553 * but we've historically allowed it because IO memory might
2554 * just have smaller alignment.
2555 */
2556 len += start & ~PAGE_MASK;
2557 pfn = start >> PAGE_SHIFT;
2558 pages = (len + ~PAGE_MASK) >> PAGE_SHIFT;
2559 if (pfn + pages < pfn)
2560 return -EINVAL;
2561
2562 /* We start the mapping 'vm_pgoff' pages into the area */
2563 if (vma->vm_pgoff > pages)
2564 return -EINVAL;
2565 pfn += vma->vm_pgoff;
2566 pages -= vma->vm_pgoff;
2567
2568 /* Can we fit all of the mapping? */
2569 vm_len = vma->vm_end - vma->vm_start;
2570 if (vm_len >> PAGE_SHIFT > pages)
2571 return -EINVAL;
2572
2573 /* Ok, let it rip */
2574 return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot);
2575 }
2576 EXPORT_SYMBOL(vm_iomap_memory);
2577
apply_to_pte_range(struct mm_struct * mm,pmd_t * pmd,unsigned long addr,unsigned long end,pte_fn_t fn,void * data,bool create,pgtbl_mod_mask * mask)2578 static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,
2579 unsigned long addr, unsigned long end,
2580 pte_fn_t fn, void *data, bool create,
2581 pgtbl_mod_mask *mask)
2582 {
2583 pte_t *pte, *mapped_pte;
2584 int err = 0;
2585 spinlock_t *ptl;
2586
2587 if (create) {
2588 mapped_pte = pte = (mm == &init_mm) ?
2589 pte_alloc_kernel_track(pmd, addr, mask) :
2590 pte_alloc_map_lock(mm, pmd, addr, &ptl);
2591 if (!pte)
2592 return -ENOMEM;
2593 } else {
2594 mapped_pte = pte = (mm == &init_mm) ?
2595 pte_offset_kernel(pmd, addr) :
2596 pte_offset_map_lock(mm, pmd, addr, &ptl);
2597 if (!pte)
2598 return -EINVAL;
2599 }
2600
2601 arch_enter_lazy_mmu_mode();
2602
2603 if (fn) {
2604 do {
2605 if (create || !pte_none(ptep_get(pte))) {
2606 err = fn(pte++, addr, data);
2607 if (err)
2608 break;
2609 }
2610 } while (addr += PAGE_SIZE, addr != end);
2611 }
2612 *mask |= PGTBL_PTE_MODIFIED;
2613
2614 arch_leave_lazy_mmu_mode();
2615
2616 if (mm != &init_mm)
2617 pte_unmap_unlock(mapped_pte, ptl);
2618 return err;
2619 }
2620
apply_to_pmd_range(struct mm_struct * mm,pud_t * pud,unsigned long addr,unsigned long end,pte_fn_t fn,void * data,bool create,pgtbl_mod_mask * mask)2621 static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud,
2622 unsigned long addr, unsigned long end,
2623 pte_fn_t fn, void *data, bool create,
2624 pgtbl_mod_mask *mask)
2625 {
2626 pmd_t *pmd;
2627 unsigned long next;
2628 int err = 0;
2629
2630 BUG_ON(pud_huge(*pud));
2631
2632 if (create) {
2633 pmd = pmd_alloc_track(mm, pud, addr, mask);
2634 if (!pmd)
2635 return -ENOMEM;
2636 } else {
2637 pmd = pmd_offset(pud, addr);
2638 }
2639 do {
2640 next = pmd_addr_end(addr, end);
2641 if (pmd_none(*pmd) && !create)
2642 continue;
2643 if (WARN_ON_ONCE(pmd_leaf(*pmd)))
2644 return -EINVAL;
2645 if (!pmd_none(*pmd) && WARN_ON_ONCE(pmd_bad(*pmd))) {
2646 if (!create)
2647 continue;
2648 pmd_clear_bad(pmd);
2649 }
2650 err = apply_to_pte_range(mm, pmd, addr, next,
2651 fn, data, create, mask);
2652 if (err)
2653 break;
2654 } while (pmd++, addr = next, addr != end);
2655
2656 return err;
2657 }
2658
apply_to_pud_range(struct mm_struct * mm,p4d_t * p4d,unsigned long addr,unsigned long end,pte_fn_t fn,void * data,bool create,pgtbl_mod_mask * mask)2659 static int apply_to_pud_range(struct mm_struct *mm, p4d_t *p4d,
2660 unsigned long addr, unsigned long end,
2661 pte_fn_t fn, void *data, bool create,
2662 pgtbl_mod_mask *mask)
2663 {
2664 pud_t *pud;
2665 unsigned long next;
2666 int err = 0;
2667
2668 if (create) {
2669 pud = pud_alloc_track(mm, p4d, addr, mask);
2670 if (!pud)
2671 return -ENOMEM;
2672 } else {
2673 pud = pud_offset(p4d, addr);
2674 }
2675 do {
2676 next = pud_addr_end(addr, end);
2677 if (pud_none(*pud) && !create)
2678 continue;
2679 if (WARN_ON_ONCE(pud_leaf(*pud)))
2680 return -EINVAL;
2681 if (!pud_none(*pud) && WARN_ON_ONCE(pud_bad(*pud))) {
2682 if (!create)
2683 continue;
2684 pud_clear_bad(pud);
2685 }
2686 err = apply_to_pmd_range(mm, pud, addr, next,
2687 fn, data, create, mask);
2688 if (err)
2689 break;
2690 } while (pud++, addr = next, addr != end);
2691
2692 return err;
2693 }
2694
apply_to_p4d_range(struct mm_struct * mm,pgd_t * pgd,unsigned long addr,unsigned long end,pte_fn_t fn,void * data,bool create,pgtbl_mod_mask * mask)2695 static int apply_to_p4d_range(struct mm_struct *mm, pgd_t *pgd,
2696 unsigned long addr, unsigned long end,
2697 pte_fn_t fn, void *data, bool create,
2698 pgtbl_mod_mask *mask)
2699 {
2700 p4d_t *p4d;
2701 unsigned long next;
2702 int err = 0;
2703
2704 if (create) {
2705 p4d = p4d_alloc_track(mm, pgd, addr, mask);
2706 if (!p4d)
2707 return -ENOMEM;
2708 } else {
2709 p4d = p4d_offset(pgd, addr);
2710 }
2711 do {
2712 next = p4d_addr_end(addr, end);
2713 if (p4d_none(*p4d) && !create)
2714 continue;
2715 if (WARN_ON_ONCE(p4d_leaf(*p4d)))
2716 return -EINVAL;
2717 if (!p4d_none(*p4d) && WARN_ON_ONCE(p4d_bad(*p4d))) {
2718 if (!create)
2719 continue;
2720 p4d_clear_bad(p4d);
2721 }
2722 err = apply_to_pud_range(mm, p4d, addr, next,
2723 fn, data, create, mask);
2724 if (err)
2725 break;
2726 } while (p4d++, addr = next, addr != end);
2727
2728 return err;
2729 }
2730
__apply_to_page_range(struct mm_struct * mm,unsigned long addr,unsigned long size,pte_fn_t fn,void * data,bool create)2731 static int __apply_to_page_range(struct mm_struct *mm, unsigned long addr,
2732 unsigned long size, pte_fn_t fn,
2733 void *data, bool create)
2734 {
2735 pgd_t *pgd;
2736 unsigned long start = addr, next;
2737 unsigned long end = addr + size;
2738 pgtbl_mod_mask mask = 0;
2739 int err = 0;
2740
2741 if (WARN_ON(addr >= end))
2742 return -EINVAL;
2743
2744 pgd = pgd_offset(mm, addr);
2745 do {
2746 next = pgd_addr_end(addr, end);
2747 if (pgd_none(*pgd) && !create)
2748 continue;
2749 if (WARN_ON_ONCE(pgd_leaf(*pgd))) {
2750 err = -EINVAL;
2751 break;
2752 }
2753 if (!pgd_none(*pgd) && WARN_ON_ONCE(pgd_bad(*pgd))) {
2754 if (!create)
2755 continue;
2756 pgd_clear_bad(pgd);
2757 }
2758 err = apply_to_p4d_range(mm, pgd, addr, next,
2759 fn, data, create, &mask);
2760 if (err)
2761 break;
2762 } while (pgd++, addr = next, addr != end);
2763
2764 if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
2765 arch_sync_kernel_mappings(start, start + size);
2766
2767 return err;
2768 }
2769
2770 /*
2771 * Scan a region of virtual memory, filling in page tables as necessary
2772 * and calling a provided function on each leaf page table.
2773 */
apply_to_page_range(struct mm_struct * mm,unsigned long addr,unsigned long size,pte_fn_t fn,void * data)2774 int apply_to_page_range(struct mm_struct *mm, unsigned long addr,
2775 unsigned long size, pte_fn_t fn, void *data)
2776 {
2777 return __apply_to_page_range(mm, addr, size, fn, data, true);
2778 }
2779 EXPORT_SYMBOL_GPL(apply_to_page_range);
2780
2781 /*
2782 * Scan a region of virtual memory, calling a provided function on
2783 * each leaf page table where it exists.
2784 *
2785 * Unlike apply_to_page_range, this does _not_ fill in page tables
2786 * where they are absent.
2787 */
apply_to_existing_page_range(struct mm_struct * mm,unsigned long addr,unsigned long size,pte_fn_t fn,void * data)2788 int apply_to_existing_page_range(struct mm_struct *mm, unsigned long addr,
2789 unsigned long size, pte_fn_t fn, void *data)
2790 {
2791 return __apply_to_page_range(mm, addr, size, fn, data, false);
2792 }
2793 EXPORT_SYMBOL_GPL(apply_to_existing_page_range);
2794
2795 /*
2796 * handle_pte_fault chooses page fault handler according to an entry which was
2797 * read non-atomically. Before making any commitment, on those architectures
2798 * or configurations (e.g. i386 with PAE) which might give a mix of unmatched
2799 * parts, do_swap_page must check under lock before unmapping the pte and
2800 * proceeding (but do_wp_page is only called after already making such a check;
2801 * and do_anonymous_page can safely check later on).
2802 */
pte_unmap_same(struct vm_fault * vmf)2803 static inline int pte_unmap_same(struct vm_fault *vmf)
2804 {
2805 int same = 1;
2806 #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPTION)
2807 if (sizeof(pte_t) > sizeof(unsigned long)) {
2808 spin_lock(vmf->ptl);
2809 same = pte_same(ptep_get(vmf->pte), vmf->orig_pte);
2810 spin_unlock(vmf->ptl);
2811 }
2812 #endif
2813 pte_unmap(vmf->pte);
2814 vmf->pte = NULL;
2815 return same;
2816 }
2817
2818 /*
2819 * Return:
2820 * 0: copied succeeded
2821 * -EHWPOISON: copy failed due to hwpoison in source page
2822 * -EAGAIN: copied failed (some other reason)
2823 */
__wp_page_copy_user(struct page * dst,struct page * src,struct vm_fault * vmf)2824 static inline int __wp_page_copy_user(struct page *dst, struct page *src,
2825 struct vm_fault *vmf)
2826 {
2827 int ret;
2828 void *kaddr;
2829 void __user *uaddr;
2830 struct vm_area_struct *vma = vmf->vma;
2831 struct mm_struct *mm = vma->vm_mm;
2832 unsigned long addr = vmf->address;
2833
2834 if (likely(src)) {
2835 if (copy_mc_user_highpage(dst, src, addr, vma)) {
2836 memory_failure_queue(page_to_pfn(src), 0);
2837 return -EHWPOISON;
2838 }
2839 return 0;
2840 }
2841
2842 /*
2843 * If the source page was a PFN mapping, we don't have
2844 * a "struct page" for it. We do a best-effort copy by
2845 * just copying from the original user address. If that
2846 * fails, we just zero-fill it. Live with it.
2847 */
2848 kaddr = kmap_atomic(dst);
2849 uaddr = (void __user *)(addr & PAGE_MASK);
2850
2851 /*
2852 * On architectures with software "accessed" bits, we would
2853 * take a double page fault, so mark it accessed here.
2854 */
2855 vmf->pte = NULL;
2856 if (!arch_has_hw_pte_young() && !pte_young(vmf->orig_pte)) {
2857 pte_t entry;
2858
2859 vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl);
2860 if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
2861 /*
2862 * Other thread has already handled the fault
2863 * and update local tlb only
2864 */
2865 if (vmf->pte)
2866 update_mmu_tlb(vma, addr, vmf->pte);
2867 ret = -EAGAIN;
2868 goto pte_unlock;
2869 }
2870
2871 entry = pte_mkyoung(vmf->orig_pte);
2872 if (ptep_set_access_flags(vma, addr, vmf->pte, entry, 0))
2873 update_mmu_cache_range(vmf, vma, addr, vmf->pte, 1);
2874 }
2875
2876 /*
2877 * This really shouldn't fail, because the page is there
2878 * in the page tables. But it might just be unreadable,
2879 * in which case we just give up and fill the result with
2880 * zeroes.
2881 */
2882 if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) {
2883 if (vmf->pte)
2884 goto warn;
2885
2886 /* Re-validate under PTL if the page is still mapped */
2887 vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl);
2888 if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
2889 /* The PTE changed under us, update local tlb */
2890 if (vmf->pte)
2891 update_mmu_tlb(vma, addr, vmf->pte);
2892 ret = -EAGAIN;
2893 goto pte_unlock;
2894 }
2895
2896 /*
2897 * The same page can be mapped back since last copy attempt.
2898 * Try to copy again under PTL.
2899 */
2900 if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) {
2901 /*
2902 * Give a warn in case there can be some obscure
2903 * use-case
2904 */
2905 warn:
2906 WARN_ON_ONCE(1);
2907 clear_page(kaddr);
2908 }
2909 }
2910
2911 ret = 0;
2912
2913 pte_unlock:
2914 if (vmf->pte)
2915 pte_unmap_unlock(vmf->pte, vmf->ptl);
2916 kunmap_atomic(kaddr);
2917 flush_dcache_page(dst);
2918
2919 return ret;
2920 }
2921
__get_fault_gfp_mask(struct vm_area_struct * vma)2922 static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma)
2923 {
2924 struct file *vm_file = vma->vm_file;
2925
2926 if (vm_file)
2927 return mapping_gfp_mask(vm_file->f_mapping) | __GFP_FS | __GFP_IO;
2928
2929 /*
2930 * Special mappings (e.g. VDSO) do not have any file so fake
2931 * a default GFP_KERNEL for them.
2932 */
2933 return GFP_KERNEL;
2934 }
2935
2936 /*
2937 * Notify the address space that the page is about to become writable so that
2938 * it can prohibit this or wait for the page to get into an appropriate state.
2939 *
2940 * We do this without the lock held, so that it can sleep if it needs to.
2941 */
do_page_mkwrite(struct vm_fault * vmf,struct folio * folio)2942 static vm_fault_t do_page_mkwrite(struct vm_fault *vmf, struct folio *folio)
2943 {
2944 vm_fault_t ret;
2945 unsigned int old_flags = vmf->flags;
2946
2947 vmf->flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE;
2948
2949 if (vmf->vma->vm_file &&
2950 IS_SWAPFILE(vmf->vma->vm_file->f_mapping->host))
2951 return VM_FAULT_SIGBUS;
2952
2953 ret = vmf->vma->vm_ops->page_mkwrite(vmf);
2954 /* Restore original flags so that caller is not surprised */
2955 vmf->flags = old_flags;
2956 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))
2957 return ret;
2958 if (unlikely(!(ret & VM_FAULT_LOCKED))) {
2959 folio_lock(folio);
2960 if (!folio->mapping) {
2961 folio_unlock(folio);
2962 return 0; /* retry */
2963 }
2964 ret |= VM_FAULT_LOCKED;
2965 } else
2966 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
2967 return ret;
2968 }
2969
2970 /*
2971 * Handle dirtying of a page in shared file mapping on a write fault.
2972 *
2973 * The function expects the page to be locked and unlocks it.
2974 */
fault_dirty_shared_page(struct vm_fault * vmf)2975 static vm_fault_t fault_dirty_shared_page(struct vm_fault *vmf)
2976 {
2977 struct vm_area_struct *vma = vmf->vma;
2978 struct address_space *mapping;
2979 struct folio *folio = page_folio(vmf->page);
2980 bool dirtied;
2981 bool page_mkwrite = vma->vm_ops && vma->vm_ops->page_mkwrite;
2982
2983 dirtied = folio_mark_dirty(folio);
2984 VM_BUG_ON_FOLIO(folio_test_anon(folio), folio);
2985 /*
2986 * Take a local copy of the address_space - folio.mapping may be zeroed
2987 * by truncate after folio_unlock(). The address_space itself remains
2988 * pinned by vma->vm_file's reference. We rely on folio_unlock()'s
2989 * release semantics to prevent the compiler from undoing this copying.
2990 */
2991 mapping = folio_raw_mapping(folio);
2992 folio_unlock(folio);
2993
2994 if (!page_mkwrite)
2995 file_update_time(vma->vm_file);
2996
2997 /*
2998 * Throttle page dirtying rate down to writeback speed.
2999 *
3000 * mapping may be NULL here because some device drivers do not
3001 * set page.mapping but still dirty their pages
3002 *
3003 * Drop the mmap_lock before waiting on IO, if we can. The file
3004 * is pinning the mapping, as per above.
3005 */
3006 if ((dirtied || page_mkwrite) && mapping) {
3007 struct file *fpin;
3008
3009 fpin = maybe_unlock_mmap_for_io(vmf, NULL);
3010 balance_dirty_pages_ratelimited(mapping);
3011 if (fpin) {
3012 fput(fpin);
3013 return VM_FAULT_COMPLETED;
3014 }
3015 }
3016
3017 return 0;
3018 }
3019
3020 /*
3021 * Handle write page faults for pages that can be reused in the current vma
3022 *
3023 * This can happen either due to the mapping being with the VM_SHARED flag,
3024 * or due to us being the last reference standing to the page. In either
3025 * case, all we need to do here is to mark the page as writable and update
3026 * any related book-keeping.
3027 */
wp_page_reuse(struct vm_fault * vmf)3028 static inline void wp_page_reuse(struct vm_fault *vmf)
3029 __releases(vmf->ptl)
3030 {
3031 struct vm_area_struct *vma = vmf->vma;
3032 struct page *page = vmf->page;
3033 pte_t entry;
3034
3035 VM_BUG_ON(!(vmf->flags & FAULT_FLAG_WRITE));
3036 VM_BUG_ON(page && PageAnon(page) && !PageAnonExclusive(page));
3037
3038 /*
3039 * Clear the pages cpupid information as the existing
3040 * information potentially belongs to a now completely
3041 * unrelated process.
3042 */
3043 if (page)
3044 page_cpupid_xchg_last(page, (1 << LAST_CPUPID_SHIFT) - 1);
3045
3046 flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
3047 entry = pte_mkyoung(vmf->orig_pte);
3048 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
3049 if (ptep_set_access_flags(vma, vmf->address, vmf->pte, entry, 1))
3050 update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
3051 pte_unmap_unlock(vmf->pte, vmf->ptl);
3052 count_vm_event(PGREUSE);
3053 }
3054
3055 /*
3056 * Handle the case of a page which we actually need to copy to a new page,
3057 * either due to COW or unsharing.
3058 *
3059 * Called with mmap_lock locked and the old page referenced, but
3060 * without the ptl held.
3061 *
3062 * High level logic flow:
3063 *
3064 * - Allocate a page, copy the content of the old page to the new one.
3065 * - Handle book keeping and accounting - cgroups, mmu-notifiers, etc.
3066 * - Take the PTL. If the pte changed, bail out and release the allocated page
3067 * - If the pte is still the way we remember it, update the page table and all
3068 * relevant references. This includes dropping the reference the page-table
3069 * held to the old page, as well as updating the rmap.
3070 * - In any case, unlock the PTL and drop the reference we took to the old page.
3071 */
wp_page_copy(struct vm_fault * vmf)3072 static vm_fault_t wp_page_copy(struct vm_fault *vmf)
3073 {
3074 const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
3075 struct vm_area_struct *vma = vmf->vma;
3076 struct mm_struct *mm = vma->vm_mm;
3077 struct folio *old_folio = NULL;
3078 struct folio *new_folio = NULL;
3079 pte_t entry;
3080 int page_copied = 0;
3081 struct mmu_notifier_range range;
3082 int ret;
3083
3084 delayacct_wpcopy_start();
3085
3086 if (vmf->page)
3087 old_folio = page_folio(vmf->page);
3088 if (unlikely(anon_vma_prepare(vma)))
3089 goto oom;
3090
3091 if (is_zero_pfn(pte_pfn(vmf->orig_pte))) {
3092 new_folio = vma_alloc_zeroed_movable_folio(vma, vmf->address);
3093 if (!new_folio)
3094 goto oom;
3095 } else {
3096 new_folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma,
3097 vmf->address, false);
3098 if (!new_folio)
3099 goto oom;
3100
3101 ret = __wp_page_copy_user(&new_folio->page, vmf->page, vmf);
3102 if (ret) {
3103 /*
3104 * COW failed, if the fault was solved by other,
3105 * it's fine. If not, userspace would re-fault on
3106 * the same address and we will handle the fault
3107 * from the second attempt.
3108 * The -EHWPOISON case will not be retried.
3109 */
3110 folio_put(new_folio);
3111 if (old_folio)
3112 folio_put(old_folio);
3113
3114 delayacct_wpcopy_end();
3115 return ret == -EHWPOISON ? VM_FAULT_HWPOISON : 0;
3116 }
3117 kmsan_copy_page_meta(&new_folio->page, vmf->page);
3118 }
3119
3120 if (mem_cgroup_charge(new_folio, mm, GFP_KERNEL))
3121 goto oom_free_new;
3122 folio_throttle_swaprate(new_folio, GFP_KERNEL);
3123
3124 __folio_mark_uptodate(new_folio);
3125
3126 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm,
3127 vmf->address & PAGE_MASK,
3128 (vmf->address & PAGE_MASK) + PAGE_SIZE);
3129 mmu_notifier_invalidate_range_start(&range);
3130
3131 /*
3132 * Re-check the pte - we dropped the lock
3133 */
3134 vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl);
3135 if (likely(vmf->pte && pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
3136 if (old_folio) {
3137 if (!folio_test_anon(old_folio)) {
3138 dec_mm_counter(mm, mm_counter_file(&old_folio->page));
3139 inc_mm_counter(mm, MM_ANONPAGES);
3140 }
3141 } else {
3142 ksm_might_unmap_zero_page(mm, vmf->orig_pte);
3143 inc_mm_counter(mm, MM_ANONPAGES);
3144 }
3145 flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
3146 entry = mk_pte(&new_folio->page, vma->vm_page_prot);
3147 entry = pte_sw_mkyoung(entry);
3148 if (unlikely(unshare)) {
3149 if (pte_soft_dirty(vmf->orig_pte))
3150 entry = pte_mksoft_dirty(entry);
3151 if (pte_uffd_wp(vmf->orig_pte))
3152 entry = pte_mkuffd_wp(entry);
3153 } else {
3154 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
3155 }
3156
3157 /*
3158 * Clear the pte entry and flush it first, before updating the
3159 * pte with the new entry, to keep TLBs on different CPUs in
3160 * sync. This code used to set the new PTE then flush TLBs, but
3161 * that left a window where the new PTE could be loaded into
3162 * some TLBs while the old PTE remains in others.
3163 */
3164 ptep_clear_flush(vma, vmf->address, vmf->pte);
3165 folio_add_new_anon_rmap(new_folio, vma, vmf->address);
3166 folio_add_lru_vma(new_folio, vma);
3167 /*
3168 * We call the notify macro here because, when using secondary
3169 * mmu page tables (such as kvm shadow page tables), we want the
3170 * new page to be mapped directly into the secondary page table.
3171 */
3172 BUG_ON(unshare && pte_write(entry));
3173 set_pte_at_notify(mm, vmf->address, vmf->pte, entry);
3174 update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
3175 if (old_folio) {
3176 /*
3177 * Only after switching the pte to the new page may
3178 * we remove the mapcount here. Otherwise another
3179 * process may come and find the rmap count decremented
3180 * before the pte is switched to the new page, and
3181 * "reuse" the old page writing into it while our pte
3182 * here still points into it and can be read by other
3183 * threads.
3184 *
3185 * The critical issue is to order this
3186 * page_remove_rmap with the ptp_clear_flush above.
3187 * Those stores are ordered by (if nothing else,)
3188 * the barrier present in the atomic_add_negative
3189 * in page_remove_rmap.
3190 *
3191 * Then the TLB flush in ptep_clear_flush ensures that
3192 * no process can access the old page before the
3193 * decremented mapcount is visible. And the old page
3194 * cannot be reused until after the decremented
3195 * mapcount is visible. So transitively, TLBs to
3196 * old page will be flushed before it can be reused.
3197 */
3198 page_remove_rmap(vmf->page, vma, false);
3199 }
3200
3201 /* Free the old page.. */
3202 new_folio = old_folio;
3203 page_copied = 1;
3204 pte_unmap_unlock(vmf->pte, vmf->ptl);
3205 } else if (vmf->pte) {
3206 update_mmu_tlb(vma, vmf->address, vmf->pte);
3207 pte_unmap_unlock(vmf->pte, vmf->ptl);
3208 }
3209
3210 mmu_notifier_invalidate_range_end(&range);
3211
3212 if (new_folio)
3213 folio_put(new_folio);
3214 if (old_folio) {
3215 if (page_copied)
3216 free_swap_cache(&old_folio->page);
3217 folio_put(old_folio);
3218 }
3219
3220 delayacct_wpcopy_end();
3221 return 0;
3222 oom_free_new:
3223 folio_put(new_folio);
3224 oom:
3225 if (old_folio)
3226 folio_put(old_folio);
3227
3228 delayacct_wpcopy_end();
3229 return VM_FAULT_OOM;
3230 }
3231
3232 /**
3233 * finish_mkwrite_fault - finish page fault for a shared mapping, making PTE
3234 * writeable once the page is prepared
3235 *
3236 * @vmf: structure describing the fault
3237 *
3238 * This function handles all that is needed to finish a write page fault in a
3239 * shared mapping due to PTE being read-only once the mapped page is prepared.
3240 * It handles locking of PTE and modifying it.
3241 *
3242 * The function expects the page to be locked or other protection against
3243 * concurrent faults / writeback (such as DAX radix tree locks).
3244 *
3245 * Return: %0 on success, %VM_FAULT_NOPAGE when PTE got changed before
3246 * we acquired PTE lock.
3247 */
finish_mkwrite_fault(struct vm_fault * vmf)3248 vm_fault_t finish_mkwrite_fault(struct vm_fault *vmf)
3249 {
3250 WARN_ON_ONCE(!(vmf->vma->vm_flags & VM_SHARED));
3251 vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, vmf->address,
3252 &vmf->ptl);
3253 if (!vmf->pte)
3254 return VM_FAULT_NOPAGE;
3255 /*
3256 * We might have raced with another page fault while we released the
3257 * pte_offset_map_lock.
3258 */
3259 if (!pte_same(ptep_get(vmf->pte), vmf->orig_pte)) {
3260 update_mmu_tlb(vmf->vma, vmf->address, vmf->pte);
3261 pte_unmap_unlock(vmf->pte, vmf->ptl);
3262 return VM_FAULT_NOPAGE;
3263 }
3264 wp_page_reuse(vmf);
3265 return 0;
3266 }
3267
3268 /*
3269 * Handle write page faults for VM_MIXEDMAP or VM_PFNMAP for a VM_SHARED
3270 * mapping
3271 */
wp_pfn_shared(struct vm_fault * vmf)3272 static vm_fault_t wp_pfn_shared(struct vm_fault *vmf)
3273 {
3274 struct vm_area_struct *vma = vmf->vma;
3275
3276 if (vma->vm_ops && vma->vm_ops->pfn_mkwrite) {
3277 vm_fault_t ret;
3278
3279 pte_unmap_unlock(vmf->pte, vmf->ptl);
3280 if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
3281 vma_end_read(vmf->vma);
3282 return VM_FAULT_RETRY;
3283 }
3284
3285 vmf->flags |= FAULT_FLAG_MKWRITE;
3286 ret = vma->vm_ops->pfn_mkwrite(vmf);
3287 if (ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))
3288 return ret;
3289 return finish_mkwrite_fault(vmf);
3290 }
3291 wp_page_reuse(vmf);
3292 return 0;
3293 }
3294
wp_page_shared(struct vm_fault * vmf,struct folio * folio)3295 static vm_fault_t wp_page_shared(struct vm_fault *vmf, struct folio *folio)
3296 __releases(vmf->ptl)
3297 {
3298 struct vm_area_struct *vma = vmf->vma;
3299 vm_fault_t ret = 0;
3300
3301 folio_get(folio);
3302
3303 if (vma->vm_ops && vma->vm_ops->page_mkwrite) {
3304 vm_fault_t tmp;
3305
3306 pte_unmap_unlock(vmf->pte, vmf->ptl);
3307 if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
3308 folio_put(folio);
3309 vma_end_read(vmf->vma);
3310 return VM_FAULT_RETRY;
3311 }
3312
3313 tmp = do_page_mkwrite(vmf, folio);
3314 if (unlikely(!tmp || (tmp &
3315 (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
3316 folio_put(folio);
3317 return tmp;
3318 }
3319 tmp = finish_mkwrite_fault(vmf);
3320 if (unlikely(tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) {
3321 folio_unlock(folio);
3322 folio_put(folio);
3323 return tmp;
3324 }
3325 } else {
3326 wp_page_reuse(vmf);
3327 folio_lock(folio);
3328 }
3329 ret |= fault_dirty_shared_page(vmf);
3330 folio_put(folio);
3331
3332 return ret;
3333 }
3334
3335 /*
3336 * This routine handles present pages, when
3337 * * users try to write to a shared page (FAULT_FLAG_WRITE)
3338 * * GUP wants to take a R/O pin on a possibly shared anonymous page
3339 * (FAULT_FLAG_UNSHARE)
3340 *
3341 * It is done by copying the page to a new address and decrementing the
3342 * shared-page counter for the old page.
3343 *
3344 * Note that this routine assumes that the protection checks have been
3345 * done by the caller (the low-level page fault routine in most cases).
3346 * Thus, with FAULT_FLAG_WRITE, we can safely just mark it writable once we've
3347 * done any necessary COW.
3348 *
3349 * In case of FAULT_FLAG_WRITE, we also mark the page dirty at this point even
3350 * though the page will change only once the write actually happens. This
3351 * avoids a few races, and potentially makes it more efficient.
3352 *
3353 * We enter with non-exclusive mmap_lock (to exclude vma changes,
3354 * but allow concurrent faults), with pte both mapped and locked.
3355 * We return with mmap_lock still held, but pte unmapped and unlocked.
3356 */
do_wp_page(struct vm_fault * vmf)3357 static vm_fault_t do_wp_page(struct vm_fault *vmf)
3358 __releases(vmf->ptl)
3359 {
3360 const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
3361 struct vm_area_struct *vma = vmf->vma;
3362 struct folio *folio = NULL;
3363
3364 if (likely(!unshare)) {
3365 if (userfaultfd_pte_wp(vma, ptep_get(vmf->pte))) {
3366 pte_unmap_unlock(vmf->pte, vmf->ptl);
3367 return handle_userfault(vmf, VM_UFFD_WP);
3368 }
3369
3370 /*
3371 * Userfaultfd write-protect can defer flushes. Ensure the TLB
3372 * is flushed in this case before copying.
3373 */
3374 if (unlikely(userfaultfd_wp(vmf->vma) &&
3375 mm_tlb_flush_pending(vmf->vma->vm_mm)))
3376 flush_tlb_page(vmf->vma, vmf->address);
3377 }
3378
3379 vmf->page = vm_normal_page(vma, vmf->address, vmf->orig_pte);
3380
3381 if (vmf->page)
3382 folio = page_folio(vmf->page);
3383
3384 /*
3385 * Shared mapping: we are guaranteed to have VM_WRITE and
3386 * FAULT_FLAG_WRITE set at this point.
3387 */
3388 if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
3389 /*
3390 * VM_MIXEDMAP !pfn_valid() case, or VM_SOFTDIRTY clear on a
3391 * VM_PFNMAP VMA.
3392 *
3393 * We should not cow pages in a shared writeable mapping.
3394 * Just mark the pages writable and/or call ops->pfn_mkwrite.
3395 */
3396 if (!vmf->page)
3397 return wp_pfn_shared(vmf);
3398 return wp_page_shared(vmf, folio);
3399 }
3400
3401 /*
3402 * Private mapping: create an exclusive anonymous page copy if reuse
3403 * is impossible. We might miss VM_WRITE for FOLL_FORCE handling.
3404 */
3405 if (folio && folio_test_anon(folio)) {
3406 /*
3407 * If the page is exclusive to this process we must reuse the
3408 * page without further checks.
3409 */
3410 if (PageAnonExclusive(vmf->page))
3411 goto reuse;
3412
3413 /*
3414 * We have to verify under folio lock: these early checks are
3415 * just an optimization to avoid locking the folio and freeing
3416 * the swapcache if there is little hope that we can reuse.
3417 *
3418 * KSM doesn't necessarily raise the folio refcount.
3419 */
3420 if (folio_test_ksm(folio) || folio_ref_count(folio) > 3)
3421 goto copy;
3422 if (!folio_test_lru(folio))
3423 /*
3424 * We cannot easily detect+handle references from
3425 * remote LRU caches or references to LRU folios.
3426 */
3427 lru_add_drain();
3428 if (folio_ref_count(folio) > 1 + folio_test_swapcache(folio))
3429 goto copy;
3430 if (!folio_trylock(folio))
3431 goto copy;
3432 if (folio_test_swapcache(folio))
3433 folio_free_swap(folio);
3434 if (folio_test_ksm(folio) || folio_ref_count(folio) != 1) {
3435 folio_unlock(folio);
3436 goto copy;
3437 }
3438 /*
3439 * Ok, we've got the only folio reference from our mapping
3440 * and the folio is locked, it's dark out, and we're wearing
3441 * sunglasses. Hit it.
3442 */
3443 page_move_anon_rmap(vmf->page, vma);
3444 folio_unlock(folio);
3445 reuse:
3446 if (unlikely(unshare)) {
3447 pte_unmap_unlock(vmf->pte, vmf->ptl);
3448 return 0;
3449 }
3450 wp_page_reuse(vmf);
3451 return 0;
3452 }
3453 copy:
3454 if ((vmf->flags & FAULT_FLAG_VMA_LOCK) && !vma->anon_vma) {
3455 pte_unmap_unlock(vmf->pte, vmf->ptl);
3456 vma_end_read(vmf->vma);
3457 return VM_FAULT_RETRY;
3458 }
3459
3460 /*
3461 * Ok, we need to copy. Oh, well..
3462 */
3463 if (folio)
3464 folio_get(folio);
3465
3466 pte_unmap_unlock(vmf->pte, vmf->ptl);
3467 #ifdef CONFIG_KSM
3468 if (folio && folio_test_ksm(folio))
3469 count_vm_event(COW_KSM);
3470 #endif
3471 return wp_page_copy(vmf);
3472 }
3473
unmap_mapping_range_vma(struct vm_area_struct * vma,unsigned long start_addr,unsigned long end_addr,struct zap_details * details)3474 static void unmap_mapping_range_vma(struct vm_area_struct *vma,
3475 unsigned long start_addr, unsigned long end_addr,
3476 struct zap_details *details)
3477 {
3478 zap_page_range_single(vma, start_addr, end_addr - start_addr, details);
3479 }
3480
unmap_mapping_range_tree(struct rb_root_cached * root,pgoff_t first_index,pgoff_t last_index,struct zap_details * details)3481 static inline void unmap_mapping_range_tree(struct rb_root_cached *root,
3482 pgoff_t first_index,
3483 pgoff_t last_index,
3484 struct zap_details *details)
3485 {
3486 struct vm_area_struct *vma;
3487 pgoff_t vba, vea, zba, zea;
3488
3489 vma_interval_tree_foreach(vma, root, first_index, last_index) {
3490 vba = vma->vm_pgoff;
3491 vea = vba + vma_pages(vma) - 1;
3492 zba = max(first_index, vba);
3493 zea = min(last_index, vea);
3494
3495 unmap_mapping_range_vma(vma,
3496 ((zba - vba) << PAGE_SHIFT) + vma->vm_start,
3497 ((zea - vba + 1) << PAGE_SHIFT) + vma->vm_start,
3498 details);
3499 }
3500 }
3501
3502 /**
3503 * unmap_mapping_folio() - Unmap single folio from processes.
3504 * @folio: The locked folio to be unmapped.
3505 *
3506 * Unmap this folio from any userspace process which still has it mmaped.
3507 * Typically, for efficiency, the range of nearby pages has already been
3508 * unmapped by unmap_mapping_pages() or unmap_mapping_range(). But once
3509 * truncation or invalidation holds the lock on a folio, it may find that
3510 * the page has been remapped again: and then uses unmap_mapping_folio()
3511 * to unmap it finally.
3512 */
unmap_mapping_folio(struct folio * folio)3513 void unmap_mapping_folio(struct folio *folio)
3514 {
3515 struct address_space *mapping = folio->mapping;
3516 struct zap_details details = { };
3517 pgoff_t first_index;
3518 pgoff_t last_index;
3519
3520 VM_BUG_ON(!folio_test_locked(folio));
3521
3522 first_index = folio->index;
3523 last_index = folio_next_index(folio) - 1;
3524
3525 details.even_cows = false;
3526 details.single_folio = folio;
3527 details.zap_flags = ZAP_FLAG_DROP_MARKER;
3528
3529 i_mmap_lock_read(mapping);
3530 if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root)))
3531 unmap_mapping_range_tree(&mapping->i_mmap, first_index,
3532 last_index, &details);
3533 i_mmap_unlock_read(mapping);
3534 }
3535
3536 /**
3537 * unmap_mapping_pages() - Unmap pages from processes.
3538 * @mapping: The address space containing pages to be unmapped.
3539 * @start: Index of first page to be unmapped.
3540 * @nr: Number of pages to be unmapped. 0 to unmap to end of file.
3541 * @even_cows: Whether to unmap even private COWed pages.
3542 *
3543 * Unmap the pages in this address space from any userspace process which
3544 * has them mmaped. Generally, you want to remove COWed pages as well when
3545 * a file is being truncated, but not when invalidating pages from the page
3546 * cache.
3547 */
unmap_mapping_pages(struct address_space * mapping,pgoff_t start,pgoff_t nr,bool even_cows)3548 void unmap_mapping_pages(struct address_space *mapping, pgoff_t start,
3549 pgoff_t nr, bool even_cows)
3550 {
3551 struct zap_details details = { };
3552 pgoff_t first_index = start;
3553 pgoff_t last_index = start + nr - 1;
3554
3555 details.even_cows = even_cows;
3556 if (last_index < first_index)
3557 last_index = ULONG_MAX;
3558
3559 i_mmap_lock_read(mapping);
3560 if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root)))
3561 unmap_mapping_range_tree(&mapping->i_mmap, first_index,
3562 last_index, &details);
3563 i_mmap_unlock_read(mapping);
3564 }
3565 EXPORT_SYMBOL_GPL(unmap_mapping_pages);
3566
3567 /**
3568 * unmap_mapping_range - unmap the portion of all mmaps in the specified
3569 * address_space corresponding to the specified byte range in the underlying
3570 * file.
3571 *
3572 * @mapping: the address space containing mmaps to be unmapped.
3573 * @holebegin: byte in first page to unmap, relative to the start of
3574 * the underlying file. This will be rounded down to a PAGE_SIZE
3575 * boundary. Note that this is different from truncate_pagecache(), which
3576 * must keep the partial page. In contrast, we must get rid of
3577 * partial pages.
3578 * @holelen: size of prospective hole in bytes. This will be rounded
3579 * up to a PAGE_SIZE boundary. A holelen of zero truncates to the
3580 * end of the file.
3581 * @even_cows: 1 when truncating a file, unmap even private COWed pages;
3582 * but 0 when invalidating pagecache, don't throw away private data.
3583 */
unmap_mapping_range(struct address_space * mapping,loff_t const holebegin,loff_t const holelen,int even_cows)3584 void unmap_mapping_range(struct address_space *mapping,
3585 loff_t const holebegin, loff_t const holelen, int even_cows)
3586 {
3587 pgoff_t hba = (pgoff_t)(holebegin) >> PAGE_SHIFT;
3588 pgoff_t hlen = ((pgoff_t)(holelen) + PAGE_SIZE - 1) >> PAGE_SHIFT;
3589
3590 /* Check for overflow. */
3591 if (sizeof(holelen) > sizeof(hlen)) {
3592 long long holeend =
3593 (holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
3594 if (holeend & ~(long long)ULONG_MAX)
3595 hlen = ULONG_MAX - hba + 1;
3596 }
3597
3598 unmap_mapping_pages(mapping, hba, hlen, even_cows);
3599 }
3600 EXPORT_SYMBOL(unmap_mapping_range);
3601
3602 /*
3603 * Restore a potential device exclusive pte to a working pte entry
3604 */
remove_device_exclusive_entry(struct vm_fault * vmf)3605 static vm_fault_t remove_device_exclusive_entry(struct vm_fault *vmf)
3606 {
3607 struct folio *folio = page_folio(vmf->page);
3608 struct vm_area_struct *vma = vmf->vma;
3609 struct mmu_notifier_range range;
3610 vm_fault_t ret;
3611
3612 /*
3613 * We need a reference to lock the folio because we don't hold
3614 * the PTL so a racing thread can remove the device-exclusive
3615 * entry and unmap it. If the folio is free the entry must
3616 * have been removed already. If it happens to have already
3617 * been re-allocated after being freed all we do is lock and
3618 * unlock it.
3619 */
3620 if (!folio_try_get(folio))
3621 return 0;
3622
3623 ret = folio_lock_or_retry(folio, vmf);
3624 if (ret) {
3625 folio_put(folio);
3626 return ret;
3627 }
3628 mmu_notifier_range_init_owner(&range, MMU_NOTIFY_EXCLUSIVE, 0,
3629 vma->vm_mm, vmf->address & PAGE_MASK,
3630 (vmf->address & PAGE_MASK) + PAGE_SIZE, NULL);
3631 mmu_notifier_invalidate_range_start(&range);
3632
3633 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
3634 &vmf->ptl);
3635 if (likely(vmf->pte && pte_same(ptep_get(vmf->pte), vmf->orig_pte)))
3636 restore_exclusive_pte(vma, vmf->page, vmf->address, vmf->pte);
3637
3638 if (vmf->pte)
3639 pte_unmap_unlock(vmf->pte, vmf->ptl);
3640 folio_unlock(folio);
3641 folio_put(folio);
3642
3643 mmu_notifier_invalidate_range_end(&range);
3644 return 0;
3645 }
3646
should_try_to_free_swap(struct folio * folio,struct vm_area_struct * vma,unsigned int fault_flags)3647 static inline bool should_try_to_free_swap(struct folio *folio,
3648 struct vm_area_struct *vma,
3649 unsigned int fault_flags)
3650 {
3651 if (!folio_test_swapcache(folio))
3652 return false;
3653 if (mem_cgroup_swap_full(folio) || (vma->vm_flags & VM_LOCKED) ||
3654 folio_test_mlocked(folio))
3655 return true;
3656 /*
3657 * If we want to map a page that's in the swapcache writable, we
3658 * have to detect via the refcount if we're really the exclusive
3659 * user. Try freeing the swapcache to get rid of the swapcache
3660 * reference only in case it's likely that we'll be the exlusive user.
3661 */
3662 return (fault_flags & FAULT_FLAG_WRITE) && !folio_test_ksm(folio) &&
3663 folio_ref_count(folio) == 2;
3664 }
3665
pte_marker_clear(struct vm_fault * vmf)3666 static vm_fault_t pte_marker_clear(struct vm_fault *vmf)
3667 {
3668 vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd,
3669 vmf->address, &vmf->ptl);
3670 if (!vmf->pte)
3671 return 0;
3672 /*
3673 * Be careful so that we will only recover a special uffd-wp pte into a
3674 * none pte. Otherwise it means the pte could have changed, so retry.
3675 *
3676 * This should also cover the case where e.g. the pte changed
3677 * quickly from a PTE_MARKER_UFFD_WP into PTE_MARKER_POISONED.
3678 * So is_pte_marker() check is not enough to safely drop the pte.
3679 */
3680 if (pte_same(vmf->orig_pte, ptep_get(vmf->pte)))
3681 pte_clear(vmf->vma->vm_mm, vmf->address, vmf->pte);
3682 pte_unmap_unlock(vmf->pte, vmf->ptl);
3683 return 0;
3684 }
3685
do_pte_missing(struct vm_fault * vmf)3686 static vm_fault_t do_pte_missing(struct vm_fault *vmf)
3687 {
3688 if (vma_is_anonymous(vmf->vma))
3689 return do_anonymous_page(vmf);
3690 else
3691 return do_fault(vmf);
3692 }
3693
3694 /*
3695 * This is actually a page-missing access, but with uffd-wp special pte
3696 * installed. It means this pte was wr-protected before being unmapped.
3697 */
pte_marker_handle_uffd_wp(struct vm_fault * vmf)3698 static vm_fault_t pte_marker_handle_uffd_wp(struct vm_fault *vmf)
3699 {
3700 /*
3701 * Just in case there're leftover special ptes even after the region
3702 * got unregistered - we can simply clear them.
3703 */
3704 if (unlikely(!userfaultfd_wp(vmf->vma)))
3705 return pte_marker_clear(vmf);
3706
3707 return do_pte_missing(vmf);
3708 }
3709
handle_pte_marker(struct vm_fault * vmf)3710 static vm_fault_t handle_pte_marker(struct vm_fault *vmf)
3711 {
3712 swp_entry_t entry = pte_to_swp_entry(vmf->orig_pte);
3713 unsigned long marker = pte_marker_get(entry);
3714
3715 /*
3716 * PTE markers should never be empty. If anything weird happened,
3717 * the best thing to do is to kill the process along with its mm.
3718 */
3719 if (WARN_ON_ONCE(!marker))
3720 return VM_FAULT_SIGBUS;
3721
3722 /* Higher priority than uffd-wp when data corrupted */
3723 if (marker & PTE_MARKER_POISONED)
3724 return VM_FAULT_HWPOISON;
3725
3726 if (pte_marker_entry_uffd_wp(entry))
3727 return pte_marker_handle_uffd_wp(vmf);
3728
3729 /* This is an unknown pte marker */
3730 return VM_FAULT_SIGBUS;
3731 }
3732
3733 /*
3734 * We enter with non-exclusive mmap_lock (to exclude vma changes,
3735 * but allow concurrent faults), and pte mapped but not yet locked.
3736 * We return with pte unmapped and unlocked.
3737 *
3738 * We return with the mmap_lock locked or unlocked in the same cases
3739 * as does filemap_fault().
3740 */
do_swap_page(struct vm_fault * vmf)3741 vm_fault_t do_swap_page(struct vm_fault *vmf)
3742 {
3743 struct vm_area_struct *vma = vmf->vma;
3744 struct folio *swapcache, *folio = NULL;
3745 struct page *page;
3746 struct swap_info_struct *si = NULL;
3747 rmap_t rmap_flags = RMAP_NONE;
3748 bool need_clear_cache = false;
3749 bool exclusive = false;
3750 swp_entry_t entry;
3751 pte_t pte;
3752 vm_fault_t ret = 0;
3753 void *shadow = NULL;
3754
3755 if (!pte_unmap_same(vmf))
3756 goto out;
3757
3758 entry = pte_to_swp_entry(vmf->orig_pte);
3759 if (unlikely(non_swap_entry(entry))) {
3760 if (is_migration_entry(entry)) {
3761 migration_entry_wait(vma->vm_mm, vmf->pmd,
3762 vmf->address);
3763 } else if (is_device_exclusive_entry(entry)) {
3764 vmf->page = pfn_swap_entry_to_page(entry);
3765 ret = remove_device_exclusive_entry(vmf);
3766 } else if (is_device_private_entry(entry)) {
3767 if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
3768 /*
3769 * migrate_to_ram is not yet ready to operate
3770 * under VMA lock.
3771 */
3772 vma_end_read(vma);
3773 ret = VM_FAULT_RETRY;
3774 goto out;
3775 }
3776
3777 vmf->page = pfn_swap_entry_to_page(entry);
3778 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
3779 vmf->address, &vmf->ptl);
3780 if (unlikely(!vmf->pte ||
3781 !pte_same(ptep_get(vmf->pte),
3782 vmf->orig_pte)))
3783 goto unlock;
3784
3785 /*
3786 * Get a page reference while we know the page can't be
3787 * freed.
3788 */
3789 get_page(vmf->page);
3790 pte_unmap_unlock(vmf->pte, vmf->ptl);
3791 ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
3792 put_page(vmf->page);
3793 } else if (is_hwpoison_entry(entry)) {
3794 ret = VM_FAULT_HWPOISON;
3795 } else if (is_pte_marker_entry(entry)) {
3796 ret = handle_pte_marker(vmf);
3797 } else {
3798 print_bad_pte(vma, vmf->address, vmf->orig_pte, NULL);
3799 ret = VM_FAULT_SIGBUS;
3800 }
3801 goto out;
3802 }
3803
3804 /* Prevent swapoff from happening to us. */
3805 si = get_swap_device(entry);
3806 if (unlikely(!si))
3807 goto out;
3808
3809 folio = swap_cache_get_folio(entry, vma, vmf->address);
3810 if (folio)
3811 page = folio_file_page(folio, swp_offset(entry));
3812 swapcache = folio;
3813
3814 if (!folio) {
3815 if (data_race(si->flags & SWP_SYNCHRONOUS_IO) &&
3816 __swap_count(entry) == 1) {
3817 /*
3818 * Prevent parallel swapin from proceeding with
3819 * the cache flag. Otherwise, another thread may
3820 * finish swapin first, free the entry, and swapout
3821 * reusing the same entry. It's undetectable as
3822 * pte_same() returns true due to entry reuse.
3823 */
3824 if (swapcache_prepare(entry)) {
3825 /* Relax a bit to prevent rapid repeated page faults */
3826 schedule_timeout_uninterruptible(1);
3827 goto out;
3828 }
3829 need_clear_cache = true;
3830
3831 /* skip swapcache */
3832 folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0,
3833 vma, vmf->address, false);
3834 page = &folio->page;
3835 if (folio) {
3836 __folio_set_locked(folio);
3837 __folio_set_swapbacked(folio);
3838
3839 if (mem_cgroup_swapin_charge_folio(folio,
3840 vma->vm_mm, GFP_KERNEL,
3841 entry)) {
3842 ret = VM_FAULT_OOM;
3843 goto out_page;
3844 }
3845 mem_cgroup_swapin_uncharge_swap(entry);
3846
3847 shadow = get_shadow_from_swap_cache(entry);
3848 if (shadow)
3849 workingset_refault(folio, shadow);
3850
3851 folio_add_lru(folio);
3852
3853 /* To provide entry to swap_readpage() */
3854 folio->swap = entry;
3855 swap_readpage(page, true, NULL);
3856 folio->private = NULL;
3857 }
3858 } else {
3859 page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE,
3860 vmf);
3861 if (page)
3862 folio = page_folio(page);
3863 swapcache = folio;
3864 }
3865
3866 if (!folio) {
3867 /*
3868 * Back out if somebody else faulted in this pte
3869 * while we released the pte lock.
3870 */
3871 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
3872 vmf->address, &vmf->ptl);
3873 if (likely(vmf->pte &&
3874 pte_same(ptep_get(vmf->pte), vmf->orig_pte)))
3875 ret = VM_FAULT_OOM;
3876 goto unlock;
3877 }
3878
3879 /* Had to read the page from swap area: Major fault */
3880 ret = VM_FAULT_MAJOR;
3881 count_vm_event(PGMAJFAULT);
3882 count_memcg_event_mm(vma->vm_mm, PGMAJFAULT);
3883 } else if (PageHWPoison(page)) {
3884 /*
3885 * hwpoisoned dirty swapcache pages are kept for killing
3886 * owner processes (which may be unknown at hwpoison time)
3887 */
3888 ret = VM_FAULT_HWPOISON;
3889 goto out_release;
3890 }
3891
3892 ret |= folio_lock_or_retry(folio, vmf);
3893 if (ret & VM_FAULT_RETRY)
3894 goto out_release;
3895
3896 if (swapcache) {
3897 /*
3898 * Make sure folio_free_swap() or swapoff did not release the
3899 * swapcache from under us. The page pin, and pte_same test
3900 * below, are not enough to exclude that. Even if it is still
3901 * swapcache, we need to check that the page's swap has not
3902 * changed.
3903 */
3904 if (unlikely(!folio_test_swapcache(folio) ||
3905 page_swap_entry(page).val != entry.val))
3906 goto out_page;
3907
3908 /*
3909 * KSM sometimes has to copy on read faults, for example, if
3910 * page->index of !PageKSM() pages would be nonlinear inside the
3911 * anon VMA -- PageKSM() is lost on actual swapout.
3912 */
3913 page = ksm_might_need_to_copy(page, vma, vmf->address);
3914 if (unlikely(!page)) {
3915 ret = VM_FAULT_OOM;
3916 goto out_page;
3917 } else if (unlikely(PTR_ERR(page) == -EHWPOISON)) {
3918 ret = VM_FAULT_HWPOISON;
3919 goto out_page;
3920 }
3921 folio = page_folio(page);
3922
3923 /*
3924 * If we want to map a page that's in the swapcache writable, we
3925 * have to detect via the refcount if we're really the exclusive
3926 * owner. Try removing the extra reference from the local LRU
3927 * caches if required.
3928 */
3929 if ((vmf->flags & FAULT_FLAG_WRITE) && folio == swapcache &&
3930 !folio_test_ksm(folio) && !folio_test_lru(folio))
3931 lru_add_drain();
3932 }
3933
3934 folio_throttle_swaprate(folio, GFP_KERNEL);
3935
3936 /*
3937 * Back out if somebody else already faulted in this pte.
3938 */
3939 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
3940 &vmf->ptl);
3941 if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte)))
3942 goto out_nomap;
3943
3944 if (unlikely(!folio_test_uptodate(folio))) {
3945 ret = VM_FAULT_SIGBUS;
3946 goto out_nomap;
3947 }
3948
3949 /*
3950 * PG_anon_exclusive reuses PG_mappedtodisk for anon pages. A swap pte
3951 * must never point at an anonymous page in the swapcache that is
3952 * PG_anon_exclusive. Sanity check that this holds and especially, that
3953 * no filesystem set PG_mappedtodisk on a page in the swapcache. Sanity
3954 * check after taking the PT lock and making sure that nobody
3955 * concurrently faulted in this page and set PG_anon_exclusive.
3956 */
3957 BUG_ON(!folio_test_anon(folio) && folio_test_mappedtodisk(folio));
3958 BUG_ON(folio_test_anon(folio) && PageAnonExclusive(page));
3959
3960 /*
3961 * Check under PT lock (to protect against concurrent fork() sharing
3962 * the swap entry concurrently) for certainly exclusive pages.
3963 */
3964 if (!folio_test_ksm(folio)) {
3965 exclusive = pte_swp_exclusive(vmf->orig_pte);
3966 if (folio != swapcache) {
3967 /*
3968 * We have a fresh page that is not exposed to the
3969 * swapcache -> certainly exclusive.
3970 */
3971 exclusive = true;
3972 } else if (exclusive && folio_test_writeback(folio) &&
3973 data_race(si->flags & SWP_STABLE_WRITES)) {
3974 /*
3975 * This is tricky: not all swap backends support
3976 * concurrent page modifications while under writeback.
3977 *
3978 * So if we stumble over such a page in the swapcache
3979 * we must not set the page exclusive, otherwise we can
3980 * map it writable without further checks and modify it
3981 * while still under writeback.
3982 *
3983 * For these problematic swap backends, simply drop the
3984 * exclusive marker: this is perfectly fine as we start
3985 * writeback only if we fully unmapped the page and
3986 * there are no unexpected references on the page after
3987 * unmapping succeeded. After fully unmapped, no
3988 * further GUP references (FOLL_GET and FOLL_PIN) can
3989 * appear, so dropping the exclusive marker and mapping
3990 * it only R/O is fine.
3991 */
3992 exclusive = false;
3993 }
3994 }
3995
3996 /*
3997 * Some architectures may have to restore extra metadata to the page
3998 * when reading from swap. This metadata may be indexed by swap entry
3999 * so this must be called before swap_free().
4000 */
4001 arch_swap_restore(entry, folio);
4002
4003 /*
4004 * Remove the swap entry and conditionally try to free up the swapcache.
4005 * We're already holding a reference on the page but haven't mapped it
4006 * yet.
4007 */
4008 swap_free(entry);
4009 if (should_try_to_free_swap(folio, vma, vmf->flags))
4010 folio_free_swap(folio);
4011
4012 inc_mm_counter(vma->vm_mm, MM_ANONPAGES);
4013 dec_mm_counter(vma->vm_mm, MM_SWAPENTS);
4014 pte = mk_pte(page, vma->vm_page_prot);
4015
4016 /*
4017 * Same logic as in do_wp_page(); however, optimize for pages that are
4018 * certainly not shared either because we just allocated them without
4019 * exposing them to the swapcache or because the swap entry indicates
4020 * exclusivity.
4021 */
4022 if (!folio_test_ksm(folio) &&
4023 (exclusive || folio_ref_count(folio) == 1)) {
4024 if (vmf->flags & FAULT_FLAG_WRITE) {
4025 pte = maybe_mkwrite(pte_mkdirty(pte), vma);
4026 vmf->flags &= ~FAULT_FLAG_WRITE;
4027 }
4028 rmap_flags |= RMAP_EXCLUSIVE;
4029 }
4030 flush_icache_page(vma, page);
4031 if (pte_swp_soft_dirty(vmf->orig_pte))
4032 pte = pte_mksoft_dirty(pte);
4033 if (pte_swp_uffd_wp(vmf->orig_pte))
4034 pte = pte_mkuffd_wp(pte);
4035 vmf->orig_pte = pte;
4036
4037 /* ksm created a completely new copy */
4038 if (unlikely(folio != swapcache && swapcache)) {
4039 page_add_new_anon_rmap(page, vma, vmf->address);
4040 folio_add_lru_vma(folio, vma);
4041 } else {
4042 page_add_anon_rmap(page, vma, vmf->address, rmap_flags);
4043 }
4044
4045 VM_BUG_ON(!folio_test_anon(folio) ||
4046 (pte_write(pte) && !PageAnonExclusive(page)));
4047 set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte);
4048 arch_do_swap_page(vma->vm_mm, vma, vmf->address, pte, vmf->orig_pte);
4049
4050 folio_unlock(folio);
4051 if (folio != swapcache && swapcache) {
4052 /*
4053 * Hold the lock to avoid the swap entry to be reused
4054 * until we take the PT lock for the pte_same() check
4055 * (to avoid false positives from pte_same). For
4056 * further safety release the lock after the swap_free
4057 * so that the swap count won't change under a
4058 * parallel locked swapcache.
4059 */
4060 folio_unlock(swapcache);
4061 folio_put(swapcache);
4062 }
4063
4064 if (vmf->flags & FAULT_FLAG_WRITE) {
4065 ret |= do_wp_page(vmf);
4066 if (ret & VM_FAULT_ERROR)
4067 ret &= VM_FAULT_ERROR;
4068 goto out;
4069 }
4070
4071 /* No need to invalidate - it was non-present before */
4072 update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
4073 unlock:
4074 if (vmf->pte)
4075 pte_unmap_unlock(vmf->pte, vmf->ptl);
4076 out:
4077 /* Clear the swap cache pin for direct swapin after PTL unlock */
4078 if (need_clear_cache)
4079 swapcache_clear(si, entry);
4080 if (si)
4081 put_swap_device(si);
4082 return ret;
4083 out_nomap:
4084 if (vmf->pte)
4085 pte_unmap_unlock(vmf->pte, vmf->ptl);
4086 out_page:
4087 folio_unlock(folio);
4088 out_release:
4089 folio_put(folio);
4090 if (folio != swapcache && swapcache) {
4091 folio_unlock(swapcache);
4092 folio_put(swapcache);
4093 }
4094 if (need_clear_cache)
4095 swapcache_clear(si, entry);
4096 if (si)
4097 put_swap_device(si);
4098 return ret;
4099 }
4100
4101 /*
4102 * We enter with non-exclusive mmap_lock (to exclude vma changes,
4103 * but allow concurrent faults), and pte mapped but not yet locked.
4104 * We return with mmap_lock still held, but pte unmapped and unlocked.
4105 */
do_anonymous_page(struct vm_fault * vmf)4106 static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
4107 {
4108 bool uffd_wp = vmf_orig_pte_uffd_wp(vmf);
4109 struct vm_area_struct *vma = vmf->vma;
4110 struct folio *folio;
4111 vm_fault_t ret = 0;
4112 pte_t entry;
4113
4114 /* File mapping without ->vm_ops ? */
4115 if (vma->vm_flags & VM_SHARED)
4116 return VM_FAULT_SIGBUS;
4117
4118 /*
4119 * Use pte_alloc() instead of pte_alloc_map(), so that OOM can
4120 * be distinguished from a transient failure of pte_offset_map().
4121 */
4122 if (pte_alloc(vma->vm_mm, vmf->pmd))
4123 return VM_FAULT_OOM;
4124
4125 /* Use the zero-page for reads */
4126 if (!(vmf->flags & FAULT_FLAG_WRITE) &&
4127 !mm_forbids_zeropage(vma->vm_mm)) {
4128 entry = pte_mkspecial(pfn_pte(my_zero_pfn(vmf->address),
4129 vma->vm_page_prot));
4130 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
4131 vmf->address, &vmf->ptl);
4132 if (!vmf->pte)
4133 goto unlock;
4134 if (vmf_pte_changed(vmf)) {
4135 update_mmu_tlb(vma, vmf->address, vmf->pte);
4136 goto unlock;
4137 }
4138 ret = check_stable_address_space(vma->vm_mm);
4139 if (ret)
4140 goto unlock;
4141 /* Deliver the page fault to userland, check inside PT lock */
4142 if (userfaultfd_missing(vma)) {
4143 pte_unmap_unlock(vmf->pte, vmf->ptl);
4144 return handle_userfault(vmf, VM_UFFD_MISSING);
4145 }
4146 goto setpte;
4147 }
4148
4149 /* Allocate our own private page. */
4150 if (unlikely(anon_vma_prepare(vma)))
4151 goto oom;
4152 folio = vma_alloc_zeroed_movable_folio(vma, vmf->address);
4153 if (!folio)
4154 goto oom;
4155
4156 if (mem_cgroup_charge(folio, vma->vm_mm, GFP_KERNEL))
4157 goto oom_free_page;
4158 folio_throttle_swaprate(folio, GFP_KERNEL);
4159
4160 /*
4161 * The memory barrier inside __folio_mark_uptodate makes sure that
4162 * preceding stores to the page contents become visible before
4163 * the set_pte_at() write.
4164 */
4165 __folio_mark_uptodate(folio);
4166
4167 entry = mk_pte(&folio->page, vma->vm_page_prot);
4168 entry = pte_sw_mkyoung(entry);
4169 if (vma->vm_flags & VM_WRITE)
4170 entry = pte_mkwrite(pte_mkdirty(entry), vma);
4171
4172 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
4173 &vmf->ptl);
4174 if (!vmf->pte)
4175 goto release;
4176 if (vmf_pte_changed(vmf)) {
4177 update_mmu_tlb(vma, vmf->address, vmf->pte);
4178 goto release;
4179 }
4180
4181 ret = check_stable_address_space(vma->vm_mm);
4182 if (ret)
4183 goto release;
4184
4185 /* Deliver the page fault to userland, check inside PT lock */
4186 if (userfaultfd_missing(vma)) {
4187 pte_unmap_unlock(vmf->pte, vmf->ptl);
4188 folio_put(folio);
4189 return handle_userfault(vmf, VM_UFFD_MISSING);
4190 }
4191
4192 inc_mm_counter(vma->vm_mm, MM_ANONPAGES);
4193 folio_add_new_anon_rmap(folio, vma, vmf->address);
4194 folio_add_lru_vma(folio, vma);
4195 setpte:
4196 if (uffd_wp)
4197 entry = pte_mkuffd_wp(entry);
4198 set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry);
4199
4200 /* No need to invalidate - it was non-present before */
4201 update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
4202 unlock:
4203 if (vmf->pte)
4204 pte_unmap_unlock(vmf->pte, vmf->ptl);
4205 return ret;
4206 release:
4207 folio_put(folio);
4208 goto unlock;
4209 oom_free_page:
4210 folio_put(folio);
4211 oom:
4212 return VM_FAULT_OOM;
4213 }
4214
4215 /*
4216 * The mmap_lock must have been held on entry, and may have been
4217 * released depending on flags and vma->vm_ops->fault() return value.
4218 * See filemap_fault() and __lock_page_retry().
4219 */
__do_fault(struct vm_fault * vmf)4220 static vm_fault_t __do_fault(struct vm_fault *vmf)
4221 {
4222 struct vm_area_struct *vma = vmf->vma;
4223 vm_fault_t ret;
4224
4225 /*
4226 * Preallocate pte before we take page_lock because this might lead to
4227 * deadlocks for memcg reclaim which waits for pages under writeback:
4228 * lock_page(A)
4229 * SetPageWriteback(A)
4230 * unlock_page(A)
4231 * lock_page(B)
4232 * lock_page(B)
4233 * pte_alloc_one
4234 * shrink_page_list
4235 * wait_on_page_writeback(A)
4236 * SetPageWriteback(B)
4237 * unlock_page(B)
4238 * # flush A, B to clear the writeback
4239 */
4240 if (pmd_none(*vmf->pmd) && !vmf->prealloc_pte) {
4241 vmf->prealloc_pte = pte_alloc_one(vma->vm_mm);
4242 if (!vmf->prealloc_pte)
4243 return VM_FAULT_OOM;
4244 }
4245
4246 ret = vma->vm_ops->fault(vmf);
4247 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY |
4248 VM_FAULT_DONE_COW)))
4249 return ret;
4250
4251 if (unlikely(PageHWPoison(vmf->page))) {
4252 struct page *page = vmf->page;
4253 vm_fault_t poisonret = VM_FAULT_HWPOISON;
4254 if (ret & VM_FAULT_LOCKED) {
4255 if (page_mapped(page))
4256 unmap_mapping_pages(page_mapping(page),
4257 page->index, 1, false);
4258 /* Retry if a clean page was removed from the cache. */
4259 if (invalidate_inode_page(page))
4260 poisonret = VM_FAULT_NOPAGE;
4261 unlock_page(page);
4262 }
4263 put_page(page);
4264 vmf->page = NULL;
4265 return poisonret;
4266 }
4267
4268 if (unlikely(!(ret & VM_FAULT_LOCKED)))
4269 lock_page(vmf->page);
4270 else
4271 VM_BUG_ON_PAGE(!PageLocked(vmf->page), vmf->page);
4272
4273 return ret;
4274 }
4275
4276 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
deposit_prealloc_pte(struct vm_fault * vmf)4277 static void deposit_prealloc_pte(struct vm_fault *vmf)
4278 {
4279 struct vm_area_struct *vma = vmf->vma;
4280
4281 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, vmf->prealloc_pte);
4282 /*
4283 * We are going to consume the prealloc table,
4284 * count that as nr_ptes.
4285 */
4286 mm_inc_nr_ptes(vma->vm_mm);
4287 vmf->prealloc_pte = NULL;
4288 }
4289
do_set_pmd(struct vm_fault * vmf,struct page * page)4290 vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page)
4291 {
4292 struct vm_area_struct *vma = vmf->vma;
4293 bool write = vmf->flags & FAULT_FLAG_WRITE;
4294 unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
4295 pmd_t entry;
4296 vm_fault_t ret = VM_FAULT_FALLBACK;
4297
4298 /*
4299 * It is too late to allocate a small folio, we already have a large
4300 * folio in the pagecache: especially s390 KVM cannot tolerate any
4301 * PMD mappings, but PTE-mapped THP are fine. So let's simply refuse any
4302 * PMD mappings if THPs are disabled.
4303 */
4304 if (thp_disabled_by_hw() || vma_thp_disabled(vma, vma->vm_flags))
4305 return ret;
4306
4307 if (!transhuge_vma_suitable(vma, haddr))
4308 return ret;
4309
4310 page = compound_head(page);
4311 if (compound_order(page) != HPAGE_PMD_ORDER)
4312 return ret;
4313
4314 /*
4315 * Just backoff if any subpage of a THP is corrupted otherwise
4316 * the corrupted page may mapped by PMD silently to escape the
4317 * check. This kind of THP just can be PTE mapped. Access to
4318 * the corrupted subpage should trigger SIGBUS as expected.
4319 */
4320 if (unlikely(PageHasHWPoisoned(page)))
4321 return ret;
4322
4323 /*
4324 * Archs like ppc64 need additional space to store information
4325 * related to pte entry. Use the preallocated table for that.
4326 */
4327 if (arch_needs_pgtable_deposit() && !vmf->prealloc_pte) {
4328 vmf->prealloc_pte = pte_alloc_one(vma->vm_mm);
4329 if (!vmf->prealloc_pte)
4330 return VM_FAULT_OOM;
4331 }
4332
4333 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
4334 if (unlikely(!pmd_none(*vmf->pmd)))
4335 goto out;
4336
4337 flush_icache_pages(vma, page, HPAGE_PMD_NR);
4338
4339 entry = mk_huge_pmd(page, vma->vm_page_prot);
4340 if (write)
4341 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
4342
4343 add_mm_counter(vma->vm_mm, mm_counter_file(page), HPAGE_PMD_NR);
4344 page_add_file_rmap(page, vma, true);
4345
4346 /*
4347 * deposit and withdraw with pmd lock held
4348 */
4349 if (arch_needs_pgtable_deposit())
4350 deposit_prealloc_pte(vmf);
4351
4352 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
4353
4354 update_mmu_cache_pmd(vma, haddr, vmf->pmd);
4355
4356 /* fault is handled */
4357 ret = 0;
4358 count_vm_event(THP_FILE_MAPPED);
4359 out:
4360 spin_unlock(vmf->ptl);
4361 return ret;
4362 }
4363 #else
do_set_pmd(struct vm_fault * vmf,struct page * page)4364 vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page)
4365 {
4366 return VM_FAULT_FALLBACK;
4367 }
4368 #endif
4369
4370 /**
4371 * set_pte_range - Set a range of PTEs to point to pages in a folio.
4372 * @vmf: Fault decription.
4373 * @folio: The folio that contains @page.
4374 * @page: The first page to create a PTE for.
4375 * @nr: The number of PTEs to create.
4376 * @addr: The first address to create a PTE for.
4377 */
set_pte_range(struct vm_fault * vmf,struct folio * folio,struct page * page,unsigned int nr,unsigned long addr)4378 void set_pte_range(struct vm_fault *vmf, struct folio *folio,
4379 struct page *page, unsigned int nr, unsigned long addr)
4380 {
4381 struct vm_area_struct *vma = vmf->vma;
4382 bool uffd_wp = vmf_orig_pte_uffd_wp(vmf);
4383 bool write = vmf->flags & FAULT_FLAG_WRITE;
4384 bool prefault = !in_range(vmf->address, addr, nr * PAGE_SIZE);
4385 pte_t entry;
4386
4387 flush_icache_pages(vma, page, nr);
4388 entry = mk_pte(page, vma->vm_page_prot);
4389
4390 if (prefault && arch_wants_old_prefaulted_pte())
4391 entry = pte_mkold(entry);
4392 else
4393 entry = pte_sw_mkyoung(entry);
4394
4395 if (write)
4396 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
4397 if (unlikely(uffd_wp))
4398 entry = pte_mkuffd_wp(entry);
4399 /* copy-on-write page */
4400 if (write && !(vma->vm_flags & VM_SHARED)) {
4401 add_mm_counter(vma->vm_mm, MM_ANONPAGES, nr);
4402 VM_BUG_ON_FOLIO(nr != 1, folio);
4403 folio_add_new_anon_rmap(folio, vma, addr);
4404 folio_add_lru_vma(folio, vma);
4405 } else {
4406 add_mm_counter(vma->vm_mm, mm_counter_file(page), nr);
4407 folio_add_file_rmap_range(folio, page, nr, vma, false);
4408 }
4409 set_ptes(vma->vm_mm, addr, vmf->pte, entry, nr);
4410
4411 /* no need to invalidate: a not-present page won't be cached */
4412 update_mmu_cache_range(vmf, vma, addr, vmf->pte, nr);
4413 }
4414
vmf_pte_changed(struct vm_fault * vmf)4415 static bool vmf_pte_changed(struct vm_fault *vmf)
4416 {
4417 if (vmf->flags & FAULT_FLAG_ORIG_PTE_VALID)
4418 return !pte_same(ptep_get(vmf->pte), vmf->orig_pte);
4419
4420 return !pte_none(ptep_get(vmf->pte));
4421 }
4422
4423 /**
4424 * finish_fault - finish page fault once we have prepared the page to fault
4425 *
4426 * @vmf: structure describing the fault
4427 *
4428 * This function handles all that is needed to finish a page fault once the
4429 * page to fault in is prepared. It handles locking of PTEs, inserts PTE for
4430 * given page, adds reverse page mapping, handles memcg charges and LRU
4431 * addition.
4432 *
4433 * The function expects the page to be locked and on success it consumes a
4434 * reference of a page being mapped (for the PTE which maps it).
4435 *
4436 * Return: %0 on success, %VM_FAULT_ code in case of error.
4437 */
finish_fault(struct vm_fault * vmf)4438 vm_fault_t finish_fault(struct vm_fault *vmf)
4439 {
4440 struct vm_area_struct *vma = vmf->vma;
4441 struct page *page;
4442 vm_fault_t ret;
4443
4444 /* Did we COW the page? */
4445 if ((vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED))
4446 page = vmf->cow_page;
4447 else
4448 page = vmf->page;
4449
4450 /*
4451 * check even for read faults because we might have lost our CoWed
4452 * page
4453 */
4454 if (!(vma->vm_flags & VM_SHARED)) {
4455 ret = check_stable_address_space(vma->vm_mm);
4456 if (ret)
4457 return ret;
4458 }
4459
4460 if (pmd_none(*vmf->pmd)) {
4461 if (PageTransCompound(page)) {
4462 ret = do_set_pmd(vmf, page);
4463 if (ret != VM_FAULT_FALLBACK)
4464 return ret;
4465 }
4466
4467 if (vmf->prealloc_pte)
4468 pmd_install(vma->vm_mm, vmf->pmd, &vmf->prealloc_pte);
4469 else if (unlikely(pte_alloc(vma->vm_mm, vmf->pmd)))
4470 return VM_FAULT_OOM;
4471 }
4472
4473 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
4474 vmf->address, &vmf->ptl);
4475 if (!vmf->pte)
4476 return VM_FAULT_NOPAGE;
4477
4478 /* Re-check under ptl */
4479 if (likely(!vmf_pte_changed(vmf))) {
4480 struct folio *folio = page_folio(page);
4481
4482 set_pte_range(vmf, folio, page, 1, vmf->address);
4483 ret = 0;
4484 } else {
4485 update_mmu_tlb(vma, vmf->address, vmf->pte);
4486 ret = VM_FAULT_NOPAGE;
4487 }
4488
4489 pte_unmap_unlock(vmf->pte, vmf->ptl);
4490 return ret;
4491 }
4492
4493 static unsigned long fault_around_pages __read_mostly =
4494 65536 >> PAGE_SHIFT;
4495
4496 #ifdef CONFIG_DEBUG_FS
fault_around_bytes_get(void * data,u64 * val)4497 static int fault_around_bytes_get(void *data, u64 *val)
4498 {
4499 *val = fault_around_pages << PAGE_SHIFT;
4500 return 0;
4501 }
4502
4503 /*
4504 * fault_around_bytes must be rounded down to the nearest page order as it's
4505 * what do_fault_around() expects to see.
4506 */
fault_around_bytes_set(void * data,u64 val)4507 static int fault_around_bytes_set(void *data, u64 val)
4508 {
4509 if (val / PAGE_SIZE > PTRS_PER_PTE)
4510 return -EINVAL;
4511
4512 /*
4513 * The minimum value is 1 page, however this results in no fault-around
4514 * at all. See should_fault_around().
4515 */
4516 fault_around_pages = max(rounddown_pow_of_two(val) >> PAGE_SHIFT, 1UL);
4517
4518 return 0;
4519 }
4520 DEFINE_DEBUGFS_ATTRIBUTE(fault_around_bytes_fops,
4521 fault_around_bytes_get, fault_around_bytes_set, "%llu\n");
4522
fault_around_debugfs(void)4523 static int __init fault_around_debugfs(void)
4524 {
4525 debugfs_create_file_unsafe("fault_around_bytes", 0644, NULL, NULL,
4526 &fault_around_bytes_fops);
4527 return 0;
4528 }
4529 late_initcall(fault_around_debugfs);
4530 #endif
4531
4532 /*
4533 * do_fault_around() tries to map few pages around the fault address. The hope
4534 * is that the pages will be needed soon and this will lower the number of
4535 * faults to handle.
4536 *
4537 * It uses vm_ops->map_pages() to map the pages, which skips the page if it's
4538 * not ready to be mapped: not up-to-date, locked, etc.
4539 *
4540 * This function doesn't cross VMA or page table boundaries, in order to call
4541 * map_pages() and acquire a PTE lock only once.
4542 *
4543 * fault_around_pages defines how many pages we'll try to map.
4544 * do_fault_around() expects it to be set to a power of two less than or equal
4545 * to PTRS_PER_PTE.
4546 *
4547 * The virtual address of the area that we map is naturally aligned to
4548 * fault_around_pages * PAGE_SIZE rounded down to the machine page size
4549 * (and therefore to page order). This way it's easier to guarantee
4550 * that we don't cross page table boundaries.
4551 */
do_fault_around(struct vm_fault * vmf)4552 static vm_fault_t do_fault_around(struct vm_fault *vmf)
4553 {
4554 pgoff_t nr_pages = READ_ONCE(fault_around_pages);
4555 pgoff_t pte_off = pte_index(vmf->address);
4556 /* The page offset of vmf->address within the VMA. */
4557 pgoff_t vma_off = vmf->pgoff - vmf->vma->vm_pgoff;
4558 pgoff_t from_pte, to_pte;
4559 vm_fault_t ret;
4560
4561 /* The PTE offset of the start address, clamped to the VMA. */
4562 from_pte = max(ALIGN_DOWN(pte_off, nr_pages),
4563 pte_off - min(pte_off, vma_off));
4564
4565 /* The PTE offset of the end address, clamped to the VMA and PTE. */
4566 to_pte = min3(from_pte + nr_pages, (pgoff_t)PTRS_PER_PTE,
4567 pte_off + vma_pages(vmf->vma) - vma_off) - 1;
4568
4569 if (pmd_none(*vmf->pmd)) {
4570 vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm);
4571 if (!vmf->prealloc_pte)
4572 return VM_FAULT_OOM;
4573 }
4574
4575 rcu_read_lock();
4576 ret = vmf->vma->vm_ops->map_pages(vmf,
4577 vmf->pgoff + from_pte - pte_off,
4578 vmf->pgoff + to_pte - pte_off);
4579 rcu_read_unlock();
4580
4581 return ret;
4582 }
4583
4584 /* Return true if we should do read fault-around, false otherwise */
should_fault_around(struct vm_fault * vmf)4585 static inline bool should_fault_around(struct vm_fault *vmf)
4586 {
4587 /* No ->map_pages? No way to fault around... */
4588 if (!vmf->vma->vm_ops->map_pages)
4589 return false;
4590
4591 if (uffd_disable_fault_around(vmf->vma))
4592 return false;
4593
4594 /* A single page implies no faulting 'around' at all. */
4595 return fault_around_pages > 1;
4596 }
4597
do_read_fault(struct vm_fault * vmf)4598 static vm_fault_t do_read_fault(struct vm_fault *vmf)
4599 {
4600 vm_fault_t ret = 0;
4601 struct folio *folio;
4602
4603 /*
4604 * Let's call ->map_pages() first and use ->fault() as fallback
4605 * if page by the offset is not ready to be mapped (cold cache or
4606 * something).
4607 */
4608 if (should_fault_around(vmf)) {
4609 ret = do_fault_around(vmf);
4610 if (ret)
4611 return ret;
4612 }
4613
4614 if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
4615 vma_end_read(vmf->vma);
4616 return VM_FAULT_RETRY;
4617 }
4618
4619 ret = __do_fault(vmf);
4620 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
4621 return ret;
4622
4623 ret |= finish_fault(vmf);
4624 folio = page_folio(vmf->page);
4625 folio_unlock(folio);
4626 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
4627 folio_put(folio);
4628 return ret;
4629 }
4630
do_cow_fault(struct vm_fault * vmf)4631 static vm_fault_t do_cow_fault(struct vm_fault *vmf)
4632 {
4633 struct vm_area_struct *vma = vmf->vma;
4634 vm_fault_t ret;
4635
4636 if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
4637 vma_end_read(vma);
4638 return VM_FAULT_RETRY;
4639 }
4640
4641 if (unlikely(anon_vma_prepare(vma)))
4642 return VM_FAULT_OOM;
4643
4644 vmf->cow_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vmf->address);
4645 if (!vmf->cow_page)
4646 return VM_FAULT_OOM;
4647
4648 if (mem_cgroup_charge(page_folio(vmf->cow_page), vma->vm_mm,
4649 GFP_KERNEL)) {
4650 put_page(vmf->cow_page);
4651 return VM_FAULT_OOM;
4652 }
4653 folio_throttle_swaprate(page_folio(vmf->cow_page), GFP_KERNEL);
4654
4655 ret = __do_fault(vmf);
4656 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
4657 goto uncharge_out;
4658 if (ret & VM_FAULT_DONE_COW)
4659 return ret;
4660
4661 copy_user_highpage(vmf->cow_page, vmf->page, vmf->address, vma);
4662 __SetPageUptodate(vmf->cow_page);
4663
4664 ret |= finish_fault(vmf);
4665 unlock_page(vmf->page);
4666 put_page(vmf->page);
4667 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
4668 goto uncharge_out;
4669 return ret;
4670 uncharge_out:
4671 put_page(vmf->cow_page);
4672 return ret;
4673 }
4674
do_shared_fault(struct vm_fault * vmf)4675 static vm_fault_t do_shared_fault(struct vm_fault *vmf)
4676 {
4677 struct vm_area_struct *vma = vmf->vma;
4678 vm_fault_t ret, tmp;
4679 struct folio *folio;
4680
4681 if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
4682 vma_end_read(vma);
4683 return VM_FAULT_RETRY;
4684 }
4685
4686 ret = __do_fault(vmf);
4687 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
4688 return ret;
4689
4690 folio = page_folio(vmf->page);
4691
4692 /*
4693 * Check if the backing address space wants to know that the page is
4694 * about to become writable
4695 */
4696 if (vma->vm_ops->page_mkwrite) {
4697 folio_unlock(folio);
4698 tmp = do_page_mkwrite(vmf, folio);
4699 if (unlikely(!tmp ||
4700 (tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
4701 folio_put(folio);
4702 return tmp;
4703 }
4704 }
4705
4706 ret |= finish_fault(vmf);
4707 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE |
4708 VM_FAULT_RETRY))) {
4709 folio_unlock(folio);
4710 folio_put(folio);
4711 return ret;
4712 }
4713
4714 ret |= fault_dirty_shared_page(vmf);
4715 return ret;
4716 }
4717
4718 /*
4719 * We enter with non-exclusive mmap_lock (to exclude vma changes,
4720 * but allow concurrent faults).
4721 * The mmap_lock may have been released depending on flags and our
4722 * return value. See filemap_fault() and __folio_lock_or_retry().
4723 * If mmap_lock is released, vma may become invalid (for example
4724 * by other thread calling munmap()).
4725 */
do_fault(struct vm_fault * vmf)4726 static vm_fault_t do_fault(struct vm_fault *vmf)
4727 {
4728 struct vm_area_struct *vma = vmf->vma;
4729 struct mm_struct *vm_mm = vma->vm_mm;
4730 vm_fault_t ret;
4731
4732 /*
4733 * The VMA was not fully populated on mmap() or missing VM_DONTEXPAND
4734 */
4735 if (!vma->vm_ops->fault) {
4736 vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd,
4737 vmf->address, &vmf->ptl);
4738 if (unlikely(!vmf->pte))
4739 ret = VM_FAULT_SIGBUS;
4740 else {
4741 /*
4742 * Make sure this is not a temporary clearing of pte
4743 * by holding ptl and checking again. A R/M/W update
4744 * of pte involves: take ptl, clearing the pte so that
4745 * we don't have concurrent modification by hardware
4746 * followed by an update.
4747 */
4748 if (unlikely(pte_none(ptep_get(vmf->pte))))
4749 ret = VM_FAULT_SIGBUS;
4750 else
4751 ret = VM_FAULT_NOPAGE;
4752
4753 pte_unmap_unlock(vmf->pte, vmf->ptl);
4754 }
4755 } else if (!(vmf->flags & FAULT_FLAG_WRITE))
4756 ret = do_read_fault(vmf);
4757 else if (!(vma->vm_flags & VM_SHARED))
4758 ret = do_cow_fault(vmf);
4759 else
4760 ret = do_shared_fault(vmf);
4761
4762 /* preallocated pagetable is unused: free it */
4763 if (vmf->prealloc_pte) {
4764 pte_free(vm_mm, vmf->prealloc_pte);
4765 vmf->prealloc_pte = NULL;
4766 }
4767 return ret;
4768 }
4769
numa_migrate_prep(struct page * page,struct vm_area_struct * vma,unsigned long addr,int page_nid,int * flags)4770 int numa_migrate_prep(struct page *page, struct vm_area_struct *vma,
4771 unsigned long addr, int page_nid, int *flags)
4772 {
4773 get_page(page);
4774
4775 /* Record the current PID acceesing VMA */
4776 vma_set_access_pid_bit(vma);
4777
4778 count_vm_numa_event(NUMA_HINT_FAULTS);
4779 if (page_nid == numa_node_id()) {
4780 count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
4781 *flags |= TNF_FAULT_LOCAL;
4782 }
4783
4784 return mpol_misplaced(page, vma, addr);
4785 }
4786
do_numa_page(struct vm_fault * vmf)4787 static vm_fault_t do_numa_page(struct vm_fault *vmf)
4788 {
4789 struct vm_area_struct *vma = vmf->vma;
4790 struct page *page = NULL;
4791 int page_nid = NUMA_NO_NODE;
4792 bool writable = false;
4793 int last_cpupid;
4794 int target_nid;
4795 pte_t pte, old_pte;
4796 int flags = 0;
4797
4798 /*
4799 * The "pte" at this point cannot be used safely without
4800 * validation through pte_unmap_same(). It's of NUMA type but
4801 * the pfn may be screwed if the read is non atomic.
4802 */
4803 spin_lock(vmf->ptl);
4804 if (unlikely(!pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
4805 pte_unmap_unlock(vmf->pte, vmf->ptl);
4806 return 0;
4807 }
4808
4809 /* Get the normal PTE */
4810 old_pte = ptep_get(vmf->pte);
4811 pte = pte_modify(old_pte, vma->vm_page_prot);
4812
4813 /*
4814 * Detect now whether the PTE could be writable; this information
4815 * is only valid while holding the PT lock.
4816 */
4817 writable = pte_write(pte);
4818 if (!writable && vma_wants_manual_pte_write_upgrade(vma) &&
4819 can_change_pte_writable(vma, vmf->address, pte))
4820 writable = true;
4821
4822 page = vm_normal_page(vma, vmf->address, pte);
4823 if (!page || is_zone_device_page(page))
4824 goto out_map;
4825
4826 /* TODO: handle PTE-mapped THP */
4827 if (PageCompound(page))
4828 goto out_map;
4829
4830 /*
4831 * Avoid grouping on RO pages in general. RO pages shouldn't hurt as
4832 * much anyway since they can be in shared cache state. This misses
4833 * the case where a mapping is writable but the process never writes
4834 * to it but pte_write gets cleared during protection updates and
4835 * pte_dirty has unpredictable behaviour between PTE scan updates,
4836 * background writeback, dirty balancing and application behaviour.
4837 */
4838 if (!writable)
4839 flags |= TNF_NO_GROUP;
4840
4841 /*
4842 * Flag if the page is shared between multiple address spaces. This
4843 * is later used when determining whether to group tasks together
4844 */
4845 if (page_mapcount(page) > 1 && (vma->vm_flags & VM_SHARED))
4846 flags |= TNF_SHARED;
4847
4848 page_nid = page_to_nid(page);
4849 /*
4850 * For memory tiering mode, cpupid of slow memory page is used
4851 * to record page access time. So use default value.
4852 */
4853 if ((sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) &&
4854 !node_is_toptier(page_nid))
4855 last_cpupid = (-1 & LAST_CPUPID_MASK);
4856 else
4857 last_cpupid = page_cpupid_last(page);
4858 target_nid = numa_migrate_prep(page, vma, vmf->address, page_nid,
4859 &flags);
4860 if (target_nid == NUMA_NO_NODE) {
4861 put_page(page);
4862 goto out_map;
4863 }
4864 pte_unmap_unlock(vmf->pte, vmf->ptl);
4865 writable = false;
4866
4867 /* Migrate to the requested node */
4868 if (migrate_misplaced_page(page, vma, target_nid)) {
4869 page_nid = target_nid;
4870 flags |= TNF_MIGRATED;
4871 task_numa_fault(last_cpupid, page_nid, 1, flags);
4872 return 0;
4873 }
4874
4875 flags |= TNF_MIGRATE_FAIL;
4876 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
4877 vmf->address, &vmf->ptl);
4878 if (unlikely(!vmf->pte))
4879 return 0;
4880 if (unlikely(!pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
4881 pte_unmap_unlock(vmf->pte, vmf->ptl);
4882 return 0;
4883 }
4884 out_map:
4885 /*
4886 * Make it present again, depending on how arch implements
4887 * non-accessible ptes, some can allow access by kernel mode.
4888 */
4889 old_pte = ptep_modify_prot_start(vma, vmf->address, vmf->pte);
4890 pte = pte_modify(old_pte, vma->vm_page_prot);
4891 pte = pte_mkyoung(pte);
4892 if (writable)
4893 pte = pte_mkwrite(pte, vma);
4894 ptep_modify_prot_commit(vma, vmf->address, vmf->pte, old_pte, pte);
4895 update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
4896 pte_unmap_unlock(vmf->pte, vmf->ptl);
4897
4898 if (page_nid != NUMA_NO_NODE)
4899 task_numa_fault(last_cpupid, page_nid, 1, flags);
4900 return 0;
4901 }
4902
create_huge_pmd(struct vm_fault * vmf)4903 static inline vm_fault_t create_huge_pmd(struct vm_fault *vmf)
4904 {
4905 struct vm_area_struct *vma = vmf->vma;
4906 if (vma_is_anonymous(vma))
4907 return do_huge_pmd_anonymous_page(vmf);
4908 if (vma->vm_ops->huge_fault)
4909 return vma->vm_ops->huge_fault(vmf, PMD_ORDER);
4910 return VM_FAULT_FALLBACK;
4911 }
4912
4913 /* `inline' is required to avoid gcc 4.1.2 build error */
wp_huge_pmd(struct vm_fault * vmf)4914 static inline vm_fault_t wp_huge_pmd(struct vm_fault *vmf)
4915 {
4916 struct vm_area_struct *vma = vmf->vma;
4917 const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
4918 vm_fault_t ret;
4919
4920 if (vma_is_anonymous(vma)) {
4921 if (likely(!unshare) &&
4922 userfaultfd_huge_pmd_wp(vma, vmf->orig_pmd))
4923 return handle_userfault(vmf, VM_UFFD_WP);
4924 return do_huge_pmd_wp_page(vmf);
4925 }
4926
4927 if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
4928 if (vma->vm_ops->huge_fault) {
4929 ret = vma->vm_ops->huge_fault(vmf, PMD_ORDER);
4930 if (!(ret & VM_FAULT_FALLBACK))
4931 return ret;
4932 }
4933 }
4934
4935 /* COW or write-notify handled on pte level: split pmd. */
4936 __split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL);
4937
4938 return VM_FAULT_FALLBACK;
4939 }
4940
create_huge_pud(struct vm_fault * vmf)4941 static vm_fault_t create_huge_pud(struct vm_fault *vmf)
4942 {
4943 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
4944 defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
4945 struct vm_area_struct *vma = vmf->vma;
4946 /* No support for anonymous transparent PUD pages yet */
4947 if (vma_is_anonymous(vma))
4948 return VM_FAULT_FALLBACK;
4949 if (vma->vm_ops->huge_fault)
4950 return vma->vm_ops->huge_fault(vmf, PUD_ORDER);
4951 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
4952 return VM_FAULT_FALLBACK;
4953 }
4954
wp_huge_pud(struct vm_fault * vmf,pud_t orig_pud)4955 static vm_fault_t wp_huge_pud(struct vm_fault *vmf, pud_t orig_pud)
4956 {
4957 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
4958 defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
4959 struct vm_area_struct *vma = vmf->vma;
4960 vm_fault_t ret;
4961
4962 /* No support for anonymous transparent PUD pages yet */
4963 if (vma_is_anonymous(vma))
4964 goto split;
4965 if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
4966 if (vma->vm_ops->huge_fault) {
4967 ret = vma->vm_ops->huge_fault(vmf, PUD_ORDER);
4968 if (!(ret & VM_FAULT_FALLBACK))
4969 return ret;
4970 }
4971 }
4972 split:
4973 /* COW or write-notify not handled on PUD level: split pud.*/
4974 __split_huge_pud(vma, vmf->pud, vmf->address);
4975 #endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
4976 return VM_FAULT_FALLBACK;
4977 }
4978
4979 /*
4980 * These routines also need to handle stuff like marking pages dirty
4981 * and/or accessed for architectures that don't do it in hardware (most
4982 * RISC architectures). The early dirtying is also good on the i386.
4983 *
4984 * There is also a hook called "update_mmu_cache()" that architectures
4985 * with external mmu caches can use to update those (ie the Sparc or
4986 * PowerPC hashed page tables that act as extended TLBs).
4987 *
4988 * We enter with non-exclusive mmap_lock (to exclude vma changes, but allow
4989 * concurrent faults).
4990 *
4991 * The mmap_lock may have been released depending on flags and our return value.
4992 * See filemap_fault() and __folio_lock_or_retry().
4993 */
handle_pte_fault(struct vm_fault * vmf)4994 static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
4995 {
4996 pte_t entry;
4997
4998 if (unlikely(pmd_none(*vmf->pmd))) {
4999 /*
5000 * Leave __pte_alloc() until later: because vm_ops->fault may
5001 * want to allocate huge page, and if we expose page table
5002 * for an instant, it will be difficult to retract from
5003 * concurrent faults and from rmap lookups.
5004 */
5005 vmf->pte = NULL;
5006 vmf->flags &= ~FAULT_FLAG_ORIG_PTE_VALID;
5007 } else {
5008 /*
5009 * A regular pmd is established and it can't morph into a huge
5010 * pmd by anon khugepaged, since that takes mmap_lock in write
5011 * mode; but shmem or file collapse to THP could still morph
5012 * it into a huge pmd: just retry later if so.
5013 */
5014 vmf->pte = pte_offset_map_nolock(vmf->vma->vm_mm, vmf->pmd,
5015 vmf->address, &vmf->ptl);
5016 if (unlikely(!vmf->pte))
5017 return 0;
5018 vmf->orig_pte = ptep_get_lockless(vmf->pte);
5019 vmf->flags |= FAULT_FLAG_ORIG_PTE_VALID;
5020
5021 if (pte_none(vmf->orig_pte)) {
5022 pte_unmap(vmf->pte);
5023 vmf->pte = NULL;
5024 }
5025 }
5026
5027 if (!vmf->pte)
5028 return do_pte_missing(vmf);
5029
5030 if (!pte_present(vmf->orig_pte))
5031 return do_swap_page(vmf);
5032
5033 if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma))
5034 return do_numa_page(vmf);
5035
5036 spin_lock(vmf->ptl);
5037 entry = vmf->orig_pte;
5038 if (unlikely(!pte_same(ptep_get(vmf->pte), entry))) {
5039 update_mmu_tlb(vmf->vma, vmf->address, vmf->pte);
5040 goto unlock;
5041 }
5042 if (vmf->flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) {
5043 if (!pte_write(entry))
5044 return do_wp_page(vmf);
5045 else if (likely(vmf->flags & FAULT_FLAG_WRITE))
5046 entry = pte_mkdirty(entry);
5047 }
5048 entry = pte_mkyoung(entry);
5049 if (ptep_set_access_flags(vmf->vma, vmf->address, vmf->pte, entry,
5050 vmf->flags & FAULT_FLAG_WRITE)) {
5051 update_mmu_cache_range(vmf, vmf->vma, vmf->address,
5052 vmf->pte, 1);
5053 } else {
5054 /* Skip spurious TLB flush for retried page fault */
5055 if (vmf->flags & FAULT_FLAG_TRIED)
5056 goto unlock;
5057 /*
5058 * This is needed only for protection faults but the arch code
5059 * is not yet telling us if this is a protection fault or not.
5060 * This still avoids useless tlb flushes for .text page faults
5061 * with threads.
5062 */
5063 if (vmf->flags & FAULT_FLAG_WRITE)
5064 flush_tlb_fix_spurious_fault(vmf->vma, vmf->address,
5065 vmf->pte);
5066 }
5067 unlock:
5068 pte_unmap_unlock(vmf->pte, vmf->ptl);
5069 return 0;
5070 }
5071
5072 /*
5073 * On entry, we hold either the VMA lock or the mmap_lock
5074 * (FAULT_FLAG_VMA_LOCK tells you which). If VM_FAULT_RETRY is set in
5075 * the result, the mmap_lock is not held on exit. See filemap_fault()
5076 * and __folio_lock_or_retry().
5077 */
__handle_mm_fault(struct vm_area_struct * vma,unsigned long address,unsigned int flags)5078 static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
5079 unsigned long address, unsigned int flags)
5080 {
5081 struct vm_fault vmf = {
5082 .vma = vma,
5083 .address = address & PAGE_MASK,
5084 .real_address = address,
5085 .flags = flags,
5086 .pgoff = linear_page_index(vma, address),
5087 .gfp_mask = __get_fault_gfp_mask(vma),
5088 };
5089 struct mm_struct *mm = vma->vm_mm;
5090 unsigned long vm_flags = vma->vm_flags;
5091 pgd_t *pgd;
5092 p4d_t *p4d;
5093 vm_fault_t ret;
5094
5095 pgd = pgd_offset(mm, address);
5096 p4d = p4d_alloc(mm, pgd, address);
5097 if (!p4d)
5098 return VM_FAULT_OOM;
5099
5100 vmf.pud = pud_alloc(mm, p4d, address);
5101 if (!vmf.pud)
5102 return VM_FAULT_OOM;
5103 retry_pud:
5104 if (pud_none(*vmf.pud) &&
5105 hugepage_vma_check(vma, vm_flags, false, true, true)) {
5106 ret = create_huge_pud(&vmf);
5107 if (!(ret & VM_FAULT_FALLBACK))
5108 return ret;
5109 } else {
5110 pud_t orig_pud = *vmf.pud;
5111
5112 barrier();
5113 if (pud_trans_huge(orig_pud) || pud_devmap(orig_pud)) {
5114
5115 /*
5116 * TODO once we support anonymous PUDs: NUMA case and
5117 * FAULT_FLAG_UNSHARE handling.
5118 */
5119 if ((flags & FAULT_FLAG_WRITE) && !pud_write(orig_pud)) {
5120 ret = wp_huge_pud(&vmf, orig_pud);
5121 if (!(ret & VM_FAULT_FALLBACK))
5122 return ret;
5123 } else {
5124 huge_pud_set_accessed(&vmf, orig_pud);
5125 return 0;
5126 }
5127 }
5128 }
5129
5130 vmf.pmd = pmd_alloc(mm, vmf.pud, address);
5131 if (!vmf.pmd)
5132 return VM_FAULT_OOM;
5133
5134 /* Huge pud page fault raced with pmd_alloc? */
5135 if (pud_trans_unstable(vmf.pud))
5136 goto retry_pud;
5137
5138 if (pmd_none(*vmf.pmd) &&
5139 hugepage_vma_check(vma, vm_flags, false, true, true)) {
5140 ret = create_huge_pmd(&vmf);
5141 if (!(ret & VM_FAULT_FALLBACK))
5142 return ret;
5143 } else {
5144 vmf.orig_pmd = pmdp_get_lockless(vmf.pmd);
5145
5146 if (unlikely(is_swap_pmd(vmf.orig_pmd))) {
5147 VM_BUG_ON(thp_migration_supported() &&
5148 !is_pmd_migration_entry(vmf.orig_pmd));
5149 if (is_pmd_migration_entry(vmf.orig_pmd))
5150 pmd_migration_entry_wait(mm, vmf.pmd);
5151 return 0;
5152 }
5153 if (pmd_trans_huge(vmf.orig_pmd) || pmd_devmap(vmf.orig_pmd)) {
5154 if (pmd_protnone(vmf.orig_pmd) && vma_is_accessible(vma))
5155 return do_huge_pmd_numa_page(&vmf);
5156
5157 if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) &&
5158 !pmd_write(vmf.orig_pmd)) {
5159 ret = wp_huge_pmd(&vmf);
5160 if (!(ret & VM_FAULT_FALLBACK))
5161 return ret;
5162 } else {
5163 huge_pmd_set_accessed(&vmf);
5164 return 0;
5165 }
5166 }
5167 }
5168
5169 return handle_pte_fault(&vmf);
5170 }
5171
5172 /**
5173 * mm_account_fault - Do page fault accounting
5174 * @mm: mm from which memcg should be extracted. It can be NULL.
5175 * @regs: the pt_regs struct pointer. When set to NULL, will skip accounting
5176 * of perf event counters, but we'll still do the per-task accounting to
5177 * the task who triggered this page fault.
5178 * @address: the faulted address.
5179 * @flags: the fault flags.
5180 * @ret: the fault retcode.
5181 *
5182 * This will take care of most of the page fault accounting. Meanwhile, it
5183 * will also include the PERF_COUNT_SW_PAGE_FAULTS_[MAJ|MIN] perf counter
5184 * updates. However, note that the handling of PERF_COUNT_SW_PAGE_FAULTS should
5185 * still be in per-arch page fault handlers at the entry of page fault.
5186 */
mm_account_fault(struct mm_struct * mm,struct pt_regs * regs,unsigned long address,unsigned int flags,vm_fault_t ret)5187 static inline void mm_account_fault(struct mm_struct *mm, struct pt_regs *regs,
5188 unsigned long address, unsigned int flags,
5189 vm_fault_t ret)
5190 {
5191 bool major;
5192
5193 /* Incomplete faults will be accounted upon completion. */
5194 if (ret & VM_FAULT_RETRY)
5195 return;
5196
5197 /*
5198 * To preserve the behavior of older kernels, PGFAULT counters record
5199 * both successful and failed faults, as opposed to perf counters,
5200 * which ignore failed cases.
5201 */
5202 count_vm_event(PGFAULT);
5203 count_memcg_event_mm(mm, PGFAULT);
5204
5205 /*
5206 * Do not account for unsuccessful faults (e.g. when the address wasn't
5207 * valid). That includes arch_vma_access_permitted() failing before
5208 * reaching here. So this is not a "this many hardware page faults"
5209 * counter. We should use the hw profiling for that.
5210 */
5211 if (ret & VM_FAULT_ERROR)
5212 return;
5213
5214 /*
5215 * We define the fault as a major fault when the final successful fault
5216 * is VM_FAULT_MAJOR, or if it retried (which implies that we couldn't
5217 * handle it immediately previously).
5218 */
5219 major = (ret & VM_FAULT_MAJOR) || (flags & FAULT_FLAG_TRIED);
5220
5221 if (major)
5222 current->maj_flt++;
5223 else
5224 current->min_flt++;
5225
5226 /*
5227 * If the fault is done for GUP, regs will be NULL. We only do the
5228 * accounting for the per thread fault counters who triggered the
5229 * fault, and we skip the perf event updates.
5230 */
5231 if (!regs)
5232 return;
5233
5234 if (major)
5235 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address);
5236 else
5237 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address);
5238 }
5239
5240 #ifdef CONFIG_LRU_GEN
lru_gen_enter_fault(struct vm_area_struct * vma)5241 static void lru_gen_enter_fault(struct vm_area_struct *vma)
5242 {
5243 /* the LRU algorithm only applies to accesses with recency */
5244 current->in_lru_fault = vma_has_recency(vma);
5245 }
5246
lru_gen_exit_fault(void)5247 static void lru_gen_exit_fault(void)
5248 {
5249 current->in_lru_fault = false;
5250 }
5251 #else
lru_gen_enter_fault(struct vm_area_struct * vma)5252 static void lru_gen_enter_fault(struct vm_area_struct *vma)
5253 {
5254 }
5255
lru_gen_exit_fault(void)5256 static void lru_gen_exit_fault(void)
5257 {
5258 }
5259 #endif /* CONFIG_LRU_GEN */
5260
sanitize_fault_flags(struct vm_area_struct * vma,unsigned int * flags)5261 static vm_fault_t sanitize_fault_flags(struct vm_area_struct *vma,
5262 unsigned int *flags)
5263 {
5264 if (unlikely(*flags & FAULT_FLAG_UNSHARE)) {
5265 if (WARN_ON_ONCE(*flags & FAULT_FLAG_WRITE))
5266 return VM_FAULT_SIGSEGV;
5267 /*
5268 * FAULT_FLAG_UNSHARE only applies to COW mappings. Let's
5269 * just treat it like an ordinary read-fault otherwise.
5270 */
5271 if (!is_cow_mapping(vma->vm_flags))
5272 *flags &= ~FAULT_FLAG_UNSHARE;
5273 } else if (*flags & FAULT_FLAG_WRITE) {
5274 /* Write faults on read-only mappings are impossible ... */
5275 if (WARN_ON_ONCE(!(vma->vm_flags & VM_MAYWRITE)))
5276 return VM_FAULT_SIGSEGV;
5277 /* ... and FOLL_FORCE only applies to COW mappings. */
5278 if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE) &&
5279 !is_cow_mapping(vma->vm_flags)))
5280 return VM_FAULT_SIGSEGV;
5281 }
5282 #ifdef CONFIG_PER_VMA_LOCK
5283 /*
5284 * Per-VMA locks can't be used with FAULT_FLAG_RETRY_NOWAIT because of
5285 * the assumption that lock is dropped on VM_FAULT_RETRY.
5286 */
5287 if (WARN_ON_ONCE((*flags &
5288 (FAULT_FLAG_VMA_LOCK | FAULT_FLAG_RETRY_NOWAIT)) ==
5289 (FAULT_FLAG_VMA_LOCK | FAULT_FLAG_RETRY_NOWAIT)))
5290 return VM_FAULT_SIGSEGV;
5291 #endif
5292
5293 return 0;
5294 }
5295
5296 /*
5297 * By the time we get here, we already hold the mm semaphore
5298 *
5299 * The mmap_lock may have been released depending on flags and our
5300 * return value. See filemap_fault() and __folio_lock_or_retry().
5301 */
handle_mm_fault(struct vm_area_struct * vma,unsigned long address,unsigned int flags,struct pt_regs * regs)5302 vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
5303 unsigned int flags, struct pt_regs *regs)
5304 {
5305 /* If the fault handler drops the mmap_lock, vma may be freed */
5306 struct mm_struct *mm = vma->vm_mm;
5307 vm_fault_t ret;
5308
5309 __set_current_state(TASK_RUNNING);
5310
5311 ret = sanitize_fault_flags(vma, &flags);
5312 if (ret)
5313 goto out;
5314
5315 if (!arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE,
5316 flags & FAULT_FLAG_INSTRUCTION,
5317 flags & FAULT_FLAG_REMOTE)) {
5318 ret = VM_FAULT_SIGSEGV;
5319 goto out;
5320 }
5321
5322 /*
5323 * Enable the memcg OOM handling for faults triggered in user
5324 * space. Kernel faults are handled more gracefully.
5325 */
5326 if (flags & FAULT_FLAG_USER)
5327 mem_cgroup_enter_user_fault();
5328
5329 lru_gen_enter_fault(vma);
5330
5331 if (unlikely(is_vm_hugetlb_page(vma)))
5332 ret = hugetlb_fault(vma->vm_mm, vma, address, flags);
5333 else
5334 ret = __handle_mm_fault(vma, address, flags);
5335
5336 lru_gen_exit_fault();
5337
5338 if (flags & FAULT_FLAG_USER) {
5339 mem_cgroup_exit_user_fault();
5340 /*
5341 * The task may have entered a memcg OOM situation but
5342 * if the allocation error was handled gracefully (no
5343 * VM_FAULT_OOM), there is no need to kill anything.
5344 * Just clean up the OOM state peacefully.
5345 */
5346 if (task_in_memcg_oom(current) && !(ret & VM_FAULT_OOM))
5347 mem_cgroup_oom_synchronize(false);
5348 }
5349 out:
5350 mm_account_fault(mm, regs, address, flags, ret);
5351
5352 return ret;
5353 }
5354 EXPORT_SYMBOL_GPL(handle_mm_fault);
5355
5356 #ifdef CONFIG_LOCK_MM_AND_FIND_VMA
5357 #include <linux/extable.h>
5358
get_mmap_lock_carefully(struct mm_struct * mm,struct pt_regs * regs)5359 static inline bool get_mmap_lock_carefully(struct mm_struct *mm, struct pt_regs *regs)
5360 {
5361 if (likely(mmap_read_trylock(mm)))
5362 return true;
5363
5364 if (regs && !user_mode(regs)) {
5365 unsigned long ip = exception_ip(regs);
5366 if (!search_exception_tables(ip))
5367 return false;
5368 }
5369
5370 return !mmap_read_lock_killable(mm);
5371 }
5372
mmap_upgrade_trylock(struct mm_struct * mm)5373 static inline bool mmap_upgrade_trylock(struct mm_struct *mm)
5374 {
5375 /*
5376 * We don't have this operation yet.
5377 *
5378 * It should be easy enough to do: it's basically a
5379 * atomic_long_try_cmpxchg_acquire()
5380 * from RWSEM_READER_BIAS -> RWSEM_WRITER_LOCKED, but
5381 * it also needs the proper lockdep magic etc.
5382 */
5383 return false;
5384 }
5385
upgrade_mmap_lock_carefully(struct mm_struct * mm,struct pt_regs * regs)5386 static inline bool upgrade_mmap_lock_carefully(struct mm_struct *mm, struct pt_regs *regs)
5387 {
5388 mmap_read_unlock(mm);
5389 if (regs && !user_mode(regs)) {
5390 unsigned long ip = exception_ip(regs);
5391 if (!search_exception_tables(ip))
5392 return false;
5393 }
5394 return !mmap_write_lock_killable(mm);
5395 }
5396
5397 /*
5398 * Helper for page fault handling.
5399 *
5400 * This is kind of equivalend to "mmap_read_lock()" followed
5401 * by "find_extend_vma()", except it's a lot more careful about
5402 * the locking (and will drop the lock on failure).
5403 *
5404 * For example, if we have a kernel bug that causes a page
5405 * fault, we don't want to just use mmap_read_lock() to get
5406 * the mm lock, because that would deadlock if the bug were
5407 * to happen while we're holding the mm lock for writing.
5408 *
5409 * So this checks the exception tables on kernel faults in
5410 * order to only do this all for instructions that are actually
5411 * expected to fault.
5412 *
5413 * We can also actually take the mm lock for writing if we
5414 * need to extend the vma, which helps the VM layer a lot.
5415 */
lock_mm_and_find_vma(struct mm_struct * mm,unsigned long addr,struct pt_regs * regs)5416 struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm,
5417 unsigned long addr, struct pt_regs *regs)
5418 {
5419 struct vm_area_struct *vma;
5420
5421 if (!get_mmap_lock_carefully(mm, regs))
5422 return NULL;
5423
5424 vma = find_vma(mm, addr);
5425 if (likely(vma && (vma->vm_start <= addr)))
5426 return vma;
5427
5428 /*
5429 * Well, dang. We might still be successful, but only
5430 * if we can extend a vma to do so.
5431 */
5432 if (!vma || !(vma->vm_flags & VM_GROWSDOWN)) {
5433 mmap_read_unlock(mm);
5434 return NULL;
5435 }
5436
5437 /*
5438 * We can try to upgrade the mmap lock atomically,
5439 * in which case we can continue to use the vma
5440 * we already looked up.
5441 *
5442 * Otherwise we'll have to drop the mmap lock and
5443 * re-take it, and also look up the vma again,
5444 * re-checking it.
5445 */
5446 if (!mmap_upgrade_trylock(mm)) {
5447 if (!upgrade_mmap_lock_carefully(mm, regs))
5448 return NULL;
5449
5450 vma = find_vma(mm, addr);
5451 if (!vma)
5452 goto fail;
5453 if (vma->vm_start <= addr)
5454 goto success;
5455 if (!(vma->vm_flags & VM_GROWSDOWN))
5456 goto fail;
5457 }
5458
5459 if (expand_stack_locked(vma, addr))
5460 goto fail;
5461
5462 success:
5463 mmap_write_downgrade(mm);
5464 return vma;
5465
5466 fail:
5467 mmap_write_unlock(mm);
5468 return NULL;
5469 }
5470 #endif
5471
5472 #ifdef CONFIG_PER_VMA_LOCK
5473 /*
5474 * Lookup and lock a VMA under RCU protection. Returned VMA is guaranteed to be
5475 * stable and not isolated. If the VMA is not found or is being modified the
5476 * function returns NULL.
5477 */
lock_vma_under_rcu(struct mm_struct * mm,unsigned long address)5478 struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm,
5479 unsigned long address)
5480 {
5481 MA_STATE(mas, &mm->mm_mt, address, address);
5482 struct vm_area_struct *vma;
5483
5484 rcu_read_lock();
5485 retry:
5486 vma = mas_walk(&mas);
5487 if (!vma)
5488 goto inval;
5489
5490 if (!vma_start_read(vma))
5491 goto inval;
5492
5493 /*
5494 * find_mergeable_anon_vma uses adjacent vmas which are not locked.
5495 * This check must happen after vma_start_read(); otherwise, a
5496 * concurrent mremap() with MREMAP_DONTUNMAP could dissociate the VMA
5497 * from its anon_vma.
5498 */
5499 if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma))
5500 goto inval_end_read;
5501
5502 /* Check since vm_start/vm_end might change before we lock the VMA */
5503 if (unlikely(address < vma->vm_start || address >= vma->vm_end))
5504 goto inval_end_read;
5505
5506 /* Check if the VMA got isolated after we found it */
5507 if (vma->detached) {
5508 vma_end_read(vma);
5509 count_vm_vma_lock_event(VMA_LOCK_MISS);
5510 /* The area was replaced with another one */
5511 goto retry;
5512 }
5513
5514 rcu_read_unlock();
5515 return vma;
5516
5517 inval_end_read:
5518 vma_end_read(vma);
5519 inval:
5520 rcu_read_unlock();
5521 count_vm_vma_lock_event(VMA_LOCK_ABORT);
5522 return NULL;
5523 }
5524 #endif /* CONFIG_PER_VMA_LOCK */
5525
5526 #ifndef __PAGETABLE_P4D_FOLDED
5527 /*
5528 * Allocate p4d page table.
5529 * We've already handled the fast-path in-line.
5530 */
__p4d_alloc(struct mm_struct * mm,pgd_t * pgd,unsigned long address)5531 int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
5532 {
5533 p4d_t *new = p4d_alloc_one(mm, address);
5534 if (!new)
5535 return -ENOMEM;
5536
5537 spin_lock(&mm->page_table_lock);
5538 if (pgd_present(*pgd)) { /* Another has populated it */
5539 p4d_free(mm, new);
5540 } else {
5541 smp_wmb(); /* See comment in pmd_install() */
5542 pgd_populate(mm, pgd, new);
5543 }
5544 spin_unlock(&mm->page_table_lock);
5545 return 0;
5546 }
5547 #endif /* __PAGETABLE_P4D_FOLDED */
5548
5549 #ifndef __PAGETABLE_PUD_FOLDED
5550 /*
5551 * Allocate page upper directory.
5552 * We've already handled the fast-path in-line.
5553 */
__pud_alloc(struct mm_struct * mm,p4d_t * p4d,unsigned long address)5554 int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address)
5555 {
5556 pud_t *new = pud_alloc_one(mm, address);
5557 if (!new)
5558 return -ENOMEM;
5559
5560 spin_lock(&mm->page_table_lock);
5561 if (!p4d_present(*p4d)) {
5562 mm_inc_nr_puds(mm);
5563 smp_wmb(); /* See comment in pmd_install() */
5564 p4d_populate(mm, p4d, new);
5565 } else /* Another has populated it */
5566 pud_free(mm, new);
5567 spin_unlock(&mm->page_table_lock);
5568 return 0;
5569 }
5570 #endif /* __PAGETABLE_PUD_FOLDED */
5571
5572 #ifndef __PAGETABLE_PMD_FOLDED
5573 /*
5574 * Allocate page middle directory.
5575 * We've already handled the fast-path in-line.
5576 */
__pmd_alloc(struct mm_struct * mm,pud_t * pud,unsigned long address)5577 int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
5578 {
5579 spinlock_t *ptl;
5580 pmd_t *new = pmd_alloc_one(mm, address);
5581 if (!new)
5582 return -ENOMEM;
5583
5584 ptl = pud_lock(mm, pud);
5585 if (!pud_present(*pud)) {
5586 mm_inc_nr_pmds(mm);
5587 smp_wmb(); /* See comment in pmd_install() */
5588 pud_populate(mm, pud, new);
5589 } else { /* Another has populated it */
5590 pmd_free(mm, new);
5591 }
5592 spin_unlock(ptl);
5593 return 0;
5594 }
5595 #endif /* __PAGETABLE_PMD_FOLDED */
5596
5597 /**
5598 * follow_pte - look up PTE at a user virtual address
5599 * @mm: the mm_struct of the target address space
5600 * @address: user virtual address
5601 * @ptepp: location to store found PTE
5602 * @ptlp: location to store the lock for the PTE
5603 *
5604 * On a successful return, the pointer to the PTE is stored in @ptepp;
5605 * the corresponding lock is taken and its location is stored in @ptlp.
5606 * The contents of the PTE are only stable until @ptlp is released;
5607 * any further use, if any, must be protected against invalidation
5608 * with MMU notifiers.
5609 *
5610 * Only IO mappings and raw PFN mappings are allowed. The mmap semaphore
5611 * should be taken for read.
5612 *
5613 * KVM uses this function. While it is arguably less bad than ``follow_pfn``,
5614 * it is not a good general-purpose API.
5615 *
5616 * Return: zero on success, -ve otherwise.
5617 */
follow_pte(struct mm_struct * mm,unsigned long address,pte_t ** ptepp,spinlock_t ** ptlp)5618 int follow_pte(struct mm_struct *mm, unsigned long address,
5619 pte_t **ptepp, spinlock_t **ptlp)
5620 {
5621 pgd_t *pgd;
5622 p4d_t *p4d;
5623 pud_t *pud;
5624 pmd_t *pmd;
5625 pte_t *ptep;
5626
5627 pgd = pgd_offset(mm, address);
5628 if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
5629 goto out;
5630
5631 p4d = p4d_offset(pgd, address);
5632 if (p4d_none(*p4d) || unlikely(p4d_bad(*p4d)))
5633 goto out;
5634
5635 pud = pud_offset(p4d, address);
5636 if (pud_none(*pud) || unlikely(pud_bad(*pud)))
5637 goto out;
5638
5639 pmd = pmd_offset(pud, address);
5640 VM_BUG_ON(pmd_trans_huge(*pmd));
5641
5642 ptep = pte_offset_map_lock(mm, pmd, address, ptlp);
5643 if (!ptep)
5644 goto out;
5645 if (!pte_present(ptep_get(ptep)))
5646 goto unlock;
5647 *ptepp = ptep;
5648 return 0;
5649 unlock:
5650 pte_unmap_unlock(ptep, *ptlp);
5651 out:
5652 return -EINVAL;
5653 }
5654 EXPORT_SYMBOL_GPL(follow_pte);
5655
5656 /**
5657 * follow_pfn - look up PFN at a user virtual address
5658 * @vma: memory mapping
5659 * @address: user virtual address
5660 * @pfn: location to store found PFN
5661 *
5662 * Only IO mappings and raw PFN mappings are allowed.
5663 *
5664 * This function does not allow the caller to read the permissions
5665 * of the PTE. Do not use it.
5666 *
5667 * Return: zero and the pfn at @pfn on success, -ve otherwise.
5668 */
follow_pfn(struct vm_area_struct * vma,unsigned long address,unsigned long * pfn)5669 int follow_pfn(struct vm_area_struct *vma, unsigned long address,
5670 unsigned long *pfn)
5671 {
5672 int ret = -EINVAL;
5673 spinlock_t *ptl;
5674 pte_t *ptep;
5675
5676 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
5677 return ret;
5678
5679 ret = follow_pte(vma->vm_mm, address, &ptep, &ptl);
5680 if (ret)
5681 return ret;
5682 *pfn = pte_pfn(ptep_get(ptep));
5683 pte_unmap_unlock(ptep, ptl);
5684 return 0;
5685 }
5686 EXPORT_SYMBOL(follow_pfn);
5687
5688 #ifdef CONFIG_HAVE_IOREMAP_PROT
follow_phys(struct vm_area_struct * vma,unsigned long address,unsigned int flags,unsigned long * prot,resource_size_t * phys)5689 int follow_phys(struct vm_area_struct *vma,
5690 unsigned long address, unsigned int flags,
5691 unsigned long *prot, resource_size_t *phys)
5692 {
5693 int ret = -EINVAL;
5694 pte_t *ptep, pte;
5695 spinlock_t *ptl;
5696
5697 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
5698 goto out;
5699
5700 if (follow_pte(vma->vm_mm, address, &ptep, &ptl))
5701 goto out;
5702 pte = ptep_get(ptep);
5703
5704 /* Never return PFNs of anon folios in COW mappings. */
5705 if (vm_normal_folio(vma, address, pte))
5706 goto unlock;
5707
5708 if ((flags & FOLL_WRITE) && !pte_write(pte))
5709 goto unlock;
5710
5711 *prot = pgprot_val(pte_pgprot(pte));
5712 *phys = (resource_size_t)pte_pfn(pte) << PAGE_SHIFT;
5713
5714 ret = 0;
5715 unlock:
5716 pte_unmap_unlock(ptep, ptl);
5717 out:
5718 return ret;
5719 }
5720
5721 /**
5722 * generic_access_phys - generic implementation for iomem mmap access
5723 * @vma: the vma to access
5724 * @addr: userspace address, not relative offset within @vma
5725 * @buf: buffer to read/write
5726 * @len: length of transfer
5727 * @write: set to FOLL_WRITE when writing, otherwise reading
5728 *
5729 * This is a generic implementation for &vm_operations_struct.access for an
5730 * iomem mapping. This callback is used by access_process_vm() when the @vma is
5731 * not page based.
5732 */
generic_access_phys(struct vm_area_struct * vma,unsigned long addr,void * buf,int len,int write)5733 int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
5734 void *buf, int len, int write)
5735 {
5736 resource_size_t phys_addr;
5737 unsigned long prot = 0;
5738 void __iomem *maddr;
5739 pte_t *ptep, pte;
5740 spinlock_t *ptl;
5741 int offset = offset_in_page(addr);
5742 int ret = -EINVAL;
5743
5744 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
5745 return -EINVAL;
5746
5747 retry:
5748 if (follow_pte(vma->vm_mm, addr, &ptep, &ptl))
5749 return -EINVAL;
5750 pte = ptep_get(ptep);
5751 pte_unmap_unlock(ptep, ptl);
5752
5753 prot = pgprot_val(pte_pgprot(pte));
5754 phys_addr = (resource_size_t)pte_pfn(pte) << PAGE_SHIFT;
5755
5756 if ((write & FOLL_WRITE) && !pte_write(pte))
5757 return -EINVAL;
5758
5759 maddr = ioremap_prot(phys_addr, PAGE_ALIGN(len + offset), prot);
5760 if (!maddr)
5761 return -ENOMEM;
5762
5763 if (follow_pte(vma->vm_mm, addr, &ptep, &ptl))
5764 goto out_unmap;
5765
5766 if (!pte_same(pte, ptep_get(ptep))) {
5767 pte_unmap_unlock(ptep, ptl);
5768 iounmap(maddr);
5769
5770 goto retry;
5771 }
5772
5773 if (write)
5774 memcpy_toio(maddr + offset, buf, len);
5775 else
5776 memcpy_fromio(buf, maddr + offset, len);
5777 ret = len;
5778 pte_unmap_unlock(ptep, ptl);
5779 out_unmap:
5780 iounmap(maddr);
5781
5782 return ret;
5783 }
5784 EXPORT_SYMBOL_GPL(generic_access_phys);
5785 #endif
5786
5787 /*
5788 * Access another process' address space as given in mm.
5789 */
__access_remote_vm(struct mm_struct * mm,unsigned long addr,void * buf,int len,unsigned int gup_flags)5790 int __access_remote_vm(struct mm_struct *mm, unsigned long addr, void *buf,
5791 int len, unsigned int gup_flags)
5792 {
5793 void *old_buf = buf;
5794 int write = gup_flags & FOLL_WRITE;
5795
5796 if (mmap_read_lock_killable(mm))
5797 return 0;
5798
5799 /* Untag the address before looking up the VMA */
5800 addr = untagged_addr_remote(mm, addr);
5801
5802 /* Avoid triggering the temporary warning in __get_user_pages */
5803 if (!vma_lookup(mm, addr) && !expand_stack(mm, addr))
5804 return 0;
5805
5806 /* ignore errors, just check how much was successfully transferred */
5807 while (len) {
5808 int bytes, offset;
5809 void *maddr;
5810 struct vm_area_struct *vma = NULL;
5811 struct page *page = get_user_page_vma_remote(mm, addr,
5812 gup_flags, &vma);
5813
5814 if (IS_ERR_OR_NULL(page)) {
5815 /* We might need to expand the stack to access it */
5816 vma = vma_lookup(mm, addr);
5817 if (!vma) {
5818 vma = expand_stack(mm, addr);
5819
5820 /* mmap_lock was dropped on failure */
5821 if (!vma)
5822 return buf - old_buf;
5823
5824 /* Try again if stack expansion worked */
5825 continue;
5826 }
5827
5828
5829 /*
5830 * Check if this is a VM_IO | VM_PFNMAP VMA, which
5831 * we can access using slightly different code.
5832 */
5833 bytes = 0;
5834 #ifdef CONFIG_HAVE_IOREMAP_PROT
5835 if (vma->vm_ops && vma->vm_ops->access)
5836 bytes = vma->vm_ops->access(vma, addr, buf,
5837 len, write);
5838 #endif
5839 if (bytes <= 0)
5840 break;
5841 } else {
5842 bytes = len;
5843 offset = addr & (PAGE_SIZE-1);
5844 if (bytes > PAGE_SIZE-offset)
5845 bytes = PAGE_SIZE-offset;
5846
5847 maddr = kmap(page);
5848 if (write) {
5849 copy_to_user_page(vma, page, addr,
5850 maddr + offset, buf, bytes);
5851 set_page_dirty_lock(page);
5852 } else {
5853 copy_from_user_page(vma, page, addr,
5854 buf, maddr + offset, bytes);
5855 }
5856 kunmap(page);
5857 put_page(page);
5858 }
5859 len -= bytes;
5860 buf += bytes;
5861 addr += bytes;
5862 }
5863 mmap_read_unlock(mm);
5864
5865 return buf - old_buf;
5866 }
5867
5868 /**
5869 * access_remote_vm - access another process' address space
5870 * @mm: the mm_struct of the target address space
5871 * @addr: start address to access
5872 * @buf: source or destination buffer
5873 * @len: number of bytes to transfer
5874 * @gup_flags: flags modifying lookup behaviour
5875 *
5876 * The caller must hold a reference on @mm.
5877 *
5878 * Return: number of bytes copied from source to destination.
5879 */
access_remote_vm(struct mm_struct * mm,unsigned long addr,void * buf,int len,unsigned int gup_flags)5880 int access_remote_vm(struct mm_struct *mm, unsigned long addr,
5881 void *buf, int len, unsigned int gup_flags)
5882 {
5883 return __access_remote_vm(mm, addr, buf, len, gup_flags);
5884 }
5885
5886 /*
5887 * Access another process' address space.
5888 * Source/target buffer must be kernel space,
5889 * Do not walk the page table directly, use get_user_pages
5890 */
access_process_vm(struct task_struct * tsk,unsigned long addr,void * buf,int len,unsigned int gup_flags)5891 int access_process_vm(struct task_struct *tsk, unsigned long addr,
5892 void *buf, int len, unsigned int gup_flags)
5893 {
5894 struct mm_struct *mm;
5895 int ret;
5896
5897 mm = get_task_mm(tsk);
5898 if (!mm)
5899 return 0;
5900
5901 ret = __access_remote_vm(mm, addr, buf, len, gup_flags);
5902
5903 mmput(mm);
5904
5905 return ret;
5906 }
5907 EXPORT_SYMBOL_GPL(access_process_vm);
5908
5909 /*
5910 * Print the name of a VMA.
5911 */
print_vma_addr(char * prefix,unsigned long ip)5912 void print_vma_addr(char *prefix, unsigned long ip)
5913 {
5914 struct mm_struct *mm = current->mm;
5915 struct vm_area_struct *vma;
5916
5917 /*
5918 * we might be running from an atomic context so we cannot sleep
5919 */
5920 if (!mmap_read_trylock(mm))
5921 return;
5922
5923 vma = find_vma(mm, ip);
5924 if (vma && vma->vm_file) {
5925 struct file *f = vma->vm_file;
5926 char *buf = (char *)__get_free_page(GFP_NOWAIT);
5927 if (buf) {
5928 char *p;
5929
5930 p = file_path(f, buf, PAGE_SIZE);
5931 if (IS_ERR(p))
5932 p = "?";
5933 printk("%s%s[%lx+%lx]", prefix, kbasename(p),
5934 vma->vm_start,
5935 vma->vm_end - vma->vm_start);
5936 free_page((unsigned long)buf);
5937 }
5938 }
5939 mmap_read_unlock(mm);
5940 }
5941
5942 #if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP)
__might_fault(const char * file,int line)5943 void __might_fault(const char *file, int line)
5944 {
5945 if (pagefault_disabled())
5946 return;
5947 __might_sleep(file, line);
5948 #if defined(CONFIG_DEBUG_ATOMIC_SLEEP)
5949 if (current->mm)
5950 might_lock_read(¤t->mm->mmap_lock);
5951 #endif
5952 }
5953 EXPORT_SYMBOL(__might_fault);
5954 #endif
5955
5956 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
5957 /*
5958 * Process all subpages of the specified huge page with the specified
5959 * operation. The target subpage will be processed last to keep its
5960 * cache lines hot.
5961 */
process_huge_page(unsigned long addr_hint,unsigned int pages_per_huge_page,int (* process_subpage)(unsigned long addr,int idx,void * arg),void * arg)5962 static inline int process_huge_page(
5963 unsigned long addr_hint, unsigned int pages_per_huge_page,
5964 int (*process_subpage)(unsigned long addr, int idx, void *arg),
5965 void *arg)
5966 {
5967 int i, n, base, l, ret;
5968 unsigned long addr = addr_hint &
5969 ~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
5970
5971 /* Process target subpage last to keep its cache lines hot */
5972 might_sleep();
5973 n = (addr_hint - addr) / PAGE_SIZE;
5974 if (2 * n <= pages_per_huge_page) {
5975 /* If target subpage in first half of huge page */
5976 base = 0;
5977 l = n;
5978 /* Process subpages at the end of huge page */
5979 for (i = pages_per_huge_page - 1; i >= 2 * n; i--) {
5980 cond_resched();
5981 ret = process_subpage(addr + i * PAGE_SIZE, i, arg);
5982 if (ret)
5983 return ret;
5984 }
5985 } else {
5986 /* If target subpage in second half of huge page */
5987 base = pages_per_huge_page - 2 * (pages_per_huge_page - n);
5988 l = pages_per_huge_page - n;
5989 /* Process subpages at the begin of huge page */
5990 for (i = 0; i < base; i++) {
5991 cond_resched();
5992 ret = process_subpage(addr + i * PAGE_SIZE, i, arg);
5993 if (ret)
5994 return ret;
5995 }
5996 }
5997 /*
5998 * Process remaining subpages in left-right-left-right pattern
5999 * towards the target subpage
6000 */
6001 for (i = 0; i < l; i++) {
6002 int left_idx = base + i;
6003 int right_idx = base + 2 * l - 1 - i;
6004
6005 cond_resched();
6006 ret = process_subpage(addr + left_idx * PAGE_SIZE, left_idx, arg);
6007 if (ret)
6008 return ret;
6009 cond_resched();
6010 ret = process_subpage(addr + right_idx * PAGE_SIZE, right_idx, arg);
6011 if (ret)
6012 return ret;
6013 }
6014 return 0;
6015 }
6016
clear_gigantic_page(struct page * page,unsigned long addr,unsigned int pages_per_huge_page)6017 static void clear_gigantic_page(struct page *page,
6018 unsigned long addr,
6019 unsigned int pages_per_huge_page)
6020 {
6021 int i;
6022 struct page *p;
6023
6024 might_sleep();
6025 for (i = 0; i < pages_per_huge_page; i++) {
6026 p = nth_page(page, i);
6027 cond_resched();
6028 clear_user_highpage(p, addr + i * PAGE_SIZE);
6029 }
6030 }
6031
clear_subpage(unsigned long addr,int idx,void * arg)6032 static int clear_subpage(unsigned long addr, int idx, void *arg)
6033 {
6034 struct page *page = arg;
6035
6036 clear_user_highpage(page + idx, addr);
6037 return 0;
6038 }
6039
clear_huge_page(struct page * page,unsigned long addr_hint,unsigned int pages_per_huge_page)6040 void clear_huge_page(struct page *page,
6041 unsigned long addr_hint, unsigned int pages_per_huge_page)
6042 {
6043 unsigned long addr = addr_hint &
6044 ~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
6045
6046 if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) {
6047 clear_gigantic_page(page, addr, pages_per_huge_page);
6048 return;
6049 }
6050
6051 process_huge_page(addr_hint, pages_per_huge_page, clear_subpage, page);
6052 }
6053
copy_user_gigantic_page(struct folio * dst,struct folio * src,unsigned long addr,struct vm_area_struct * vma,unsigned int pages_per_huge_page)6054 static int copy_user_gigantic_page(struct folio *dst, struct folio *src,
6055 unsigned long addr,
6056 struct vm_area_struct *vma,
6057 unsigned int pages_per_huge_page)
6058 {
6059 int i;
6060 struct page *dst_page;
6061 struct page *src_page;
6062
6063 for (i = 0; i < pages_per_huge_page; i++) {
6064 dst_page = folio_page(dst, i);
6065 src_page = folio_page(src, i);
6066
6067 cond_resched();
6068 if (copy_mc_user_highpage(dst_page, src_page,
6069 addr + i*PAGE_SIZE, vma)) {
6070 memory_failure_queue(page_to_pfn(src_page), 0);
6071 return -EHWPOISON;
6072 }
6073 }
6074 return 0;
6075 }
6076
6077 struct copy_subpage_arg {
6078 struct page *dst;
6079 struct page *src;
6080 struct vm_area_struct *vma;
6081 };
6082
copy_subpage(unsigned long addr,int idx,void * arg)6083 static int copy_subpage(unsigned long addr, int idx, void *arg)
6084 {
6085 struct copy_subpage_arg *copy_arg = arg;
6086
6087 if (copy_mc_user_highpage(copy_arg->dst + idx, copy_arg->src + idx,
6088 addr, copy_arg->vma)) {
6089 memory_failure_queue(page_to_pfn(copy_arg->src + idx), 0);
6090 return -EHWPOISON;
6091 }
6092 return 0;
6093 }
6094
copy_user_large_folio(struct folio * dst,struct folio * src,unsigned long addr_hint,struct vm_area_struct * vma)6095 int copy_user_large_folio(struct folio *dst, struct folio *src,
6096 unsigned long addr_hint, struct vm_area_struct *vma)
6097 {
6098 unsigned int pages_per_huge_page = folio_nr_pages(dst);
6099 unsigned long addr = addr_hint &
6100 ~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
6101 struct copy_subpage_arg arg = {
6102 .dst = &dst->page,
6103 .src = &src->page,
6104 .vma = vma,
6105 };
6106
6107 if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES))
6108 return copy_user_gigantic_page(dst, src, addr, vma,
6109 pages_per_huge_page);
6110
6111 return process_huge_page(addr_hint, pages_per_huge_page, copy_subpage, &arg);
6112 }
6113
copy_folio_from_user(struct folio * dst_folio,const void __user * usr_src,bool allow_pagefault)6114 long copy_folio_from_user(struct folio *dst_folio,
6115 const void __user *usr_src,
6116 bool allow_pagefault)
6117 {
6118 void *kaddr;
6119 unsigned long i, rc = 0;
6120 unsigned int nr_pages = folio_nr_pages(dst_folio);
6121 unsigned long ret_val = nr_pages * PAGE_SIZE;
6122 struct page *subpage;
6123
6124 for (i = 0; i < nr_pages; i++) {
6125 subpage = folio_page(dst_folio, i);
6126 kaddr = kmap_local_page(subpage);
6127 if (!allow_pagefault)
6128 pagefault_disable();
6129 rc = copy_from_user(kaddr, usr_src + i * PAGE_SIZE, PAGE_SIZE);
6130 if (!allow_pagefault)
6131 pagefault_enable();
6132 kunmap_local(kaddr);
6133
6134 ret_val -= (PAGE_SIZE - rc);
6135 if (rc)
6136 break;
6137
6138 flush_dcache_page(subpage);
6139
6140 cond_resched();
6141 }
6142 return ret_val;
6143 }
6144 #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
6145
6146 #if USE_SPLIT_PTE_PTLOCKS && ALLOC_SPLIT_PTLOCKS
6147
6148 static struct kmem_cache *page_ptl_cachep;
6149
ptlock_cache_init(void)6150 void __init ptlock_cache_init(void)
6151 {
6152 page_ptl_cachep = kmem_cache_create("page->ptl", sizeof(spinlock_t), 0,
6153 SLAB_PANIC, NULL);
6154 }
6155
ptlock_alloc(struct ptdesc * ptdesc)6156 bool ptlock_alloc(struct ptdesc *ptdesc)
6157 {
6158 spinlock_t *ptl;
6159
6160 ptl = kmem_cache_alloc(page_ptl_cachep, GFP_KERNEL);
6161 if (!ptl)
6162 return false;
6163 ptdesc->ptl = ptl;
6164 return true;
6165 }
6166
ptlock_free(struct ptdesc * ptdesc)6167 void ptlock_free(struct ptdesc *ptdesc)
6168 {
6169 kmem_cache_free(page_ptl_cachep, ptdesc->ptl);
6170 }
6171 #endif
6172