xref: /openbmc/linux/mm/memory.c (revision 5f8b7d4b2e9604d03ae06f1a2dd5a1f34c33e533)
1  // SPDX-License-Identifier: GPL-2.0-only
2  /*
3   *  linux/mm/memory.c
4   *
5   *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
6   */
7  
8  /*
9   * demand-loading started 01.12.91 - seems it is high on the list of
10   * things wanted, and it should be easy to implement. - Linus
11   */
12  
13  /*
14   * Ok, demand-loading was easy, shared pages a little bit tricker. Shared
15   * pages started 02.12.91, seems to work. - Linus.
16   *
17   * Tested sharing by executing about 30 /bin/sh: under the old kernel it
18   * would have taken more than the 6M I have free, but it worked well as
19   * far as I could see.
20   *
21   * Also corrected some "invalidate()"s - I wasn't doing enough of them.
22   */
23  
24  /*
25   * Real VM (paging to/from disk) started 18.12.91. Much more work and
26   * thought has to go into this. Oh, well..
27   * 19.12.91  -  works, somewhat. Sometimes I get faults, don't know why.
28   *		Found it. Everything seems to work now.
29   * 20.12.91  -  Ok, making the swap-device changeable like the root.
30   */
31  
32  /*
33   * 05.04.94  -  Multi-page memory management added for v1.1.
34   *              Idea by Alex Bligh (alex@cconcepts.co.uk)
35   *
36   * 16.07.99  -  Support of BIGMEM added by Gerhard Wichert, Siemens AG
37   *		(Gerhard.Wichert@pdb.siemens.de)
38   *
39   * Aug/Sep 2004 Changed to four level page tables (Andi Kleen)
40   */
41  
42  #include <linux/kernel_stat.h>
43  #include <linux/mm.h>
44  #include <linux/mm_inline.h>
45  #include <linux/sched/mm.h>
46  #include <linux/sched/coredump.h>
47  #include <linux/sched/numa_balancing.h>
48  #include <linux/sched/task.h>
49  #include <linux/hugetlb.h>
50  #include <linux/mman.h>
51  #include <linux/swap.h>
52  #include <linux/highmem.h>
53  #include <linux/pagemap.h>
54  #include <linux/memremap.h>
55  #include <linux/kmsan.h>
56  #include <linux/ksm.h>
57  #include <linux/rmap.h>
58  #include <linux/export.h>
59  #include <linux/delayacct.h>
60  #include <linux/init.h>
61  #include <linux/pfn_t.h>
62  #include <linux/writeback.h>
63  #include <linux/memcontrol.h>
64  #include <linux/mmu_notifier.h>
65  #include <linux/swapops.h>
66  #include <linux/elf.h>
67  #include <linux/gfp.h>
68  #include <linux/migrate.h>
69  #include <linux/string.h>
70  #include <linux/memory-tiers.h>
71  #include <linux/debugfs.h>
72  #include <linux/userfaultfd_k.h>
73  #include <linux/dax.h>
74  #include <linux/oom.h>
75  #include <linux/numa.h>
76  #include <linux/perf_event.h>
77  #include <linux/ptrace.h>
78  #include <linux/vmalloc.h>
79  #include <linux/sched/sysctl.h>
80  
81  #include <trace/events/kmem.h>
82  
83  #include <asm/io.h>
84  #include <asm/mmu_context.h>
85  #include <asm/pgalloc.h>
86  #include <linux/uaccess.h>
87  #include <asm/tlb.h>
88  #include <asm/tlbflush.h>
89  
90  #include "pgalloc-track.h"
91  #include "internal.h"
92  #include "swap.h"
93  
94  #if defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) && !defined(CONFIG_COMPILE_TEST)
95  #warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.
96  #endif
97  
98  #ifndef CONFIG_NUMA
99  unsigned long max_mapnr;
100  EXPORT_SYMBOL(max_mapnr);
101  
102  struct page *mem_map;
103  EXPORT_SYMBOL(mem_map);
104  #endif
105  
106  static vm_fault_t do_fault(struct vm_fault *vmf);
107  static vm_fault_t do_anonymous_page(struct vm_fault *vmf);
108  static bool vmf_pte_changed(struct vm_fault *vmf);
109  
110  /*
111   * Return true if the original pte was a uffd-wp pte marker (so the pte was
112   * wr-protected).
113   */
vmf_orig_pte_uffd_wp(struct vm_fault * vmf)114  static bool vmf_orig_pte_uffd_wp(struct vm_fault *vmf)
115  {
116  	if (!(vmf->flags & FAULT_FLAG_ORIG_PTE_VALID))
117  		return false;
118  
119  	return pte_marker_uffd_wp(vmf->orig_pte);
120  }
121  
122  /*
123   * A number of key systems in x86 including ioremap() rely on the assumption
124   * that high_memory defines the upper bound on direct map memory, then end
125   * of ZONE_NORMAL.  Under CONFIG_DISCONTIG this means that max_low_pfn and
126   * highstart_pfn must be the same; there must be no gap between ZONE_NORMAL
127   * and ZONE_HIGHMEM.
128   */
129  void *high_memory;
130  EXPORT_SYMBOL(high_memory);
131  
132  /*
133   * Randomize the address space (stacks, mmaps, brk, etc.).
134   *
135   * ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization,
136   *   as ancient (libc5 based) binaries can segfault. )
137   */
138  int randomize_va_space __read_mostly =
139  #ifdef CONFIG_COMPAT_BRK
140  					1;
141  #else
142  					2;
143  #endif
144  
145  #ifndef arch_wants_old_prefaulted_pte
arch_wants_old_prefaulted_pte(void)146  static inline bool arch_wants_old_prefaulted_pte(void)
147  {
148  	/*
149  	 * Transitioning a PTE from 'old' to 'young' can be expensive on
150  	 * some architectures, even if it's performed in hardware. By
151  	 * default, "false" means prefaulted entries will be 'young'.
152  	 */
153  	return false;
154  }
155  #endif
156  
disable_randmaps(char * s)157  static int __init disable_randmaps(char *s)
158  {
159  	randomize_va_space = 0;
160  	return 1;
161  }
162  __setup("norandmaps", disable_randmaps);
163  
164  unsigned long zero_pfn __read_mostly;
165  EXPORT_SYMBOL(zero_pfn);
166  
167  unsigned long highest_memmap_pfn __read_mostly;
168  
169  /*
170   * CONFIG_MMU architectures set up ZERO_PAGE in their paging_init()
171   */
init_zero_pfn(void)172  static int __init init_zero_pfn(void)
173  {
174  	zero_pfn = page_to_pfn(ZERO_PAGE(0));
175  	return 0;
176  }
177  early_initcall(init_zero_pfn);
178  
mm_trace_rss_stat(struct mm_struct * mm,int member)179  void mm_trace_rss_stat(struct mm_struct *mm, int member)
180  {
181  	trace_rss_stat(mm, member);
182  }
183  
184  /*
185   * Note: this doesn't free the actual pages themselves. That
186   * has been handled earlier when unmapping all the memory regions.
187   */
free_pte_range(struct mmu_gather * tlb,pmd_t * pmd,unsigned long addr)188  static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
189  			   unsigned long addr)
190  {
191  	pgtable_t token = pmd_pgtable(*pmd);
192  	pmd_clear(pmd);
193  	pte_free_tlb(tlb, token, addr);
194  	mm_dec_nr_ptes(tlb->mm);
195  }
196  
free_pmd_range(struct mmu_gather * tlb,pud_t * pud,unsigned long addr,unsigned long end,unsigned long floor,unsigned long ceiling)197  static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
198  				unsigned long addr, unsigned long end,
199  				unsigned long floor, unsigned long ceiling)
200  {
201  	pmd_t *pmd;
202  	unsigned long next;
203  	unsigned long start;
204  
205  	start = addr;
206  	pmd = pmd_offset(pud, addr);
207  	do {
208  		next = pmd_addr_end(addr, end);
209  		if (pmd_none_or_clear_bad(pmd))
210  			continue;
211  		free_pte_range(tlb, pmd, addr);
212  	} while (pmd++, addr = next, addr != end);
213  
214  	start &= PUD_MASK;
215  	if (start < floor)
216  		return;
217  	if (ceiling) {
218  		ceiling &= PUD_MASK;
219  		if (!ceiling)
220  			return;
221  	}
222  	if (end - 1 > ceiling - 1)
223  		return;
224  
225  	pmd = pmd_offset(pud, start);
226  	pud_clear(pud);
227  	pmd_free_tlb(tlb, pmd, start);
228  	mm_dec_nr_pmds(tlb->mm);
229  }
230  
free_pud_range(struct mmu_gather * tlb,p4d_t * p4d,unsigned long addr,unsigned long end,unsigned long floor,unsigned long ceiling)231  static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,
232  				unsigned long addr, unsigned long end,
233  				unsigned long floor, unsigned long ceiling)
234  {
235  	pud_t *pud;
236  	unsigned long next;
237  	unsigned long start;
238  
239  	start = addr;
240  	pud = pud_offset(p4d, addr);
241  	do {
242  		next = pud_addr_end(addr, end);
243  		if (pud_none_or_clear_bad(pud))
244  			continue;
245  		free_pmd_range(tlb, pud, addr, next, floor, ceiling);
246  	} while (pud++, addr = next, addr != end);
247  
248  	start &= P4D_MASK;
249  	if (start < floor)
250  		return;
251  	if (ceiling) {
252  		ceiling &= P4D_MASK;
253  		if (!ceiling)
254  			return;
255  	}
256  	if (end - 1 > ceiling - 1)
257  		return;
258  
259  	pud = pud_offset(p4d, start);
260  	p4d_clear(p4d);
261  	pud_free_tlb(tlb, pud, start);
262  	mm_dec_nr_puds(tlb->mm);
263  }
264  
free_p4d_range(struct mmu_gather * tlb,pgd_t * pgd,unsigned long addr,unsigned long end,unsigned long floor,unsigned long ceiling)265  static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd,
266  				unsigned long addr, unsigned long end,
267  				unsigned long floor, unsigned long ceiling)
268  {
269  	p4d_t *p4d;
270  	unsigned long next;
271  	unsigned long start;
272  
273  	start = addr;
274  	p4d = p4d_offset(pgd, addr);
275  	do {
276  		next = p4d_addr_end(addr, end);
277  		if (p4d_none_or_clear_bad(p4d))
278  			continue;
279  		free_pud_range(tlb, p4d, addr, next, floor, ceiling);
280  	} while (p4d++, addr = next, addr != end);
281  
282  	start &= PGDIR_MASK;
283  	if (start < floor)
284  		return;
285  	if (ceiling) {
286  		ceiling &= PGDIR_MASK;
287  		if (!ceiling)
288  			return;
289  	}
290  	if (end - 1 > ceiling - 1)
291  		return;
292  
293  	p4d = p4d_offset(pgd, start);
294  	pgd_clear(pgd);
295  	p4d_free_tlb(tlb, p4d, start);
296  }
297  
298  /*
299   * This function frees user-level page tables of a process.
300   */
free_pgd_range(struct mmu_gather * tlb,unsigned long addr,unsigned long end,unsigned long floor,unsigned long ceiling)301  void free_pgd_range(struct mmu_gather *tlb,
302  			unsigned long addr, unsigned long end,
303  			unsigned long floor, unsigned long ceiling)
304  {
305  	pgd_t *pgd;
306  	unsigned long next;
307  
308  	/*
309  	 * The next few lines have given us lots of grief...
310  	 *
311  	 * Why are we testing PMD* at this top level?  Because often
312  	 * there will be no work to do at all, and we'd prefer not to
313  	 * go all the way down to the bottom just to discover that.
314  	 *
315  	 * Why all these "- 1"s?  Because 0 represents both the bottom
316  	 * of the address space and the top of it (using -1 for the
317  	 * top wouldn't help much: the masks would do the wrong thing).
318  	 * The rule is that addr 0 and floor 0 refer to the bottom of
319  	 * the address space, but end 0 and ceiling 0 refer to the top
320  	 * Comparisons need to use "end - 1" and "ceiling - 1" (though
321  	 * that end 0 case should be mythical).
322  	 *
323  	 * Wherever addr is brought up or ceiling brought down, we must
324  	 * be careful to reject "the opposite 0" before it confuses the
325  	 * subsequent tests.  But what about where end is brought down
326  	 * by PMD_SIZE below? no, end can't go down to 0 there.
327  	 *
328  	 * Whereas we round start (addr) and ceiling down, by different
329  	 * masks at different levels, in order to test whether a table
330  	 * now has no other vmas using it, so can be freed, we don't
331  	 * bother to round floor or end up - the tests don't need that.
332  	 */
333  
334  	addr &= PMD_MASK;
335  	if (addr < floor) {
336  		addr += PMD_SIZE;
337  		if (!addr)
338  			return;
339  	}
340  	if (ceiling) {
341  		ceiling &= PMD_MASK;
342  		if (!ceiling)
343  			return;
344  	}
345  	if (end - 1 > ceiling - 1)
346  		end -= PMD_SIZE;
347  	if (addr > end - 1)
348  		return;
349  	/*
350  	 * We add page table cache pages with PAGE_SIZE,
351  	 * (see pte_free_tlb()), flush the tlb if we need
352  	 */
353  	tlb_change_page_size(tlb, PAGE_SIZE);
354  	pgd = pgd_offset(tlb->mm, addr);
355  	do {
356  		next = pgd_addr_end(addr, end);
357  		if (pgd_none_or_clear_bad(pgd))
358  			continue;
359  		free_p4d_range(tlb, pgd, addr, next, floor, ceiling);
360  	} while (pgd++, addr = next, addr != end);
361  }
362  
free_pgtables(struct mmu_gather * tlb,struct ma_state * mas,struct vm_area_struct * vma,unsigned long floor,unsigned long ceiling,bool mm_wr_locked)363  void free_pgtables(struct mmu_gather *tlb, struct ma_state *mas,
364  		   struct vm_area_struct *vma, unsigned long floor,
365  		   unsigned long ceiling, bool mm_wr_locked)
366  {
367  	do {
368  		unsigned long addr = vma->vm_start;
369  		struct vm_area_struct *next;
370  
371  		/*
372  		 * Note: USER_PGTABLES_CEILING may be passed as ceiling and may
373  		 * be 0.  This will underflow and is okay.
374  		 */
375  		next = mas_find(mas, ceiling - 1);
376  
377  		/*
378  		 * Hide vma from rmap and truncate_pagecache before freeing
379  		 * pgtables
380  		 */
381  		if (mm_wr_locked)
382  			vma_start_write(vma);
383  		unlink_anon_vmas(vma);
384  		unlink_file_vma(vma);
385  
386  		if (is_vm_hugetlb_page(vma)) {
387  			hugetlb_free_pgd_range(tlb, addr, vma->vm_end,
388  				floor, next ? next->vm_start : ceiling);
389  		} else {
390  			/*
391  			 * Optimization: gather nearby vmas into one call down
392  			 */
393  			while (next && next->vm_start <= vma->vm_end + PMD_SIZE
394  			       && !is_vm_hugetlb_page(next)) {
395  				vma = next;
396  				next = mas_find(mas, ceiling - 1);
397  				if (mm_wr_locked)
398  					vma_start_write(vma);
399  				unlink_anon_vmas(vma);
400  				unlink_file_vma(vma);
401  			}
402  			free_pgd_range(tlb, addr, vma->vm_end,
403  				floor, next ? next->vm_start : ceiling);
404  		}
405  		vma = next;
406  	} while (vma);
407  }
408  
pmd_install(struct mm_struct * mm,pmd_t * pmd,pgtable_t * pte)409  void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte)
410  {
411  	spinlock_t *ptl = pmd_lock(mm, pmd);
412  
413  	if (likely(pmd_none(*pmd))) {	/* Has another populated it ? */
414  		mm_inc_nr_ptes(mm);
415  		/*
416  		 * Ensure all pte setup (eg. pte page lock and page clearing) are
417  		 * visible before the pte is made visible to other CPUs by being
418  		 * put into page tables.
419  		 *
420  		 * The other side of the story is the pointer chasing in the page
421  		 * table walking code (when walking the page table without locking;
422  		 * ie. most of the time). Fortunately, these data accesses consist
423  		 * of a chain of data-dependent loads, meaning most CPUs (alpha
424  		 * being the notable exception) will already guarantee loads are
425  		 * seen in-order. See the alpha page table accessors for the
426  		 * smp_rmb() barriers in page table walking code.
427  		 */
428  		smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */
429  		pmd_populate(mm, pmd, *pte);
430  		*pte = NULL;
431  	}
432  	spin_unlock(ptl);
433  }
434  
__pte_alloc(struct mm_struct * mm,pmd_t * pmd)435  int __pte_alloc(struct mm_struct *mm, pmd_t *pmd)
436  {
437  	pgtable_t new = pte_alloc_one(mm);
438  	if (!new)
439  		return -ENOMEM;
440  
441  	pmd_install(mm, pmd, &new);
442  	if (new)
443  		pte_free(mm, new);
444  	return 0;
445  }
446  
__pte_alloc_kernel(pmd_t * pmd)447  int __pte_alloc_kernel(pmd_t *pmd)
448  {
449  	pte_t *new = pte_alloc_one_kernel(&init_mm);
450  	if (!new)
451  		return -ENOMEM;
452  
453  	spin_lock(&init_mm.page_table_lock);
454  	if (likely(pmd_none(*pmd))) {	/* Has another populated it ? */
455  		smp_wmb(); /* See comment in pmd_install() */
456  		pmd_populate_kernel(&init_mm, pmd, new);
457  		new = NULL;
458  	}
459  	spin_unlock(&init_mm.page_table_lock);
460  	if (new)
461  		pte_free_kernel(&init_mm, new);
462  	return 0;
463  }
464  
init_rss_vec(int * rss)465  static inline void init_rss_vec(int *rss)
466  {
467  	memset(rss, 0, sizeof(int) * NR_MM_COUNTERS);
468  }
469  
add_mm_rss_vec(struct mm_struct * mm,int * rss)470  static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss)
471  {
472  	int i;
473  
474  	if (current->mm == mm)
475  		sync_mm_rss(mm);
476  	for (i = 0; i < NR_MM_COUNTERS; i++)
477  		if (rss[i])
478  			add_mm_counter(mm, i, rss[i]);
479  }
480  
481  /*
482   * This function is called to print an error when a bad pte
483   * is found. For example, we might have a PFN-mapped pte in
484   * a region that doesn't allow it.
485   *
486   * The calling function must still handle the error.
487   */
print_bad_pte(struct vm_area_struct * vma,unsigned long addr,pte_t pte,struct page * page)488  static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr,
489  			  pte_t pte, struct page *page)
490  {
491  	pgd_t *pgd = pgd_offset(vma->vm_mm, addr);
492  	p4d_t *p4d = p4d_offset(pgd, addr);
493  	pud_t *pud = pud_offset(p4d, addr);
494  	pmd_t *pmd = pmd_offset(pud, addr);
495  	struct address_space *mapping;
496  	pgoff_t index;
497  	static unsigned long resume;
498  	static unsigned long nr_shown;
499  	static unsigned long nr_unshown;
500  
501  	/*
502  	 * Allow a burst of 60 reports, then keep quiet for that minute;
503  	 * or allow a steady drip of one report per second.
504  	 */
505  	if (nr_shown == 60) {
506  		if (time_before(jiffies, resume)) {
507  			nr_unshown++;
508  			return;
509  		}
510  		if (nr_unshown) {
511  			pr_alert("BUG: Bad page map: %lu messages suppressed\n",
512  				 nr_unshown);
513  			nr_unshown = 0;
514  		}
515  		nr_shown = 0;
516  	}
517  	if (nr_shown++ == 0)
518  		resume = jiffies + 60 * HZ;
519  
520  	mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL;
521  	index = linear_page_index(vma, addr);
522  
523  	pr_alert("BUG: Bad page map in process %s  pte:%08llx pmd:%08llx\n",
524  		 current->comm,
525  		 (long long)pte_val(pte), (long long)pmd_val(*pmd));
526  	if (page)
527  		dump_page(page, "bad pte");
528  	pr_alert("addr:%px vm_flags:%08lx anon_vma:%px mapping:%px index:%lx\n",
529  		 (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index);
530  	pr_alert("file:%pD fault:%ps mmap:%ps read_folio:%ps\n",
531  		 vma->vm_file,
532  		 vma->vm_ops ? vma->vm_ops->fault : NULL,
533  		 vma->vm_file ? vma->vm_file->f_op->mmap : NULL,
534  		 mapping ? mapping->a_ops->read_folio : NULL);
535  	dump_stack();
536  	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
537  }
538  
539  /*
540   * vm_normal_page -- This function gets the "struct page" associated with a pte.
541   *
542   * "Special" mappings do not wish to be associated with a "struct page" (either
543   * it doesn't exist, or it exists but they don't want to touch it). In this
544   * case, NULL is returned here. "Normal" mappings do have a struct page.
545   *
546   * There are 2 broad cases. Firstly, an architecture may define a pte_special()
547   * pte bit, in which case this function is trivial. Secondly, an architecture
548   * may not have a spare pte bit, which requires a more complicated scheme,
549   * described below.
550   *
551   * A raw VM_PFNMAP mapping (ie. one that is not COWed) is always considered a
552   * special mapping (even if there are underlying and valid "struct pages").
553   * COWed pages of a VM_PFNMAP are always normal.
554   *
555   * The way we recognize COWed pages within VM_PFNMAP mappings is through the
556   * rules set up by "remap_pfn_range()": the vma will have the VM_PFNMAP bit
557   * set, and the vm_pgoff will point to the first PFN mapped: thus every special
558   * mapping will always honor the rule
559   *
560   *	pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT)
561   *
562   * And for normal mappings this is false.
563   *
564   * This restricts such mappings to be a linear translation from virtual address
565   * to pfn. To get around this restriction, we allow arbitrary mappings so long
566   * as the vma is not a COW mapping; in that case, we know that all ptes are
567   * special (because none can have been COWed).
568   *
569   *
570   * In order to support COW of arbitrary special mappings, we have VM_MIXEDMAP.
571   *
572   * VM_MIXEDMAP mappings can likewise contain memory with or without "struct
573   * page" backing, however the difference is that _all_ pages with a struct
574   * page (that is, those where pfn_valid is true) are refcounted and considered
575   * normal pages by the VM. The disadvantage is that pages are refcounted
576   * (which can be slower and simply not an option for some PFNMAP users). The
577   * advantage is that we don't have to follow the strict linearity rule of
578   * PFNMAP mappings in order to support COWable mappings.
579   *
580   */
vm_normal_page(struct vm_area_struct * vma,unsigned long addr,pte_t pte)581  struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
582  			    pte_t pte)
583  {
584  	unsigned long pfn = pte_pfn(pte);
585  
586  	if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL)) {
587  		if (likely(!pte_special(pte)))
588  			goto check_pfn;
589  		if (vma->vm_ops && vma->vm_ops->find_special_page)
590  			return vma->vm_ops->find_special_page(vma, addr);
591  		if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
592  			return NULL;
593  		if (is_zero_pfn(pfn))
594  			return NULL;
595  		if (pte_devmap(pte))
596  		/*
597  		 * NOTE: New users of ZONE_DEVICE will not set pte_devmap()
598  		 * and will have refcounts incremented on their struct pages
599  		 * when they are inserted into PTEs, thus they are safe to
600  		 * return here. Legacy ZONE_DEVICE pages that set pte_devmap()
601  		 * do not have refcounts. Example of legacy ZONE_DEVICE is
602  		 * MEMORY_DEVICE_FS_DAX type in pmem or virtio_fs drivers.
603  		 */
604  			return NULL;
605  
606  		print_bad_pte(vma, addr, pte, NULL);
607  		return NULL;
608  	}
609  
610  	/* !CONFIG_ARCH_HAS_PTE_SPECIAL case follows: */
611  
612  	if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
613  		if (vma->vm_flags & VM_MIXEDMAP) {
614  			if (!pfn_valid(pfn))
615  				return NULL;
616  			goto out;
617  		} else {
618  			unsigned long off;
619  			off = (addr - vma->vm_start) >> PAGE_SHIFT;
620  			if (pfn == vma->vm_pgoff + off)
621  				return NULL;
622  			if (!is_cow_mapping(vma->vm_flags))
623  				return NULL;
624  		}
625  	}
626  
627  	if (is_zero_pfn(pfn))
628  		return NULL;
629  
630  check_pfn:
631  	if (unlikely(pfn > highest_memmap_pfn)) {
632  		print_bad_pte(vma, addr, pte, NULL);
633  		return NULL;
634  	}
635  
636  	/*
637  	 * NOTE! We still have PageReserved() pages in the page tables.
638  	 * eg. VDSO mappings can cause them to exist.
639  	 */
640  out:
641  	return pfn_to_page(pfn);
642  }
643  
vm_normal_folio(struct vm_area_struct * vma,unsigned long addr,pte_t pte)644  struct folio *vm_normal_folio(struct vm_area_struct *vma, unsigned long addr,
645  			    pte_t pte)
646  {
647  	struct page *page = vm_normal_page(vma, addr, pte);
648  
649  	if (page)
650  		return page_folio(page);
651  	return NULL;
652  }
653  
654  #ifdef CONFIG_TRANSPARENT_HUGEPAGE
vm_normal_page_pmd(struct vm_area_struct * vma,unsigned long addr,pmd_t pmd)655  struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
656  				pmd_t pmd)
657  {
658  	unsigned long pfn = pmd_pfn(pmd);
659  
660  	/*
661  	 * There is no pmd_special() but there may be special pmds, e.g.
662  	 * in a direct-access (dax) mapping, so let's just replicate the
663  	 * !CONFIG_ARCH_HAS_PTE_SPECIAL case from vm_normal_page() here.
664  	 */
665  	if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
666  		if (vma->vm_flags & VM_MIXEDMAP) {
667  			if (!pfn_valid(pfn))
668  				return NULL;
669  			goto out;
670  		} else {
671  			unsigned long off;
672  			off = (addr - vma->vm_start) >> PAGE_SHIFT;
673  			if (pfn == vma->vm_pgoff + off)
674  				return NULL;
675  			if (!is_cow_mapping(vma->vm_flags))
676  				return NULL;
677  		}
678  	}
679  
680  	if (pmd_devmap(pmd))
681  		return NULL;
682  	if (is_huge_zero_pmd(pmd))
683  		return NULL;
684  	if (unlikely(pfn > highest_memmap_pfn))
685  		return NULL;
686  
687  	/*
688  	 * NOTE! We still have PageReserved() pages in the page tables.
689  	 * eg. VDSO mappings can cause them to exist.
690  	 */
691  out:
692  	return pfn_to_page(pfn);
693  }
694  #endif
695  
restore_exclusive_pte(struct vm_area_struct * vma,struct page * page,unsigned long address,pte_t * ptep)696  static void restore_exclusive_pte(struct vm_area_struct *vma,
697  				  struct page *page, unsigned long address,
698  				  pte_t *ptep)
699  {
700  	pte_t orig_pte;
701  	pte_t pte;
702  	swp_entry_t entry;
703  
704  	orig_pte = ptep_get(ptep);
705  	pte = pte_mkold(mk_pte(page, READ_ONCE(vma->vm_page_prot)));
706  	if (pte_swp_soft_dirty(orig_pte))
707  		pte = pte_mksoft_dirty(pte);
708  
709  	entry = pte_to_swp_entry(orig_pte);
710  	if (pte_swp_uffd_wp(orig_pte))
711  		pte = pte_mkuffd_wp(pte);
712  	else if (is_writable_device_exclusive_entry(entry))
713  		pte = maybe_mkwrite(pte_mkdirty(pte), vma);
714  
715  	VM_BUG_ON(pte_write(pte) && !(PageAnon(page) && PageAnonExclusive(page)));
716  
717  	/*
718  	 * No need to take a page reference as one was already
719  	 * created when the swap entry was made.
720  	 */
721  	if (PageAnon(page))
722  		page_add_anon_rmap(page, vma, address, RMAP_NONE);
723  	else
724  		/*
725  		 * Currently device exclusive access only supports anonymous
726  		 * memory so the entry shouldn't point to a filebacked page.
727  		 */
728  		WARN_ON_ONCE(1);
729  
730  	set_pte_at(vma->vm_mm, address, ptep, pte);
731  
732  	/*
733  	 * No need to invalidate - it was non-present before. However
734  	 * secondary CPUs may have mappings that need invalidating.
735  	 */
736  	update_mmu_cache(vma, address, ptep);
737  }
738  
739  /*
740   * Tries to restore an exclusive pte if the page lock can be acquired without
741   * sleeping.
742   */
743  static int
try_restore_exclusive_pte(pte_t * src_pte,struct vm_area_struct * vma,unsigned long addr)744  try_restore_exclusive_pte(pte_t *src_pte, struct vm_area_struct *vma,
745  			unsigned long addr)
746  {
747  	swp_entry_t entry = pte_to_swp_entry(ptep_get(src_pte));
748  	struct page *page = pfn_swap_entry_to_page(entry);
749  
750  	if (trylock_page(page)) {
751  		restore_exclusive_pte(vma, page, addr, src_pte);
752  		unlock_page(page);
753  		return 0;
754  	}
755  
756  	return -EBUSY;
757  }
758  
759  /*
760   * copy one vm_area from one task to the other. Assumes the page tables
761   * already present in the new task to be cleared in the whole range
762   * covered by this vma.
763   */
764  
765  static unsigned long
copy_nonpresent_pte(struct mm_struct * dst_mm,struct mm_struct * src_mm,pte_t * dst_pte,pte_t * src_pte,struct vm_area_struct * dst_vma,struct vm_area_struct * src_vma,unsigned long addr,int * rss)766  copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
767  		pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *dst_vma,
768  		struct vm_area_struct *src_vma, unsigned long addr, int *rss)
769  {
770  	unsigned long vm_flags = dst_vma->vm_flags;
771  	pte_t orig_pte = ptep_get(src_pte);
772  	pte_t pte = orig_pte;
773  	struct page *page;
774  	swp_entry_t entry = pte_to_swp_entry(orig_pte);
775  
776  	if (likely(!non_swap_entry(entry))) {
777  		if (swap_duplicate(entry) < 0)
778  			return -EIO;
779  
780  		/* make sure dst_mm is on swapoff's mmlist. */
781  		if (unlikely(list_empty(&dst_mm->mmlist))) {
782  			spin_lock(&mmlist_lock);
783  			if (list_empty(&dst_mm->mmlist))
784  				list_add(&dst_mm->mmlist,
785  						&src_mm->mmlist);
786  			spin_unlock(&mmlist_lock);
787  		}
788  		/* Mark the swap entry as shared. */
789  		if (pte_swp_exclusive(orig_pte)) {
790  			pte = pte_swp_clear_exclusive(orig_pte);
791  			set_pte_at(src_mm, addr, src_pte, pte);
792  		}
793  		rss[MM_SWAPENTS]++;
794  	} else if (is_migration_entry(entry)) {
795  		page = pfn_swap_entry_to_page(entry);
796  
797  		rss[mm_counter(page)]++;
798  
799  		if (!is_readable_migration_entry(entry) &&
800  				is_cow_mapping(vm_flags)) {
801  			/*
802  			 * COW mappings require pages in both parent and child
803  			 * to be set to read. A previously exclusive entry is
804  			 * now shared.
805  			 */
806  			entry = make_readable_migration_entry(
807  							swp_offset(entry));
808  			pte = swp_entry_to_pte(entry);
809  			if (pte_swp_soft_dirty(orig_pte))
810  				pte = pte_swp_mksoft_dirty(pte);
811  			if (pte_swp_uffd_wp(orig_pte))
812  				pte = pte_swp_mkuffd_wp(pte);
813  			set_pte_at(src_mm, addr, src_pte, pte);
814  		}
815  	} else if (is_device_private_entry(entry)) {
816  		page = pfn_swap_entry_to_page(entry);
817  
818  		/*
819  		 * Update rss count even for unaddressable pages, as
820  		 * they should treated just like normal pages in this
821  		 * respect.
822  		 *
823  		 * We will likely want to have some new rss counters
824  		 * for unaddressable pages, at some point. But for now
825  		 * keep things as they are.
826  		 */
827  		get_page(page);
828  		rss[mm_counter(page)]++;
829  		/* Cannot fail as these pages cannot get pinned. */
830  		BUG_ON(page_try_dup_anon_rmap(page, false, src_vma));
831  
832  		/*
833  		 * We do not preserve soft-dirty information, because so
834  		 * far, checkpoint/restore is the only feature that
835  		 * requires that. And checkpoint/restore does not work
836  		 * when a device driver is involved (you cannot easily
837  		 * save and restore device driver state).
838  		 */
839  		if (is_writable_device_private_entry(entry) &&
840  		    is_cow_mapping(vm_flags)) {
841  			entry = make_readable_device_private_entry(
842  							swp_offset(entry));
843  			pte = swp_entry_to_pte(entry);
844  			if (pte_swp_uffd_wp(orig_pte))
845  				pte = pte_swp_mkuffd_wp(pte);
846  			set_pte_at(src_mm, addr, src_pte, pte);
847  		}
848  	} else if (is_device_exclusive_entry(entry)) {
849  		/*
850  		 * Make device exclusive entries present by restoring the
851  		 * original entry then copying as for a present pte. Device
852  		 * exclusive entries currently only support private writable
853  		 * (ie. COW) mappings.
854  		 */
855  		VM_BUG_ON(!is_cow_mapping(src_vma->vm_flags));
856  		if (try_restore_exclusive_pte(src_pte, src_vma, addr))
857  			return -EBUSY;
858  		return -ENOENT;
859  	} else if (is_pte_marker_entry(entry)) {
860  		pte_marker marker = copy_pte_marker(entry, dst_vma);
861  
862  		if (marker)
863  			set_pte_at(dst_mm, addr, dst_pte,
864  				   make_pte_marker(marker));
865  		return 0;
866  	}
867  	if (!userfaultfd_wp(dst_vma))
868  		pte = pte_swp_clear_uffd_wp(pte);
869  	set_pte_at(dst_mm, addr, dst_pte, pte);
870  	return 0;
871  }
872  
873  /*
874   * Copy a present and normal page.
875   *
876   * NOTE! The usual case is that this isn't required;
877   * instead, the caller can just increase the page refcount
878   * and re-use the pte the traditional way.
879   *
880   * And if we need a pre-allocated page but don't yet have
881   * one, return a negative error to let the preallocation
882   * code know so that it can do so outside the page table
883   * lock.
884   */
885  static inline int
copy_present_page(struct vm_area_struct * dst_vma,struct vm_area_struct * src_vma,pte_t * dst_pte,pte_t * src_pte,unsigned long addr,int * rss,struct folio ** prealloc,struct page * page)886  copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
887  		  pte_t *dst_pte, pte_t *src_pte, unsigned long addr, int *rss,
888  		  struct folio **prealloc, struct page *page)
889  {
890  	struct folio *new_folio;
891  	pte_t pte;
892  
893  	new_folio = *prealloc;
894  	if (!new_folio)
895  		return -EAGAIN;
896  
897  	/*
898  	 * We have a prealloc page, all good!  Take it
899  	 * over and copy the page & arm it.
900  	 */
901  	*prealloc = NULL;
902  	copy_user_highpage(&new_folio->page, page, addr, src_vma);
903  	__folio_mark_uptodate(new_folio);
904  	folio_add_new_anon_rmap(new_folio, dst_vma, addr);
905  	folio_add_lru_vma(new_folio, dst_vma);
906  	rss[MM_ANONPAGES]++;
907  
908  	/* All done, just insert the new page copy in the child */
909  	pte = mk_pte(&new_folio->page, dst_vma->vm_page_prot);
910  	pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma);
911  	if (userfaultfd_pte_wp(dst_vma, ptep_get(src_pte)))
912  		/* Uffd-wp needs to be delivered to dest pte as well */
913  		pte = pte_mkuffd_wp(pte);
914  	set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte);
915  	return 0;
916  }
917  
918  /*
919   * Copy one pte.  Returns 0 if succeeded, or -EAGAIN if one preallocated page
920   * is required to copy this pte.
921   */
922  static inline int
copy_present_pte(struct vm_area_struct * dst_vma,struct vm_area_struct * src_vma,pte_t * dst_pte,pte_t * src_pte,unsigned long addr,int * rss,struct folio ** prealloc)923  copy_present_pte(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
924  		 pte_t *dst_pte, pte_t *src_pte, unsigned long addr, int *rss,
925  		 struct folio **prealloc)
926  {
927  	struct mm_struct *src_mm = src_vma->vm_mm;
928  	unsigned long vm_flags = src_vma->vm_flags;
929  	pte_t pte = ptep_get(src_pte);
930  	struct page *page;
931  	struct folio *folio;
932  
933  	page = vm_normal_page(src_vma, addr, pte);
934  	if (page)
935  		folio = page_folio(page);
936  	if (page && folio_test_anon(folio)) {
937  		/*
938  		 * If this page may have been pinned by the parent process,
939  		 * copy the page immediately for the child so that we'll always
940  		 * guarantee the pinned page won't be randomly replaced in the
941  		 * future.
942  		 */
943  		folio_get(folio);
944  		if (unlikely(page_try_dup_anon_rmap(page, false, src_vma))) {
945  			/* Page may be pinned, we have to copy. */
946  			folio_put(folio);
947  			return copy_present_page(dst_vma, src_vma, dst_pte, src_pte,
948  						 addr, rss, prealloc, page);
949  		}
950  		rss[MM_ANONPAGES]++;
951  	} else if (page) {
952  		folio_get(folio);
953  		page_dup_file_rmap(page, false);
954  		rss[mm_counter_file(page)]++;
955  	}
956  
957  	/*
958  	 * If it's a COW mapping, write protect it both
959  	 * in the parent and the child
960  	 */
961  	if (is_cow_mapping(vm_flags) && pte_write(pte)) {
962  		ptep_set_wrprotect(src_mm, addr, src_pte);
963  		pte = pte_wrprotect(pte);
964  	}
965  	VM_BUG_ON(page && folio_test_anon(folio) && PageAnonExclusive(page));
966  
967  	/*
968  	 * If it's a shared mapping, mark it clean in
969  	 * the child
970  	 */
971  	if (vm_flags & VM_SHARED)
972  		pte = pte_mkclean(pte);
973  	pte = pte_mkold(pte);
974  
975  	if (!userfaultfd_wp(dst_vma))
976  		pte = pte_clear_uffd_wp(pte);
977  
978  	set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte);
979  	return 0;
980  }
981  
page_copy_prealloc(struct mm_struct * src_mm,struct vm_area_struct * vma,unsigned long addr)982  static inline struct folio *page_copy_prealloc(struct mm_struct *src_mm,
983  		struct vm_area_struct *vma, unsigned long addr)
984  {
985  	struct folio *new_folio;
986  
987  	new_folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma, addr, false);
988  	if (!new_folio)
989  		return NULL;
990  
991  	if (mem_cgroup_charge(new_folio, src_mm, GFP_KERNEL)) {
992  		folio_put(new_folio);
993  		return NULL;
994  	}
995  	folio_throttle_swaprate(new_folio, GFP_KERNEL);
996  
997  	return new_folio;
998  }
999  
1000  static int
copy_pte_range(struct vm_area_struct * dst_vma,struct vm_area_struct * src_vma,pmd_t * dst_pmd,pmd_t * src_pmd,unsigned long addr,unsigned long end)1001  copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1002  	       pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
1003  	       unsigned long end)
1004  {
1005  	struct mm_struct *dst_mm = dst_vma->vm_mm;
1006  	struct mm_struct *src_mm = src_vma->vm_mm;
1007  	pte_t *orig_src_pte, *orig_dst_pte;
1008  	pte_t *src_pte, *dst_pte;
1009  	pte_t ptent;
1010  	spinlock_t *src_ptl, *dst_ptl;
1011  	int progress, ret = 0;
1012  	int rss[NR_MM_COUNTERS];
1013  	swp_entry_t entry = (swp_entry_t){0};
1014  	struct folio *prealloc = NULL;
1015  
1016  again:
1017  	progress = 0;
1018  	init_rss_vec(rss);
1019  
1020  	/*
1021  	 * copy_pmd_range()'s prior pmd_none_or_clear_bad(src_pmd), and the
1022  	 * error handling here, assume that exclusive mmap_lock on dst and src
1023  	 * protects anon from unexpected THP transitions; with shmem and file
1024  	 * protected by mmap_lock-less collapse skipping areas with anon_vma
1025  	 * (whereas vma_needs_copy() skips areas without anon_vma).  A rework
1026  	 * can remove such assumptions later, but this is good enough for now.
1027  	 */
1028  	dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl);
1029  	if (!dst_pte) {
1030  		ret = -ENOMEM;
1031  		goto out;
1032  	}
1033  	src_pte = pte_offset_map_nolock(src_mm, src_pmd, addr, &src_ptl);
1034  	if (!src_pte) {
1035  		pte_unmap_unlock(dst_pte, dst_ptl);
1036  		/* ret == 0 */
1037  		goto out;
1038  	}
1039  	spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
1040  	orig_src_pte = src_pte;
1041  	orig_dst_pte = dst_pte;
1042  	arch_enter_lazy_mmu_mode();
1043  
1044  	do {
1045  		/*
1046  		 * We are holding two locks at this point - either of them
1047  		 * could generate latencies in another task on another CPU.
1048  		 */
1049  		if (progress >= 32) {
1050  			progress = 0;
1051  			if (need_resched() ||
1052  			    spin_needbreak(src_ptl) || spin_needbreak(dst_ptl))
1053  				break;
1054  		}
1055  		ptent = ptep_get(src_pte);
1056  		if (pte_none(ptent)) {
1057  			progress++;
1058  			continue;
1059  		}
1060  		if (unlikely(!pte_present(ptent))) {
1061  			ret = copy_nonpresent_pte(dst_mm, src_mm,
1062  						  dst_pte, src_pte,
1063  						  dst_vma, src_vma,
1064  						  addr, rss);
1065  			if (ret == -EIO) {
1066  				entry = pte_to_swp_entry(ptep_get(src_pte));
1067  				break;
1068  			} else if (ret == -EBUSY) {
1069  				break;
1070  			} else if (!ret) {
1071  				progress += 8;
1072  				continue;
1073  			}
1074  
1075  			/*
1076  			 * Device exclusive entry restored, continue by copying
1077  			 * the now present pte.
1078  			 */
1079  			WARN_ON_ONCE(ret != -ENOENT);
1080  		}
1081  		/* copy_present_pte() will clear `*prealloc' if consumed */
1082  		ret = copy_present_pte(dst_vma, src_vma, dst_pte, src_pte,
1083  				       addr, rss, &prealloc);
1084  		/*
1085  		 * If we need a pre-allocated page for this pte, drop the
1086  		 * locks, allocate, and try again.
1087  		 */
1088  		if (unlikely(ret == -EAGAIN))
1089  			break;
1090  		if (unlikely(prealloc)) {
1091  			/*
1092  			 * pre-alloc page cannot be reused by next time so as
1093  			 * to strictly follow mempolicy (e.g., alloc_page_vma()
1094  			 * will allocate page according to address).  This
1095  			 * could only happen if one pinned pte changed.
1096  			 */
1097  			folio_put(prealloc);
1098  			prealloc = NULL;
1099  		}
1100  		progress += 8;
1101  	} while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end);
1102  
1103  	arch_leave_lazy_mmu_mode();
1104  	pte_unmap_unlock(orig_src_pte, src_ptl);
1105  	add_mm_rss_vec(dst_mm, rss);
1106  	pte_unmap_unlock(orig_dst_pte, dst_ptl);
1107  	cond_resched();
1108  
1109  	if (ret == -EIO) {
1110  		VM_WARN_ON_ONCE(!entry.val);
1111  		if (add_swap_count_continuation(entry, GFP_KERNEL) < 0) {
1112  			ret = -ENOMEM;
1113  			goto out;
1114  		}
1115  		entry.val = 0;
1116  	} else if (ret == -EBUSY) {
1117  		goto out;
1118  	} else if (ret ==  -EAGAIN) {
1119  		prealloc = page_copy_prealloc(src_mm, src_vma, addr);
1120  		if (!prealloc)
1121  			return -ENOMEM;
1122  	} else if (ret) {
1123  		VM_WARN_ON_ONCE(1);
1124  	}
1125  
1126  	/* We've captured and resolved the error. Reset, try again. */
1127  	ret = 0;
1128  
1129  	if (addr != end)
1130  		goto again;
1131  out:
1132  	if (unlikely(prealloc))
1133  		folio_put(prealloc);
1134  	return ret;
1135  }
1136  
1137  static inline int
copy_pmd_range(struct vm_area_struct * dst_vma,struct vm_area_struct * src_vma,pud_t * dst_pud,pud_t * src_pud,unsigned long addr,unsigned long end)1138  copy_pmd_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1139  	       pud_t *dst_pud, pud_t *src_pud, unsigned long addr,
1140  	       unsigned long end)
1141  {
1142  	struct mm_struct *dst_mm = dst_vma->vm_mm;
1143  	struct mm_struct *src_mm = src_vma->vm_mm;
1144  	pmd_t *src_pmd, *dst_pmd;
1145  	unsigned long next;
1146  
1147  	dst_pmd = pmd_alloc(dst_mm, dst_pud, addr);
1148  	if (!dst_pmd)
1149  		return -ENOMEM;
1150  	src_pmd = pmd_offset(src_pud, addr);
1151  	do {
1152  		next = pmd_addr_end(addr, end);
1153  		if (is_swap_pmd(*src_pmd) || pmd_trans_huge(*src_pmd)
1154  			|| pmd_devmap(*src_pmd)) {
1155  			int err;
1156  			VM_BUG_ON_VMA(next-addr != HPAGE_PMD_SIZE, src_vma);
1157  			err = copy_huge_pmd(dst_mm, src_mm, dst_pmd, src_pmd,
1158  					    addr, dst_vma, src_vma);
1159  			if (err == -ENOMEM)
1160  				return -ENOMEM;
1161  			if (!err)
1162  				continue;
1163  			/* fall through */
1164  		}
1165  		if (pmd_none_or_clear_bad(src_pmd))
1166  			continue;
1167  		if (copy_pte_range(dst_vma, src_vma, dst_pmd, src_pmd,
1168  				   addr, next))
1169  			return -ENOMEM;
1170  	} while (dst_pmd++, src_pmd++, addr = next, addr != end);
1171  	return 0;
1172  }
1173  
1174  static inline int
copy_pud_range(struct vm_area_struct * dst_vma,struct vm_area_struct * src_vma,p4d_t * dst_p4d,p4d_t * src_p4d,unsigned long addr,unsigned long end)1175  copy_pud_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1176  	       p4d_t *dst_p4d, p4d_t *src_p4d, unsigned long addr,
1177  	       unsigned long end)
1178  {
1179  	struct mm_struct *dst_mm = dst_vma->vm_mm;
1180  	struct mm_struct *src_mm = src_vma->vm_mm;
1181  	pud_t *src_pud, *dst_pud;
1182  	unsigned long next;
1183  
1184  	dst_pud = pud_alloc(dst_mm, dst_p4d, addr);
1185  	if (!dst_pud)
1186  		return -ENOMEM;
1187  	src_pud = pud_offset(src_p4d, addr);
1188  	do {
1189  		next = pud_addr_end(addr, end);
1190  		if (pud_trans_huge(*src_pud) || pud_devmap(*src_pud)) {
1191  			int err;
1192  
1193  			VM_BUG_ON_VMA(next-addr != HPAGE_PUD_SIZE, src_vma);
1194  			err = copy_huge_pud(dst_mm, src_mm,
1195  					    dst_pud, src_pud, addr, src_vma);
1196  			if (err == -ENOMEM)
1197  				return -ENOMEM;
1198  			if (!err)
1199  				continue;
1200  			/* fall through */
1201  		}
1202  		if (pud_none_or_clear_bad(src_pud))
1203  			continue;
1204  		if (copy_pmd_range(dst_vma, src_vma, dst_pud, src_pud,
1205  				   addr, next))
1206  			return -ENOMEM;
1207  	} while (dst_pud++, src_pud++, addr = next, addr != end);
1208  	return 0;
1209  }
1210  
1211  static inline int
copy_p4d_range(struct vm_area_struct * dst_vma,struct vm_area_struct * src_vma,pgd_t * dst_pgd,pgd_t * src_pgd,unsigned long addr,unsigned long end)1212  copy_p4d_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1213  	       pgd_t *dst_pgd, pgd_t *src_pgd, unsigned long addr,
1214  	       unsigned long end)
1215  {
1216  	struct mm_struct *dst_mm = dst_vma->vm_mm;
1217  	p4d_t *src_p4d, *dst_p4d;
1218  	unsigned long next;
1219  
1220  	dst_p4d = p4d_alloc(dst_mm, dst_pgd, addr);
1221  	if (!dst_p4d)
1222  		return -ENOMEM;
1223  	src_p4d = p4d_offset(src_pgd, addr);
1224  	do {
1225  		next = p4d_addr_end(addr, end);
1226  		if (p4d_none_or_clear_bad(src_p4d))
1227  			continue;
1228  		if (copy_pud_range(dst_vma, src_vma, dst_p4d, src_p4d,
1229  				   addr, next))
1230  			return -ENOMEM;
1231  	} while (dst_p4d++, src_p4d++, addr = next, addr != end);
1232  	return 0;
1233  }
1234  
1235  /*
1236   * Return true if the vma needs to copy the pgtable during this fork().  Return
1237   * false when we can speed up fork() by allowing lazy page faults later until
1238   * when the child accesses the memory range.
1239   */
1240  static bool
vma_needs_copy(struct vm_area_struct * dst_vma,struct vm_area_struct * src_vma)1241  vma_needs_copy(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
1242  {
1243  	/*
1244  	 * Always copy pgtables when dst_vma has uffd-wp enabled even if it's
1245  	 * file-backed (e.g. shmem). Because when uffd-wp is enabled, pgtable
1246  	 * contains uffd-wp protection information, that's something we can't
1247  	 * retrieve from page cache, and skip copying will lose those info.
1248  	 */
1249  	if (userfaultfd_wp(dst_vma))
1250  		return true;
1251  
1252  	if (src_vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
1253  		return true;
1254  
1255  	if (src_vma->anon_vma)
1256  		return true;
1257  
1258  	/*
1259  	 * Don't copy ptes where a page fault will fill them correctly.  Fork
1260  	 * becomes much lighter when there are big shared or private readonly
1261  	 * mappings. The tradeoff is that copy_page_range is more efficient
1262  	 * than faulting.
1263  	 */
1264  	return false;
1265  }
1266  
1267  int
copy_page_range(struct vm_area_struct * dst_vma,struct vm_area_struct * src_vma)1268  copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
1269  {
1270  	pgd_t *src_pgd, *dst_pgd;
1271  	unsigned long next;
1272  	unsigned long addr = src_vma->vm_start;
1273  	unsigned long end = src_vma->vm_end;
1274  	struct mm_struct *dst_mm = dst_vma->vm_mm;
1275  	struct mm_struct *src_mm = src_vma->vm_mm;
1276  	struct mmu_notifier_range range;
1277  	bool is_cow;
1278  	int ret;
1279  
1280  	if (!vma_needs_copy(dst_vma, src_vma))
1281  		return 0;
1282  
1283  	if (is_vm_hugetlb_page(src_vma))
1284  		return copy_hugetlb_page_range(dst_mm, src_mm, dst_vma, src_vma);
1285  
1286  	if (unlikely(src_vma->vm_flags & VM_PFNMAP)) {
1287  		/*
1288  		 * We do not free on error cases below as remove_vma
1289  		 * gets called on error from higher level routine
1290  		 */
1291  		ret = track_pfn_copy(src_vma);
1292  		if (ret)
1293  			return ret;
1294  	}
1295  
1296  	/*
1297  	 * We need to invalidate the secondary MMU mappings only when
1298  	 * there could be a permission downgrade on the ptes of the
1299  	 * parent mm. And a permission downgrade will only happen if
1300  	 * is_cow_mapping() returns true.
1301  	 */
1302  	is_cow = is_cow_mapping(src_vma->vm_flags);
1303  
1304  	if (is_cow) {
1305  		mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE,
1306  					0, src_mm, addr, end);
1307  		mmu_notifier_invalidate_range_start(&range);
1308  		/*
1309  		 * Disabling preemption is not needed for the write side, as
1310  		 * the read side doesn't spin, but goes to the mmap_lock.
1311  		 *
1312  		 * Use the raw variant of the seqcount_t write API to avoid
1313  		 * lockdep complaining about preemptibility.
1314  		 */
1315  		vma_assert_write_locked(src_vma);
1316  		raw_write_seqcount_begin(&src_mm->write_protect_seq);
1317  	}
1318  
1319  	ret = 0;
1320  	dst_pgd = pgd_offset(dst_mm, addr);
1321  	src_pgd = pgd_offset(src_mm, addr);
1322  	do {
1323  		next = pgd_addr_end(addr, end);
1324  		if (pgd_none_or_clear_bad(src_pgd))
1325  			continue;
1326  		if (unlikely(copy_p4d_range(dst_vma, src_vma, dst_pgd, src_pgd,
1327  					    addr, next))) {
1328  			untrack_pfn_clear(dst_vma);
1329  			ret = -ENOMEM;
1330  			break;
1331  		}
1332  	} while (dst_pgd++, src_pgd++, addr = next, addr != end);
1333  
1334  	if (is_cow) {
1335  		raw_write_seqcount_end(&src_mm->write_protect_seq);
1336  		mmu_notifier_invalidate_range_end(&range);
1337  	}
1338  	return ret;
1339  }
1340  
1341  /* Whether we should zap all COWed (private) pages too */
should_zap_cows(struct zap_details * details)1342  static inline bool should_zap_cows(struct zap_details *details)
1343  {
1344  	/* By default, zap all pages */
1345  	if (!details)
1346  		return true;
1347  
1348  	/* Or, we zap COWed pages only if the caller wants to */
1349  	return details->even_cows;
1350  }
1351  
1352  /* Decides whether we should zap this page with the page pointer specified */
should_zap_page(struct zap_details * details,struct page * page)1353  static inline bool should_zap_page(struct zap_details *details, struct page *page)
1354  {
1355  	/* If we can make a decision without *page.. */
1356  	if (should_zap_cows(details))
1357  		return true;
1358  
1359  	/* E.g. the caller passes NULL for the case of a zero page */
1360  	if (!page)
1361  		return true;
1362  
1363  	/* Otherwise we should only zap non-anon pages */
1364  	return !PageAnon(page);
1365  }
1366  
zap_drop_file_uffd_wp(struct zap_details * details)1367  static inline bool zap_drop_file_uffd_wp(struct zap_details *details)
1368  {
1369  	if (!details)
1370  		return false;
1371  
1372  	return details->zap_flags & ZAP_FLAG_DROP_MARKER;
1373  }
1374  
1375  /*
1376   * This function makes sure that we'll replace the none pte with an uffd-wp
1377   * swap special pte marker when necessary. Must be with the pgtable lock held.
1378   */
1379  static inline void
zap_install_uffd_wp_if_needed(struct vm_area_struct * vma,unsigned long addr,pte_t * pte,struct zap_details * details,pte_t pteval)1380  zap_install_uffd_wp_if_needed(struct vm_area_struct *vma,
1381  			      unsigned long addr, pte_t *pte,
1382  			      struct zap_details *details, pte_t pteval)
1383  {
1384  	/* Zap on anonymous always means dropping everything */
1385  	if (vma_is_anonymous(vma))
1386  		return;
1387  
1388  	if (zap_drop_file_uffd_wp(details))
1389  		return;
1390  
1391  	pte_install_uffd_wp_if_needed(vma, addr, pte, pteval);
1392  }
1393  
zap_pte_range(struct mmu_gather * tlb,struct vm_area_struct * vma,pmd_t * pmd,unsigned long addr,unsigned long end,struct zap_details * details)1394  static unsigned long zap_pte_range(struct mmu_gather *tlb,
1395  				struct vm_area_struct *vma, pmd_t *pmd,
1396  				unsigned long addr, unsigned long end,
1397  				struct zap_details *details)
1398  {
1399  	struct mm_struct *mm = tlb->mm;
1400  	int force_flush = 0;
1401  	int rss[NR_MM_COUNTERS];
1402  	spinlock_t *ptl;
1403  	pte_t *start_pte;
1404  	pte_t *pte;
1405  	swp_entry_t entry;
1406  
1407  	tlb_change_page_size(tlb, PAGE_SIZE);
1408  	init_rss_vec(rss);
1409  	start_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
1410  	if (!pte)
1411  		return addr;
1412  
1413  	flush_tlb_batched_pending(mm);
1414  	arch_enter_lazy_mmu_mode();
1415  	do {
1416  		pte_t ptent = ptep_get(pte);
1417  		struct page *page;
1418  
1419  		if (pte_none(ptent))
1420  			continue;
1421  
1422  		if (need_resched())
1423  			break;
1424  
1425  		if (pte_present(ptent)) {
1426  			unsigned int delay_rmap;
1427  
1428  			page = vm_normal_page(vma, addr, ptent);
1429  			if (unlikely(!should_zap_page(details, page)))
1430  				continue;
1431  			ptent = ptep_get_and_clear_full(mm, addr, pte,
1432  							tlb->fullmm);
1433  			arch_check_zapped_pte(vma, ptent);
1434  			tlb_remove_tlb_entry(tlb, pte, addr);
1435  			zap_install_uffd_wp_if_needed(vma, addr, pte, details,
1436  						      ptent);
1437  			if (unlikely(!page)) {
1438  				ksm_might_unmap_zero_page(mm, ptent);
1439  				continue;
1440  			}
1441  
1442  			delay_rmap = 0;
1443  			if (!PageAnon(page)) {
1444  				if (pte_dirty(ptent)) {
1445  					set_page_dirty(page);
1446  					if (tlb_delay_rmap(tlb)) {
1447  						delay_rmap = 1;
1448  						force_flush = 1;
1449  					}
1450  				}
1451  				if (pte_young(ptent) && likely(vma_has_recency(vma)))
1452  					mark_page_accessed(page);
1453  			}
1454  			rss[mm_counter(page)]--;
1455  			if (!delay_rmap) {
1456  				page_remove_rmap(page, vma, false);
1457  				if (unlikely(page_mapcount(page) < 0))
1458  					print_bad_pte(vma, addr, ptent, page);
1459  			}
1460  			if (unlikely(__tlb_remove_page(tlb, page, delay_rmap))) {
1461  				force_flush = 1;
1462  				addr += PAGE_SIZE;
1463  				break;
1464  			}
1465  			continue;
1466  		}
1467  
1468  		entry = pte_to_swp_entry(ptent);
1469  		if (is_device_private_entry(entry) ||
1470  		    is_device_exclusive_entry(entry)) {
1471  			page = pfn_swap_entry_to_page(entry);
1472  			if (unlikely(!should_zap_page(details, page)))
1473  				continue;
1474  			/*
1475  			 * Both device private/exclusive mappings should only
1476  			 * work with anonymous page so far, so we don't need to
1477  			 * consider uffd-wp bit when zap. For more information,
1478  			 * see zap_install_uffd_wp_if_needed().
1479  			 */
1480  			WARN_ON_ONCE(!vma_is_anonymous(vma));
1481  			rss[mm_counter(page)]--;
1482  			if (is_device_private_entry(entry))
1483  				page_remove_rmap(page, vma, false);
1484  			put_page(page);
1485  		} else if (!non_swap_entry(entry)) {
1486  			/* Genuine swap entry, hence a private anon page */
1487  			if (!should_zap_cows(details))
1488  				continue;
1489  			rss[MM_SWAPENTS]--;
1490  			if (unlikely(!free_swap_and_cache(entry)))
1491  				print_bad_pte(vma, addr, ptent, NULL);
1492  		} else if (is_migration_entry(entry)) {
1493  			page = pfn_swap_entry_to_page(entry);
1494  			if (!should_zap_page(details, page))
1495  				continue;
1496  			rss[mm_counter(page)]--;
1497  		} else if (pte_marker_entry_uffd_wp(entry)) {
1498  			/*
1499  			 * For anon: always drop the marker; for file: only
1500  			 * drop the marker if explicitly requested.
1501  			 */
1502  			if (!vma_is_anonymous(vma) &&
1503  			    !zap_drop_file_uffd_wp(details))
1504  				continue;
1505  		} else if (is_hwpoison_entry(entry) ||
1506  			   is_poisoned_swp_entry(entry)) {
1507  			if (!should_zap_cows(details))
1508  				continue;
1509  		} else {
1510  			/* We should have covered all the swap entry types */
1511  			WARN_ON_ONCE(1);
1512  		}
1513  		pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
1514  		zap_install_uffd_wp_if_needed(vma, addr, pte, details, ptent);
1515  	} while (pte++, addr += PAGE_SIZE, addr != end);
1516  
1517  	add_mm_rss_vec(mm, rss);
1518  	arch_leave_lazy_mmu_mode();
1519  
1520  	/* Do the actual TLB flush before dropping ptl */
1521  	if (force_flush) {
1522  		tlb_flush_mmu_tlbonly(tlb);
1523  		tlb_flush_rmaps(tlb, vma);
1524  	}
1525  	pte_unmap_unlock(start_pte, ptl);
1526  
1527  	/*
1528  	 * If we forced a TLB flush (either due to running out of
1529  	 * batch buffers or because we needed to flush dirty TLB
1530  	 * entries before releasing the ptl), free the batched
1531  	 * memory too. Come back again if we didn't do everything.
1532  	 */
1533  	if (force_flush)
1534  		tlb_flush_mmu(tlb);
1535  
1536  	return addr;
1537  }
1538  
zap_pmd_range(struct mmu_gather * tlb,struct vm_area_struct * vma,pud_t * pud,unsigned long addr,unsigned long end,struct zap_details * details)1539  static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,
1540  				struct vm_area_struct *vma, pud_t *pud,
1541  				unsigned long addr, unsigned long end,
1542  				struct zap_details *details)
1543  {
1544  	pmd_t *pmd;
1545  	unsigned long next;
1546  
1547  	pmd = pmd_offset(pud, addr);
1548  	do {
1549  		next = pmd_addr_end(addr, end);
1550  		if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) {
1551  			if (next - addr != HPAGE_PMD_SIZE)
1552  				__split_huge_pmd(vma, pmd, addr, false, NULL);
1553  			else if (zap_huge_pmd(tlb, vma, pmd, addr)) {
1554  				addr = next;
1555  				continue;
1556  			}
1557  			/* fall through */
1558  		} else if (details && details->single_folio &&
1559  			   folio_test_pmd_mappable(details->single_folio) &&
1560  			   next - addr == HPAGE_PMD_SIZE && pmd_none(*pmd)) {
1561  			spinlock_t *ptl = pmd_lock(tlb->mm, pmd);
1562  			/*
1563  			 * Take and drop THP pmd lock so that we cannot return
1564  			 * prematurely, while zap_huge_pmd() has cleared *pmd,
1565  			 * but not yet decremented compound_mapcount().
1566  			 */
1567  			spin_unlock(ptl);
1568  		}
1569  		if (pmd_none(*pmd)) {
1570  			addr = next;
1571  			continue;
1572  		}
1573  		addr = zap_pte_range(tlb, vma, pmd, addr, next, details);
1574  		if (addr != next)
1575  			pmd--;
1576  	} while (pmd++, cond_resched(), addr != end);
1577  
1578  	return addr;
1579  }
1580  
zap_pud_range(struct mmu_gather * tlb,struct vm_area_struct * vma,p4d_t * p4d,unsigned long addr,unsigned long end,struct zap_details * details)1581  static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
1582  				struct vm_area_struct *vma, p4d_t *p4d,
1583  				unsigned long addr, unsigned long end,
1584  				struct zap_details *details)
1585  {
1586  	pud_t *pud;
1587  	unsigned long next;
1588  
1589  	pud = pud_offset(p4d, addr);
1590  	do {
1591  		next = pud_addr_end(addr, end);
1592  		if (pud_trans_huge(*pud) || pud_devmap(*pud)) {
1593  			if (next - addr != HPAGE_PUD_SIZE) {
1594  				mmap_assert_locked(tlb->mm);
1595  				split_huge_pud(vma, pud, addr);
1596  			} else if (zap_huge_pud(tlb, vma, pud, addr))
1597  				goto next;
1598  			/* fall through */
1599  		}
1600  		if (pud_none_or_clear_bad(pud))
1601  			continue;
1602  		next = zap_pmd_range(tlb, vma, pud, addr, next, details);
1603  next:
1604  		cond_resched();
1605  	} while (pud++, addr = next, addr != end);
1606  
1607  	return addr;
1608  }
1609  
zap_p4d_range(struct mmu_gather * tlb,struct vm_area_struct * vma,pgd_t * pgd,unsigned long addr,unsigned long end,struct zap_details * details)1610  static inline unsigned long zap_p4d_range(struct mmu_gather *tlb,
1611  				struct vm_area_struct *vma, pgd_t *pgd,
1612  				unsigned long addr, unsigned long end,
1613  				struct zap_details *details)
1614  {
1615  	p4d_t *p4d;
1616  	unsigned long next;
1617  
1618  	p4d = p4d_offset(pgd, addr);
1619  	do {
1620  		next = p4d_addr_end(addr, end);
1621  		if (p4d_none_or_clear_bad(p4d))
1622  			continue;
1623  		next = zap_pud_range(tlb, vma, p4d, addr, next, details);
1624  	} while (p4d++, addr = next, addr != end);
1625  
1626  	return addr;
1627  }
1628  
unmap_page_range(struct mmu_gather * tlb,struct vm_area_struct * vma,unsigned long addr,unsigned long end,struct zap_details * details)1629  void unmap_page_range(struct mmu_gather *tlb,
1630  			     struct vm_area_struct *vma,
1631  			     unsigned long addr, unsigned long end,
1632  			     struct zap_details *details)
1633  {
1634  	pgd_t *pgd;
1635  	unsigned long next;
1636  
1637  	BUG_ON(addr >= end);
1638  	tlb_start_vma(tlb, vma);
1639  	pgd = pgd_offset(vma->vm_mm, addr);
1640  	do {
1641  		next = pgd_addr_end(addr, end);
1642  		if (pgd_none_or_clear_bad(pgd))
1643  			continue;
1644  		next = zap_p4d_range(tlb, vma, pgd, addr, next, details);
1645  	} while (pgd++, addr = next, addr != end);
1646  	tlb_end_vma(tlb, vma);
1647  }
1648  
1649  
unmap_single_vma(struct mmu_gather * tlb,struct vm_area_struct * vma,unsigned long start_addr,unsigned long end_addr,struct zap_details * details,bool mm_wr_locked)1650  static void unmap_single_vma(struct mmu_gather *tlb,
1651  		struct vm_area_struct *vma, unsigned long start_addr,
1652  		unsigned long end_addr,
1653  		struct zap_details *details, bool mm_wr_locked)
1654  {
1655  	unsigned long start = max(vma->vm_start, start_addr);
1656  	unsigned long end;
1657  
1658  	if (start >= vma->vm_end)
1659  		return;
1660  	end = min(vma->vm_end, end_addr);
1661  	if (end <= vma->vm_start)
1662  		return;
1663  
1664  	if (vma->vm_file)
1665  		uprobe_munmap(vma, start, end);
1666  
1667  	if (unlikely(vma->vm_flags & VM_PFNMAP))
1668  		untrack_pfn(vma, 0, 0, mm_wr_locked);
1669  
1670  	if (start != end) {
1671  		if (unlikely(is_vm_hugetlb_page(vma))) {
1672  			/*
1673  			 * It is undesirable to test vma->vm_file as it
1674  			 * should be non-null for valid hugetlb area.
1675  			 * However, vm_file will be NULL in the error
1676  			 * cleanup path of mmap_region. When
1677  			 * hugetlbfs ->mmap method fails,
1678  			 * mmap_region() nullifies vma->vm_file
1679  			 * before calling this function to clean up.
1680  			 * Since no pte has actually been setup, it is
1681  			 * safe to do nothing in this case.
1682  			 */
1683  			if (vma->vm_file) {
1684  				zap_flags_t zap_flags = details ?
1685  				    details->zap_flags : 0;
1686  				__unmap_hugepage_range(tlb, vma, start, end,
1687  							     NULL, zap_flags);
1688  			}
1689  		} else
1690  			unmap_page_range(tlb, vma, start, end, details);
1691  	}
1692  }
1693  
1694  /**
1695   * unmap_vmas - unmap a range of memory covered by a list of vma's
1696   * @tlb: address of the caller's struct mmu_gather
1697   * @mas: the maple state
1698   * @vma: the starting vma
1699   * @start_addr: virtual address at which to start unmapping
1700   * @end_addr: virtual address at which to end unmapping
1701   * @tree_end: The maximum index to check
1702   * @mm_wr_locked: lock flag
1703   *
1704   * Unmap all pages in the vma list.
1705   *
1706   * Only addresses between `start' and `end' will be unmapped.
1707   *
1708   * The VMA list must be sorted in ascending virtual address order.
1709   *
1710   * unmap_vmas() assumes that the caller will flush the whole unmapped address
1711   * range after unmap_vmas() returns.  So the only responsibility here is to
1712   * ensure that any thus-far unmapped pages are flushed before unmap_vmas()
1713   * drops the lock and schedules.
1714   */
unmap_vmas(struct mmu_gather * tlb,struct ma_state * mas,struct vm_area_struct * vma,unsigned long start_addr,unsigned long end_addr,unsigned long tree_end,bool mm_wr_locked)1715  void unmap_vmas(struct mmu_gather *tlb, struct ma_state *mas,
1716  		struct vm_area_struct *vma, unsigned long start_addr,
1717  		unsigned long end_addr, unsigned long tree_end,
1718  		bool mm_wr_locked)
1719  {
1720  	struct mmu_notifier_range range;
1721  	struct zap_details details = {
1722  		.zap_flags = ZAP_FLAG_DROP_MARKER | ZAP_FLAG_UNMAP,
1723  		/* Careful - we need to zap private pages too! */
1724  		.even_cows = true,
1725  	};
1726  
1727  	mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma->vm_mm,
1728  				start_addr, end_addr);
1729  	mmu_notifier_invalidate_range_start(&range);
1730  	do {
1731  		unsigned long start = start_addr;
1732  		unsigned long end = end_addr;
1733  		hugetlb_zap_begin(vma, &start, &end);
1734  		unmap_single_vma(tlb, vma, start, end, &details,
1735  				 mm_wr_locked);
1736  		hugetlb_zap_end(vma, &details);
1737  	} while ((vma = mas_find(mas, tree_end - 1)) != NULL);
1738  	mmu_notifier_invalidate_range_end(&range);
1739  }
1740  
1741  /**
1742   * zap_page_range_single - remove user pages in a given range
1743   * @vma: vm_area_struct holding the applicable pages
1744   * @address: starting address of pages to zap
1745   * @size: number of bytes to zap
1746   * @details: details of shared cache invalidation
1747   *
1748   * The range must fit into one VMA.
1749   */
zap_page_range_single(struct vm_area_struct * vma,unsigned long address,unsigned long size,struct zap_details * details)1750  void zap_page_range_single(struct vm_area_struct *vma, unsigned long address,
1751  		unsigned long size, struct zap_details *details)
1752  {
1753  	const unsigned long end = address + size;
1754  	struct mmu_notifier_range range;
1755  	struct mmu_gather tlb;
1756  
1757  	lru_add_drain();
1758  	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
1759  				address, end);
1760  	hugetlb_zap_begin(vma, &range.start, &range.end);
1761  	tlb_gather_mmu(&tlb, vma->vm_mm);
1762  	update_hiwater_rss(vma->vm_mm);
1763  	mmu_notifier_invalidate_range_start(&range);
1764  	/*
1765  	 * unmap 'address-end' not 'range.start-range.end' as range
1766  	 * could have been expanded for hugetlb pmd sharing.
1767  	 */
1768  	unmap_single_vma(&tlb, vma, address, end, details, false);
1769  	mmu_notifier_invalidate_range_end(&range);
1770  	tlb_finish_mmu(&tlb);
1771  	hugetlb_zap_end(vma, details);
1772  }
1773  
1774  /**
1775   * zap_vma_ptes - remove ptes mapping the vma
1776   * @vma: vm_area_struct holding ptes to be zapped
1777   * @address: starting address of pages to zap
1778   * @size: number of bytes to zap
1779   *
1780   * This function only unmaps ptes assigned to VM_PFNMAP vmas.
1781   *
1782   * The entire address range must be fully contained within the vma.
1783   *
1784   */
zap_vma_ptes(struct vm_area_struct * vma,unsigned long address,unsigned long size)1785  void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
1786  		unsigned long size)
1787  {
1788  	if (!range_in_vma(vma, address, address + size) ||
1789  	    		!(vma->vm_flags & VM_PFNMAP))
1790  		return;
1791  
1792  	zap_page_range_single(vma, address, size, NULL);
1793  }
1794  EXPORT_SYMBOL_GPL(zap_vma_ptes);
1795  
walk_to_pmd(struct mm_struct * mm,unsigned long addr)1796  static pmd_t *walk_to_pmd(struct mm_struct *mm, unsigned long addr)
1797  {
1798  	pgd_t *pgd;
1799  	p4d_t *p4d;
1800  	pud_t *pud;
1801  	pmd_t *pmd;
1802  
1803  	pgd = pgd_offset(mm, addr);
1804  	p4d = p4d_alloc(mm, pgd, addr);
1805  	if (!p4d)
1806  		return NULL;
1807  	pud = pud_alloc(mm, p4d, addr);
1808  	if (!pud)
1809  		return NULL;
1810  	pmd = pmd_alloc(mm, pud, addr);
1811  	if (!pmd)
1812  		return NULL;
1813  
1814  	VM_BUG_ON(pmd_trans_huge(*pmd));
1815  	return pmd;
1816  }
1817  
__get_locked_pte(struct mm_struct * mm,unsigned long addr,spinlock_t ** ptl)1818  pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
1819  			spinlock_t **ptl)
1820  {
1821  	pmd_t *pmd = walk_to_pmd(mm, addr);
1822  
1823  	if (!pmd)
1824  		return NULL;
1825  	return pte_alloc_map_lock(mm, pmd, addr, ptl);
1826  }
1827  
validate_page_before_insert(struct page * page)1828  static int validate_page_before_insert(struct page *page)
1829  {
1830  	if (PageAnon(page) || PageSlab(page) || page_has_type(page))
1831  		return -EINVAL;
1832  	flush_dcache_page(page);
1833  	return 0;
1834  }
1835  
insert_page_into_pte_locked(struct vm_area_struct * vma,pte_t * pte,unsigned long addr,struct page * page,pgprot_t prot)1836  static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte,
1837  			unsigned long addr, struct page *page, pgprot_t prot)
1838  {
1839  	if (!pte_none(ptep_get(pte)))
1840  		return -EBUSY;
1841  	/* Ok, finally just insert the thing.. */
1842  	get_page(page);
1843  	inc_mm_counter(vma->vm_mm, mm_counter_file(page));
1844  	page_add_file_rmap(page, vma, false);
1845  	set_pte_at(vma->vm_mm, addr, pte, mk_pte(page, prot));
1846  	return 0;
1847  }
1848  
1849  /*
1850   * This is the old fallback for page remapping.
1851   *
1852   * For historical reasons, it only allows reserved pages. Only
1853   * old drivers should use this, and they needed to mark their
1854   * pages reserved for the old functions anyway.
1855   */
insert_page(struct vm_area_struct * vma,unsigned long addr,struct page * page,pgprot_t prot)1856  static int insert_page(struct vm_area_struct *vma, unsigned long addr,
1857  			struct page *page, pgprot_t prot)
1858  {
1859  	int retval;
1860  	pte_t *pte;
1861  	spinlock_t *ptl;
1862  
1863  	retval = validate_page_before_insert(page);
1864  	if (retval)
1865  		goto out;
1866  	retval = -ENOMEM;
1867  	pte = get_locked_pte(vma->vm_mm, addr, &ptl);
1868  	if (!pte)
1869  		goto out;
1870  	retval = insert_page_into_pte_locked(vma, pte, addr, page, prot);
1871  	pte_unmap_unlock(pte, ptl);
1872  out:
1873  	return retval;
1874  }
1875  
insert_page_in_batch_locked(struct vm_area_struct * vma,pte_t * pte,unsigned long addr,struct page * page,pgprot_t prot)1876  static int insert_page_in_batch_locked(struct vm_area_struct *vma, pte_t *pte,
1877  			unsigned long addr, struct page *page, pgprot_t prot)
1878  {
1879  	int err;
1880  
1881  	if (!page_count(page))
1882  		return -EINVAL;
1883  	err = validate_page_before_insert(page);
1884  	if (err)
1885  		return err;
1886  	return insert_page_into_pte_locked(vma, pte, addr, page, prot);
1887  }
1888  
1889  /* insert_pages() amortizes the cost of spinlock operations
1890   * when inserting pages in a loop.
1891   */
insert_pages(struct vm_area_struct * vma,unsigned long addr,struct page ** pages,unsigned long * num,pgprot_t prot)1892  static int insert_pages(struct vm_area_struct *vma, unsigned long addr,
1893  			struct page **pages, unsigned long *num, pgprot_t prot)
1894  {
1895  	pmd_t *pmd = NULL;
1896  	pte_t *start_pte, *pte;
1897  	spinlock_t *pte_lock;
1898  	struct mm_struct *const mm = vma->vm_mm;
1899  	unsigned long curr_page_idx = 0;
1900  	unsigned long remaining_pages_total = *num;
1901  	unsigned long pages_to_write_in_pmd;
1902  	int ret;
1903  more:
1904  	ret = -EFAULT;
1905  	pmd = walk_to_pmd(mm, addr);
1906  	if (!pmd)
1907  		goto out;
1908  
1909  	pages_to_write_in_pmd = min_t(unsigned long,
1910  		remaining_pages_total, PTRS_PER_PTE - pte_index(addr));
1911  
1912  	/* Allocate the PTE if necessary; takes PMD lock once only. */
1913  	ret = -ENOMEM;
1914  	if (pte_alloc(mm, pmd))
1915  		goto out;
1916  
1917  	while (pages_to_write_in_pmd) {
1918  		int pte_idx = 0;
1919  		const int batch_size = min_t(int, pages_to_write_in_pmd, 8);
1920  
1921  		start_pte = pte_offset_map_lock(mm, pmd, addr, &pte_lock);
1922  		if (!start_pte) {
1923  			ret = -EFAULT;
1924  			goto out;
1925  		}
1926  		for (pte = start_pte; pte_idx < batch_size; ++pte, ++pte_idx) {
1927  			int err = insert_page_in_batch_locked(vma, pte,
1928  				addr, pages[curr_page_idx], prot);
1929  			if (unlikely(err)) {
1930  				pte_unmap_unlock(start_pte, pte_lock);
1931  				ret = err;
1932  				remaining_pages_total -= pte_idx;
1933  				goto out;
1934  			}
1935  			addr += PAGE_SIZE;
1936  			++curr_page_idx;
1937  		}
1938  		pte_unmap_unlock(start_pte, pte_lock);
1939  		pages_to_write_in_pmd -= batch_size;
1940  		remaining_pages_total -= batch_size;
1941  	}
1942  	if (remaining_pages_total)
1943  		goto more;
1944  	ret = 0;
1945  out:
1946  	*num = remaining_pages_total;
1947  	return ret;
1948  }
1949  
1950  /**
1951   * vm_insert_pages - insert multiple pages into user vma, batching the pmd lock.
1952   * @vma: user vma to map to
1953   * @addr: target start user address of these pages
1954   * @pages: source kernel pages
1955   * @num: in: number of pages to map. out: number of pages that were *not*
1956   * mapped. (0 means all pages were successfully mapped).
1957   *
1958   * Preferred over vm_insert_page() when inserting multiple pages.
1959   *
1960   * In case of error, we may have mapped a subset of the provided
1961   * pages. It is the caller's responsibility to account for this case.
1962   *
1963   * The same restrictions apply as in vm_insert_page().
1964   */
vm_insert_pages(struct vm_area_struct * vma,unsigned long addr,struct page ** pages,unsigned long * num)1965  int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr,
1966  			struct page **pages, unsigned long *num)
1967  {
1968  	const unsigned long end_addr = addr + (*num * PAGE_SIZE) - 1;
1969  
1970  	if (addr < vma->vm_start || end_addr >= vma->vm_end)
1971  		return -EFAULT;
1972  	if (!(vma->vm_flags & VM_MIXEDMAP)) {
1973  		BUG_ON(mmap_read_trylock(vma->vm_mm));
1974  		BUG_ON(vma->vm_flags & VM_PFNMAP);
1975  		vm_flags_set(vma, VM_MIXEDMAP);
1976  	}
1977  	/* Defer page refcount checking till we're about to map that page. */
1978  	return insert_pages(vma, addr, pages, num, vma->vm_page_prot);
1979  }
1980  EXPORT_SYMBOL(vm_insert_pages);
1981  
1982  /**
1983   * vm_insert_page - insert single page into user vma
1984   * @vma: user vma to map to
1985   * @addr: target user address of this page
1986   * @page: source kernel page
1987   *
1988   * This allows drivers to insert individual pages they've allocated
1989   * into a user vma.
1990   *
1991   * The page has to be a nice clean _individual_ kernel allocation.
1992   * If you allocate a compound page, you need to have marked it as
1993   * such (__GFP_COMP), or manually just split the page up yourself
1994   * (see split_page()).
1995   *
1996   * NOTE! Traditionally this was done with "remap_pfn_range()" which
1997   * took an arbitrary page protection parameter. This doesn't allow
1998   * that. Your vma protection will have to be set up correctly, which
1999   * means that if you want a shared writable mapping, you'd better
2000   * ask for a shared writable mapping!
2001   *
2002   * The page does not need to be reserved.
2003   *
2004   * Usually this function is called from f_op->mmap() handler
2005   * under mm->mmap_lock write-lock, so it can change vma->vm_flags.
2006   * Caller must set VM_MIXEDMAP on vma if it wants to call this
2007   * function from other places, for example from page-fault handler.
2008   *
2009   * Return: %0 on success, negative error code otherwise.
2010   */
vm_insert_page(struct vm_area_struct * vma,unsigned long addr,struct page * page)2011  int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
2012  			struct page *page)
2013  {
2014  	if (addr < vma->vm_start || addr >= vma->vm_end)
2015  		return -EFAULT;
2016  	if (!page_count(page))
2017  		return -EINVAL;
2018  	if (!(vma->vm_flags & VM_MIXEDMAP)) {
2019  		BUG_ON(mmap_read_trylock(vma->vm_mm));
2020  		BUG_ON(vma->vm_flags & VM_PFNMAP);
2021  		vm_flags_set(vma, VM_MIXEDMAP);
2022  	}
2023  	return insert_page(vma, addr, page, vma->vm_page_prot);
2024  }
2025  EXPORT_SYMBOL(vm_insert_page);
2026  
2027  /*
2028   * __vm_map_pages - maps range of kernel pages into user vma
2029   * @vma: user vma to map to
2030   * @pages: pointer to array of source kernel pages
2031   * @num: number of pages in page array
2032   * @offset: user's requested vm_pgoff
2033   *
2034   * This allows drivers to map range of kernel pages into a user vma.
2035   *
2036   * Return: 0 on success and error code otherwise.
2037   */
__vm_map_pages(struct vm_area_struct * vma,struct page ** pages,unsigned long num,unsigned long offset)2038  static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages,
2039  				unsigned long num, unsigned long offset)
2040  {
2041  	unsigned long count = vma_pages(vma);
2042  	unsigned long uaddr = vma->vm_start;
2043  	int ret, i;
2044  
2045  	/* Fail if the user requested offset is beyond the end of the object */
2046  	if (offset >= num)
2047  		return -ENXIO;
2048  
2049  	/* Fail if the user requested size exceeds available object size */
2050  	if (count > num - offset)
2051  		return -ENXIO;
2052  
2053  	for (i = 0; i < count; i++) {
2054  		ret = vm_insert_page(vma, uaddr, pages[offset + i]);
2055  		if (ret < 0)
2056  			return ret;
2057  		uaddr += PAGE_SIZE;
2058  	}
2059  
2060  	return 0;
2061  }
2062  
2063  /**
2064   * vm_map_pages - maps range of kernel pages starts with non zero offset
2065   * @vma: user vma to map to
2066   * @pages: pointer to array of source kernel pages
2067   * @num: number of pages in page array
2068   *
2069   * Maps an object consisting of @num pages, catering for the user's
2070   * requested vm_pgoff
2071   *
2072   * If we fail to insert any page into the vma, the function will return
2073   * immediately leaving any previously inserted pages present.  Callers
2074   * from the mmap handler may immediately return the error as their caller
2075   * will destroy the vma, removing any successfully inserted pages. Other
2076   * callers should make their own arrangements for calling unmap_region().
2077   *
2078   * Context: Process context. Called by mmap handlers.
2079   * Return: 0 on success and error code otherwise.
2080   */
vm_map_pages(struct vm_area_struct * vma,struct page ** pages,unsigned long num)2081  int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
2082  				unsigned long num)
2083  {
2084  	return __vm_map_pages(vma, pages, num, vma->vm_pgoff);
2085  }
2086  EXPORT_SYMBOL(vm_map_pages);
2087  
2088  /**
2089   * vm_map_pages_zero - map range of kernel pages starts with zero offset
2090   * @vma: user vma to map to
2091   * @pages: pointer to array of source kernel pages
2092   * @num: number of pages in page array
2093   *
2094   * Similar to vm_map_pages(), except that it explicitly sets the offset
2095   * to 0. This function is intended for the drivers that did not consider
2096   * vm_pgoff.
2097   *
2098   * Context: Process context. Called by mmap handlers.
2099   * Return: 0 on success and error code otherwise.
2100   */
vm_map_pages_zero(struct vm_area_struct * vma,struct page ** pages,unsigned long num)2101  int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
2102  				unsigned long num)
2103  {
2104  	return __vm_map_pages(vma, pages, num, 0);
2105  }
2106  EXPORT_SYMBOL(vm_map_pages_zero);
2107  
insert_pfn(struct vm_area_struct * vma,unsigned long addr,pfn_t pfn,pgprot_t prot,bool mkwrite)2108  static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr,
2109  			pfn_t pfn, pgprot_t prot, bool mkwrite)
2110  {
2111  	struct mm_struct *mm = vma->vm_mm;
2112  	pte_t *pte, entry;
2113  	spinlock_t *ptl;
2114  
2115  	pte = get_locked_pte(mm, addr, &ptl);
2116  	if (!pte)
2117  		return VM_FAULT_OOM;
2118  	entry = ptep_get(pte);
2119  	if (!pte_none(entry)) {
2120  		if (mkwrite) {
2121  			/*
2122  			 * For read faults on private mappings the PFN passed
2123  			 * in may not match the PFN we have mapped if the
2124  			 * mapped PFN is a writeable COW page.  In the mkwrite
2125  			 * case we are creating a writable PTE for a shared
2126  			 * mapping and we expect the PFNs to match. If they
2127  			 * don't match, we are likely racing with block
2128  			 * allocation and mapping invalidation so just skip the
2129  			 * update.
2130  			 */
2131  			if (pte_pfn(entry) != pfn_t_to_pfn(pfn)) {
2132  				WARN_ON_ONCE(!is_zero_pfn(pte_pfn(entry)));
2133  				goto out_unlock;
2134  			}
2135  			entry = pte_mkyoung(entry);
2136  			entry = maybe_mkwrite(pte_mkdirty(entry), vma);
2137  			if (ptep_set_access_flags(vma, addr, pte, entry, 1))
2138  				update_mmu_cache(vma, addr, pte);
2139  		}
2140  		goto out_unlock;
2141  	}
2142  
2143  	/* Ok, finally just insert the thing.. */
2144  	if (pfn_t_devmap(pfn))
2145  		entry = pte_mkdevmap(pfn_t_pte(pfn, prot));
2146  	else
2147  		entry = pte_mkspecial(pfn_t_pte(pfn, prot));
2148  
2149  	if (mkwrite) {
2150  		entry = pte_mkyoung(entry);
2151  		entry = maybe_mkwrite(pte_mkdirty(entry), vma);
2152  	}
2153  
2154  	set_pte_at(mm, addr, pte, entry);
2155  	update_mmu_cache(vma, addr, pte); /* XXX: why not for insert_page? */
2156  
2157  out_unlock:
2158  	pte_unmap_unlock(pte, ptl);
2159  	return VM_FAULT_NOPAGE;
2160  }
2161  
2162  /**
2163   * vmf_insert_pfn_prot - insert single pfn into user vma with specified pgprot
2164   * @vma: user vma to map to
2165   * @addr: target user address of this page
2166   * @pfn: source kernel pfn
2167   * @pgprot: pgprot flags for the inserted page
2168   *
2169   * This is exactly like vmf_insert_pfn(), except that it allows drivers
2170   * to override pgprot on a per-page basis.
2171   *
2172   * This only makes sense for IO mappings, and it makes no sense for
2173   * COW mappings.  In general, using multiple vmas is preferable;
2174   * vmf_insert_pfn_prot should only be used if using multiple VMAs is
2175   * impractical.
2176   *
2177   * pgprot typically only differs from @vma->vm_page_prot when drivers set
2178   * caching- and encryption bits different than those of @vma->vm_page_prot,
2179   * because the caching- or encryption mode may not be known at mmap() time.
2180   *
2181   * This is ok as long as @vma->vm_page_prot is not used by the core vm
2182   * to set caching and encryption bits for those vmas (except for COW pages).
2183   * This is ensured by core vm only modifying these page table entries using
2184   * functions that don't touch caching- or encryption bits, using pte_modify()
2185   * if needed. (See for example mprotect()).
2186   *
2187   * Also when new page-table entries are created, this is only done using the
2188   * fault() callback, and never using the value of vma->vm_page_prot,
2189   * except for page-table entries that point to anonymous pages as the result
2190   * of COW.
2191   *
2192   * Context: Process context.  May allocate using %GFP_KERNEL.
2193   * Return: vm_fault_t value.
2194   */
vmf_insert_pfn_prot(struct vm_area_struct * vma,unsigned long addr,unsigned long pfn,pgprot_t pgprot)2195  vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
2196  			unsigned long pfn, pgprot_t pgprot)
2197  {
2198  	/*
2199  	 * Technically, architectures with pte_special can avoid all these
2200  	 * restrictions (same for remap_pfn_range).  However we would like
2201  	 * consistency in testing and feature parity among all, so we should
2202  	 * try to keep these invariants in place for everybody.
2203  	 */
2204  	BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
2205  	BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
2206  						(VM_PFNMAP|VM_MIXEDMAP));
2207  	BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
2208  	BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn));
2209  
2210  	if (addr < vma->vm_start || addr >= vma->vm_end)
2211  		return VM_FAULT_SIGBUS;
2212  
2213  	if (!pfn_modify_allowed(pfn, pgprot))
2214  		return VM_FAULT_SIGBUS;
2215  
2216  	track_pfn_insert(vma, &pgprot, __pfn_to_pfn_t(pfn, PFN_DEV));
2217  
2218  	return insert_pfn(vma, addr, __pfn_to_pfn_t(pfn, PFN_DEV), pgprot,
2219  			false);
2220  }
2221  EXPORT_SYMBOL(vmf_insert_pfn_prot);
2222  
2223  /**
2224   * vmf_insert_pfn - insert single pfn into user vma
2225   * @vma: user vma to map to
2226   * @addr: target user address of this page
2227   * @pfn: source kernel pfn
2228   *
2229   * Similar to vm_insert_page, this allows drivers to insert individual pages
2230   * they've allocated into a user vma. Same comments apply.
2231   *
2232   * This function should only be called from a vm_ops->fault handler, and
2233   * in that case the handler should return the result of this function.
2234   *
2235   * vma cannot be a COW mapping.
2236   *
2237   * As this is called only for pages that do not currently exist, we
2238   * do not need to flush old virtual caches or the TLB.
2239   *
2240   * Context: Process context.  May allocate using %GFP_KERNEL.
2241   * Return: vm_fault_t value.
2242   */
vmf_insert_pfn(struct vm_area_struct * vma,unsigned long addr,unsigned long pfn)2243  vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
2244  			unsigned long pfn)
2245  {
2246  	return vmf_insert_pfn_prot(vma, addr, pfn, vma->vm_page_prot);
2247  }
2248  EXPORT_SYMBOL(vmf_insert_pfn);
2249  
vm_mixed_ok(struct vm_area_struct * vma,pfn_t pfn)2250  static bool vm_mixed_ok(struct vm_area_struct *vma, pfn_t pfn)
2251  {
2252  	/* these checks mirror the abort conditions in vm_normal_page */
2253  	if (vma->vm_flags & VM_MIXEDMAP)
2254  		return true;
2255  	if (pfn_t_devmap(pfn))
2256  		return true;
2257  	if (pfn_t_special(pfn))
2258  		return true;
2259  	if (is_zero_pfn(pfn_t_to_pfn(pfn)))
2260  		return true;
2261  	return false;
2262  }
2263  
__vm_insert_mixed(struct vm_area_struct * vma,unsigned long addr,pfn_t pfn,bool mkwrite)2264  static vm_fault_t __vm_insert_mixed(struct vm_area_struct *vma,
2265  		unsigned long addr, pfn_t pfn, bool mkwrite)
2266  {
2267  	pgprot_t pgprot = vma->vm_page_prot;
2268  	int err;
2269  
2270  	BUG_ON(!vm_mixed_ok(vma, pfn));
2271  
2272  	if (addr < vma->vm_start || addr >= vma->vm_end)
2273  		return VM_FAULT_SIGBUS;
2274  
2275  	track_pfn_insert(vma, &pgprot, pfn);
2276  
2277  	if (!pfn_modify_allowed(pfn_t_to_pfn(pfn), pgprot))
2278  		return VM_FAULT_SIGBUS;
2279  
2280  	/*
2281  	 * If we don't have pte special, then we have to use the pfn_valid()
2282  	 * based VM_MIXEDMAP scheme (see vm_normal_page), and thus we *must*
2283  	 * refcount the page if pfn_valid is true (hence insert_page rather
2284  	 * than insert_pfn).  If a zero_pfn were inserted into a VM_MIXEDMAP
2285  	 * without pte special, it would there be refcounted as a normal page.
2286  	 */
2287  	if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL) &&
2288  	    !pfn_t_devmap(pfn) && pfn_t_valid(pfn)) {
2289  		struct page *page;
2290  
2291  		/*
2292  		 * At this point we are committed to insert_page()
2293  		 * regardless of whether the caller specified flags that
2294  		 * result in pfn_t_has_page() == false.
2295  		 */
2296  		page = pfn_to_page(pfn_t_to_pfn(pfn));
2297  		err = insert_page(vma, addr, page, pgprot);
2298  	} else {
2299  		return insert_pfn(vma, addr, pfn, pgprot, mkwrite);
2300  	}
2301  
2302  	if (err == -ENOMEM)
2303  		return VM_FAULT_OOM;
2304  	if (err < 0 && err != -EBUSY)
2305  		return VM_FAULT_SIGBUS;
2306  
2307  	return VM_FAULT_NOPAGE;
2308  }
2309  
vmf_insert_mixed(struct vm_area_struct * vma,unsigned long addr,pfn_t pfn)2310  vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
2311  		pfn_t pfn)
2312  {
2313  	return __vm_insert_mixed(vma, addr, pfn, false);
2314  }
2315  EXPORT_SYMBOL(vmf_insert_mixed);
2316  
2317  /*
2318   *  If the insertion of PTE failed because someone else already added a
2319   *  different entry in the mean time, we treat that as success as we assume
2320   *  the same entry was actually inserted.
2321   */
vmf_insert_mixed_mkwrite(struct vm_area_struct * vma,unsigned long addr,pfn_t pfn)2322  vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma,
2323  		unsigned long addr, pfn_t pfn)
2324  {
2325  	return __vm_insert_mixed(vma, addr, pfn, true);
2326  }
2327  EXPORT_SYMBOL(vmf_insert_mixed_mkwrite);
2328  
2329  /*
2330   * maps a range of physical memory into the requested pages. the old
2331   * mappings are removed. any references to nonexistent pages results
2332   * in null mappings (currently treated as "copy-on-access")
2333   */
remap_pte_range(struct mm_struct * mm,pmd_t * pmd,unsigned long addr,unsigned long end,unsigned long pfn,pgprot_t prot)2334  static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd,
2335  			unsigned long addr, unsigned long end,
2336  			unsigned long pfn, pgprot_t prot)
2337  {
2338  	pte_t *pte, *mapped_pte;
2339  	spinlock_t *ptl;
2340  	int err = 0;
2341  
2342  	mapped_pte = pte = pte_alloc_map_lock(mm, pmd, addr, &ptl);
2343  	if (!pte)
2344  		return -ENOMEM;
2345  	arch_enter_lazy_mmu_mode();
2346  	do {
2347  		BUG_ON(!pte_none(ptep_get(pte)));
2348  		if (!pfn_modify_allowed(pfn, prot)) {
2349  			err = -EACCES;
2350  			break;
2351  		}
2352  		set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot)));
2353  		pfn++;
2354  	} while (pte++, addr += PAGE_SIZE, addr != end);
2355  	arch_leave_lazy_mmu_mode();
2356  	pte_unmap_unlock(mapped_pte, ptl);
2357  	return err;
2358  }
2359  
remap_pmd_range(struct mm_struct * mm,pud_t * pud,unsigned long addr,unsigned long end,unsigned long pfn,pgprot_t prot)2360  static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud,
2361  			unsigned long addr, unsigned long end,
2362  			unsigned long pfn, pgprot_t prot)
2363  {
2364  	pmd_t *pmd;
2365  	unsigned long next;
2366  	int err;
2367  
2368  	pfn -= addr >> PAGE_SHIFT;
2369  	pmd = pmd_alloc(mm, pud, addr);
2370  	if (!pmd)
2371  		return -ENOMEM;
2372  	VM_BUG_ON(pmd_trans_huge(*pmd));
2373  	do {
2374  		next = pmd_addr_end(addr, end);
2375  		err = remap_pte_range(mm, pmd, addr, next,
2376  				pfn + (addr >> PAGE_SHIFT), prot);
2377  		if (err)
2378  			return err;
2379  	} while (pmd++, addr = next, addr != end);
2380  	return 0;
2381  }
2382  
remap_pud_range(struct mm_struct * mm,p4d_t * p4d,unsigned long addr,unsigned long end,unsigned long pfn,pgprot_t prot)2383  static inline int remap_pud_range(struct mm_struct *mm, p4d_t *p4d,
2384  			unsigned long addr, unsigned long end,
2385  			unsigned long pfn, pgprot_t prot)
2386  {
2387  	pud_t *pud;
2388  	unsigned long next;
2389  	int err;
2390  
2391  	pfn -= addr >> PAGE_SHIFT;
2392  	pud = pud_alloc(mm, p4d, addr);
2393  	if (!pud)
2394  		return -ENOMEM;
2395  	do {
2396  		next = pud_addr_end(addr, end);
2397  		err = remap_pmd_range(mm, pud, addr, next,
2398  				pfn + (addr >> PAGE_SHIFT), prot);
2399  		if (err)
2400  			return err;
2401  	} while (pud++, addr = next, addr != end);
2402  	return 0;
2403  }
2404  
remap_p4d_range(struct mm_struct * mm,pgd_t * pgd,unsigned long addr,unsigned long end,unsigned long pfn,pgprot_t prot)2405  static inline int remap_p4d_range(struct mm_struct *mm, pgd_t *pgd,
2406  			unsigned long addr, unsigned long end,
2407  			unsigned long pfn, pgprot_t prot)
2408  {
2409  	p4d_t *p4d;
2410  	unsigned long next;
2411  	int err;
2412  
2413  	pfn -= addr >> PAGE_SHIFT;
2414  	p4d = p4d_alloc(mm, pgd, addr);
2415  	if (!p4d)
2416  		return -ENOMEM;
2417  	do {
2418  		next = p4d_addr_end(addr, end);
2419  		err = remap_pud_range(mm, p4d, addr, next,
2420  				pfn + (addr >> PAGE_SHIFT), prot);
2421  		if (err)
2422  			return err;
2423  	} while (p4d++, addr = next, addr != end);
2424  	return 0;
2425  }
2426  
remap_pfn_range_internal(struct vm_area_struct * vma,unsigned long addr,unsigned long pfn,unsigned long size,pgprot_t prot)2427  static int remap_pfn_range_internal(struct vm_area_struct *vma, unsigned long addr,
2428  		unsigned long pfn, unsigned long size, pgprot_t prot)
2429  {
2430  	pgd_t *pgd;
2431  	unsigned long next;
2432  	unsigned long end = addr + PAGE_ALIGN(size);
2433  	struct mm_struct *mm = vma->vm_mm;
2434  	int err;
2435  
2436  	if (WARN_ON_ONCE(!PAGE_ALIGNED(addr)))
2437  		return -EINVAL;
2438  
2439  	/*
2440  	 * Physically remapped pages are special. Tell the
2441  	 * rest of the world about it:
2442  	 *   VM_IO tells people not to look at these pages
2443  	 *	(accesses can have side effects).
2444  	 *   VM_PFNMAP tells the core MM that the base pages are just
2445  	 *	raw PFN mappings, and do not have a "struct page" associated
2446  	 *	with them.
2447  	 *   VM_DONTEXPAND
2448  	 *      Disable vma merging and expanding with mremap().
2449  	 *   VM_DONTDUMP
2450  	 *      Omit vma from core dump, even when VM_IO turned off.
2451  	 *
2452  	 * There's a horrible special case to handle copy-on-write
2453  	 * behaviour that some programs depend on. We mark the "original"
2454  	 * un-COW'ed pages by matching them up with "vma->vm_pgoff".
2455  	 * See vm_normal_page() for details.
2456  	 */
2457  	if (is_cow_mapping(vma->vm_flags)) {
2458  		if (addr != vma->vm_start || end != vma->vm_end)
2459  			return -EINVAL;
2460  		vma->vm_pgoff = pfn;
2461  	}
2462  
2463  	vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
2464  
2465  	BUG_ON(addr >= end);
2466  	pfn -= addr >> PAGE_SHIFT;
2467  	pgd = pgd_offset(mm, addr);
2468  	flush_cache_range(vma, addr, end);
2469  	do {
2470  		next = pgd_addr_end(addr, end);
2471  		err = remap_p4d_range(mm, pgd, addr, next,
2472  				pfn + (addr >> PAGE_SHIFT), prot);
2473  		if (err)
2474  			return err;
2475  	} while (pgd++, addr = next, addr != end);
2476  
2477  	return 0;
2478  }
2479  
2480  /*
2481   * Variant of remap_pfn_range that does not call track_pfn_remap.  The caller
2482   * must have pre-validated the caching bits of the pgprot_t.
2483   */
remap_pfn_range_notrack(struct vm_area_struct * vma,unsigned long addr,unsigned long pfn,unsigned long size,pgprot_t prot)2484  int remap_pfn_range_notrack(struct vm_area_struct *vma, unsigned long addr,
2485  		unsigned long pfn, unsigned long size, pgprot_t prot)
2486  {
2487  	int error = remap_pfn_range_internal(vma, addr, pfn, size, prot);
2488  
2489  	if (!error)
2490  		return 0;
2491  
2492  	/*
2493  	 * A partial pfn range mapping is dangerous: it does not
2494  	 * maintain page reference counts, and callers may free
2495  	 * pages due to the error. So zap it early.
2496  	 */
2497  	zap_page_range_single(vma, addr, size, NULL);
2498  	return error;
2499  }
2500  
2501  /**
2502   * remap_pfn_range - remap kernel memory to userspace
2503   * @vma: user vma to map to
2504   * @addr: target page aligned user address to start at
2505   * @pfn: page frame number of kernel physical memory address
2506   * @size: size of mapping area
2507   * @prot: page protection flags for this mapping
2508   *
2509   * Note: this is only safe if the mm semaphore is held when called.
2510   *
2511   * Return: %0 on success, negative error code otherwise.
2512   */
remap_pfn_range(struct vm_area_struct * vma,unsigned long addr,unsigned long pfn,unsigned long size,pgprot_t prot)2513  int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
2514  		    unsigned long pfn, unsigned long size, pgprot_t prot)
2515  {
2516  	int err;
2517  
2518  	err = track_pfn_remap(vma, &prot, pfn, addr, PAGE_ALIGN(size));
2519  	if (err)
2520  		return -EINVAL;
2521  
2522  	err = remap_pfn_range_notrack(vma, addr, pfn, size, prot);
2523  	if (err)
2524  		untrack_pfn(vma, pfn, PAGE_ALIGN(size), true);
2525  	return err;
2526  }
2527  EXPORT_SYMBOL(remap_pfn_range);
2528  
2529  /**
2530   * vm_iomap_memory - remap memory to userspace
2531   * @vma: user vma to map to
2532   * @start: start of the physical memory to be mapped
2533   * @len: size of area
2534   *
2535   * This is a simplified io_remap_pfn_range() for common driver use. The
2536   * driver just needs to give us the physical memory range to be mapped,
2537   * we'll figure out the rest from the vma information.
2538   *
2539   * NOTE! Some drivers might want to tweak vma->vm_page_prot first to get
2540   * whatever write-combining details or similar.
2541   *
2542   * Return: %0 on success, negative error code otherwise.
2543   */
vm_iomap_memory(struct vm_area_struct * vma,phys_addr_t start,unsigned long len)2544  int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len)
2545  {
2546  	unsigned long vm_len, pfn, pages;
2547  
2548  	/* Check that the physical memory area passed in looks valid */
2549  	if (start + len < start)
2550  		return -EINVAL;
2551  	/*
2552  	 * You *really* shouldn't map things that aren't page-aligned,
2553  	 * but we've historically allowed it because IO memory might
2554  	 * just have smaller alignment.
2555  	 */
2556  	len += start & ~PAGE_MASK;
2557  	pfn = start >> PAGE_SHIFT;
2558  	pages = (len + ~PAGE_MASK) >> PAGE_SHIFT;
2559  	if (pfn + pages < pfn)
2560  		return -EINVAL;
2561  
2562  	/* We start the mapping 'vm_pgoff' pages into the area */
2563  	if (vma->vm_pgoff > pages)
2564  		return -EINVAL;
2565  	pfn += vma->vm_pgoff;
2566  	pages -= vma->vm_pgoff;
2567  
2568  	/* Can we fit all of the mapping? */
2569  	vm_len = vma->vm_end - vma->vm_start;
2570  	if (vm_len >> PAGE_SHIFT > pages)
2571  		return -EINVAL;
2572  
2573  	/* Ok, let it rip */
2574  	return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot);
2575  }
2576  EXPORT_SYMBOL(vm_iomap_memory);
2577  
apply_to_pte_range(struct mm_struct * mm,pmd_t * pmd,unsigned long addr,unsigned long end,pte_fn_t fn,void * data,bool create,pgtbl_mod_mask * mask)2578  static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,
2579  				     unsigned long addr, unsigned long end,
2580  				     pte_fn_t fn, void *data, bool create,
2581  				     pgtbl_mod_mask *mask)
2582  {
2583  	pte_t *pte, *mapped_pte;
2584  	int err = 0;
2585  	spinlock_t *ptl;
2586  
2587  	if (create) {
2588  		mapped_pte = pte = (mm == &init_mm) ?
2589  			pte_alloc_kernel_track(pmd, addr, mask) :
2590  			pte_alloc_map_lock(mm, pmd, addr, &ptl);
2591  		if (!pte)
2592  			return -ENOMEM;
2593  	} else {
2594  		mapped_pte = pte = (mm == &init_mm) ?
2595  			pte_offset_kernel(pmd, addr) :
2596  			pte_offset_map_lock(mm, pmd, addr, &ptl);
2597  		if (!pte)
2598  			return -EINVAL;
2599  	}
2600  
2601  	arch_enter_lazy_mmu_mode();
2602  
2603  	if (fn) {
2604  		do {
2605  			if (create || !pte_none(ptep_get(pte))) {
2606  				err = fn(pte++, addr, data);
2607  				if (err)
2608  					break;
2609  			}
2610  		} while (addr += PAGE_SIZE, addr != end);
2611  	}
2612  	*mask |= PGTBL_PTE_MODIFIED;
2613  
2614  	arch_leave_lazy_mmu_mode();
2615  
2616  	if (mm != &init_mm)
2617  		pte_unmap_unlock(mapped_pte, ptl);
2618  	return err;
2619  }
2620  
apply_to_pmd_range(struct mm_struct * mm,pud_t * pud,unsigned long addr,unsigned long end,pte_fn_t fn,void * data,bool create,pgtbl_mod_mask * mask)2621  static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud,
2622  				     unsigned long addr, unsigned long end,
2623  				     pte_fn_t fn, void *data, bool create,
2624  				     pgtbl_mod_mask *mask)
2625  {
2626  	pmd_t *pmd;
2627  	unsigned long next;
2628  	int err = 0;
2629  
2630  	BUG_ON(pud_huge(*pud));
2631  
2632  	if (create) {
2633  		pmd = pmd_alloc_track(mm, pud, addr, mask);
2634  		if (!pmd)
2635  			return -ENOMEM;
2636  	} else {
2637  		pmd = pmd_offset(pud, addr);
2638  	}
2639  	do {
2640  		next = pmd_addr_end(addr, end);
2641  		if (pmd_none(*pmd) && !create)
2642  			continue;
2643  		if (WARN_ON_ONCE(pmd_leaf(*pmd)))
2644  			return -EINVAL;
2645  		if (!pmd_none(*pmd) && WARN_ON_ONCE(pmd_bad(*pmd))) {
2646  			if (!create)
2647  				continue;
2648  			pmd_clear_bad(pmd);
2649  		}
2650  		err = apply_to_pte_range(mm, pmd, addr, next,
2651  					 fn, data, create, mask);
2652  		if (err)
2653  			break;
2654  	} while (pmd++, addr = next, addr != end);
2655  
2656  	return err;
2657  }
2658  
apply_to_pud_range(struct mm_struct * mm,p4d_t * p4d,unsigned long addr,unsigned long end,pte_fn_t fn,void * data,bool create,pgtbl_mod_mask * mask)2659  static int apply_to_pud_range(struct mm_struct *mm, p4d_t *p4d,
2660  				     unsigned long addr, unsigned long end,
2661  				     pte_fn_t fn, void *data, bool create,
2662  				     pgtbl_mod_mask *mask)
2663  {
2664  	pud_t *pud;
2665  	unsigned long next;
2666  	int err = 0;
2667  
2668  	if (create) {
2669  		pud = pud_alloc_track(mm, p4d, addr, mask);
2670  		if (!pud)
2671  			return -ENOMEM;
2672  	} else {
2673  		pud = pud_offset(p4d, addr);
2674  	}
2675  	do {
2676  		next = pud_addr_end(addr, end);
2677  		if (pud_none(*pud) && !create)
2678  			continue;
2679  		if (WARN_ON_ONCE(pud_leaf(*pud)))
2680  			return -EINVAL;
2681  		if (!pud_none(*pud) && WARN_ON_ONCE(pud_bad(*pud))) {
2682  			if (!create)
2683  				continue;
2684  			pud_clear_bad(pud);
2685  		}
2686  		err = apply_to_pmd_range(mm, pud, addr, next,
2687  					 fn, data, create, mask);
2688  		if (err)
2689  			break;
2690  	} while (pud++, addr = next, addr != end);
2691  
2692  	return err;
2693  }
2694  
apply_to_p4d_range(struct mm_struct * mm,pgd_t * pgd,unsigned long addr,unsigned long end,pte_fn_t fn,void * data,bool create,pgtbl_mod_mask * mask)2695  static int apply_to_p4d_range(struct mm_struct *mm, pgd_t *pgd,
2696  				     unsigned long addr, unsigned long end,
2697  				     pte_fn_t fn, void *data, bool create,
2698  				     pgtbl_mod_mask *mask)
2699  {
2700  	p4d_t *p4d;
2701  	unsigned long next;
2702  	int err = 0;
2703  
2704  	if (create) {
2705  		p4d = p4d_alloc_track(mm, pgd, addr, mask);
2706  		if (!p4d)
2707  			return -ENOMEM;
2708  	} else {
2709  		p4d = p4d_offset(pgd, addr);
2710  	}
2711  	do {
2712  		next = p4d_addr_end(addr, end);
2713  		if (p4d_none(*p4d) && !create)
2714  			continue;
2715  		if (WARN_ON_ONCE(p4d_leaf(*p4d)))
2716  			return -EINVAL;
2717  		if (!p4d_none(*p4d) && WARN_ON_ONCE(p4d_bad(*p4d))) {
2718  			if (!create)
2719  				continue;
2720  			p4d_clear_bad(p4d);
2721  		}
2722  		err = apply_to_pud_range(mm, p4d, addr, next,
2723  					 fn, data, create, mask);
2724  		if (err)
2725  			break;
2726  	} while (p4d++, addr = next, addr != end);
2727  
2728  	return err;
2729  }
2730  
__apply_to_page_range(struct mm_struct * mm,unsigned long addr,unsigned long size,pte_fn_t fn,void * data,bool create)2731  static int __apply_to_page_range(struct mm_struct *mm, unsigned long addr,
2732  				 unsigned long size, pte_fn_t fn,
2733  				 void *data, bool create)
2734  {
2735  	pgd_t *pgd;
2736  	unsigned long start = addr, next;
2737  	unsigned long end = addr + size;
2738  	pgtbl_mod_mask mask = 0;
2739  	int err = 0;
2740  
2741  	if (WARN_ON(addr >= end))
2742  		return -EINVAL;
2743  
2744  	pgd = pgd_offset(mm, addr);
2745  	do {
2746  		next = pgd_addr_end(addr, end);
2747  		if (pgd_none(*pgd) && !create)
2748  			continue;
2749  		if (WARN_ON_ONCE(pgd_leaf(*pgd)))
2750  			return -EINVAL;
2751  		if (!pgd_none(*pgd) && WARN_ON_ONCE(pgd_bad(*pgd))) {
2752  			if (!create)
2753  				continue;
2754  			pgd_clear_bad(pgd);
2755  		}
2756  		err = apply_to_p4d_range(mm, pgd, addr, next,
2757  					 fn, data, create, &mask);
2758  		if (err)
2759  			break;
2760  	} while (pgd++, addr = next, addr != end);
2761  
2762  	if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
2763  		arch_sync_kernel_mappings(start, start + size);
2764  
2765  	return err;
2766  }
2767  
2768  /*
2769   * Scan a region of virtual memory, filling in page tables as necessary
2770   * and calling a provided function on each leaf page table.
2771   */
apply_to_page_range(struct mm_struct * mm,unsigned long addr,unsigned long size,pte_fn_t fn,void * data)2772  int apply_to_page_range(struct mm_struct *mm, unsigned long addr,
2773  			unsigned long size, pte_fn_t fn, void *data)
2774  {
2775  	return __apply_to_page_range(mm, addr, size, fn, data, true);
2776  }
2777  EXPORT_SYMBOL_GPL(apply_to_page_range);
2778  
2779  /*
2780   * Scan a region of virtual memory, calling a provided function on
2781   * each leaf page table where it exists.
2782   *
2783   * Unlike apply_to_page_range, this does _not_ fill in page tables
2784   * where they are absent.
2785   */
apply_to_existing_page_range(struct mm_struct * mm,unsigned long addr,unsigned long size,pte_fn_t fn,void * data)2786  int apply_to_existing_page_range(struct mm_struct *mm, unsigned long addr,
2787  				 unsigned long size, pte_fn_t fn, void *data)
2788  {
2789  	return __apply_to_page_range(mm, addr, size, fn, data, false);
2790  }
2791  EXPORT_SYMBOL_GPL(apply_to_existing_page_range);
2792  
2793  /*
2794   * handle_pte_fault chooses page fault handler according to an entry which was
2795   * read non-atomically.  Before making any commitment, on those architectures
2796   * or configurations (e.g. i386 with PAE) which might give a mix of unmatched
2797   * parts, do_swap_page must check under lock before unmapping the pte and
2798   * proceeding (but do_wp_page is only called after already making such a check;
2799   * and do_anonymous_page can safely check later on).
2800   */
pte_unmap_same(struct vm_fault * vmf)2801  static inline int pte_unmap_same(struct vm_fault *vmf)
2802  {
2803  	int same = 1;
2804  #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPTION)
2805  	if (sizeof(pte_t) > sizeof(unsigned long)) {
2806  		spin_lock(vmf->ptl);
2807  		same = pte_same(ptep_get(vmf->pte), vmf->orig_pte);
2808  		spin_unlock(vmf->ptl);
2809  	}
2810  #endif
2811  	pte_unmap(vmf->pte);
2812  	vmf->pte = NULL;
2813  	return same;
2814  }
2815  
2816  /*
2817   * Return:
2818   *	0:		copied succeeded
2819   *	-EHWPOISON:	copy failed due to hwpoison in source page
2820   *	-EAGAIN:	copied failed (some other reason)
2821   */
__wp_page_copy_user(struct page * dst,struct page * src,struct vm_fault * vmf)2822  static inline int __wp_page_copy_user(struct page *dst, struct page *src,
2823  				      struct vm_fault *vmf)
2824  {
2825  	int ret;
2826  	void *kaddr;
2827  	void __user *uaddr;
2828  	struct vm_area_struct *vma = vmf->vma;
2829  	struct mm_struct *mm = vma->vm_mm;
2830  	unsigned long addr = vmf->address;
2831  
2832  	if (likely(src)) {
2833  		if (copy_mc_user_highpage(dst, src, addr, vma)) {
2834  			memory_failure_queue(page_to_pfn(src), 0);
2835  			return -EHWPOISON;
2836  		}
2837  		return 0;
2838  	}
2839  
2840  	/*
2841  	 * If the source page was a PFN mapping, we don't have
2842  	 * a "struct page" for it. We do a best-effort copy by
2843  	 * just copying from the original user address. If that
2844  	 * fails, we just zero-fill it. Live with it.
2845  	 */
2846  	kaddr = kmap_atomic(dst);
2847  	uaddr = (void __user *)(addr & PAGE_MASK);
2848  
2849  	/*
2850  	 * On architectures with software "accessed" bits, we would
2851  	 * take a double page fault, so mark it accessed here.
2852  	 */
2853  	vmf->pte = NULL;
2854  	if (!arch_has_hw_pte_young() && !pte_young(vmf->orig_pte)) {
2855  		pte_t entry;
2856  
2857  		vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl);
2858  		if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
2859  			/*
2860  			 * Other thread has already handled the fault
2861  			 * and update local tlb only
2862  			 */
2863  			if (vmf->pte)
2864  				update_mmu_tlb(vma, addr, vmf->pte);
2865  			ret = -EAGAIN;
2866  			goto pte_unlock;
2867  		}
2868  
2869  		entry = pte_mkyoung(vmf->orig_pte);
2870  		if (ptep_set_access_flags(vma, addr, vmf->pte, entry, 0))
2871  			update_mmu_cache_range(vmf, vma, addr, vmf->pte, 1);
2872  	}
2873  
2874  	/*
2875  	 * This really shouldn't fail, because the page is there
2876  	 * in the page tables. But it might just be unreadable,
2877  	 * in which case we just give up and fill the result with
2878  	 * zeroes.
2879  	 */
2880  	if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) {
2881  		if (vmf->pte)
2882  			goto warn;
2883  
2884  		/* Re-validate under PTL if the page is still mapped */
2885  		vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl);
2886  		if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
2887  			/* The PTE changed under us, update local tlb */
2888  			if (vmf->pte)
2889  				update_mmu_tlb(vma, addr, vmf->pte);
2890  			ret = -EAGAIN;
2891  			goto pte_unlock;
2892  		}
2893  
2894  		/*
2895  		 * The same page can be mapped back since last copy attempt.
2896  		 * Try to copy again under PTL.
2897  		 */
2898  		if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) {
2899  			/*
2900  			 * Give a warn in case there can be some obscure
2901  			 * use-case
2902  			 */
2903  warn:
2904  			WARN_ON_ONCE(1);
2905  			clear_page(kaddr);
2906  		}
2907  	}
2908  
2909  	ret = 0;
2910  
2911  pte_unlock:
2912  	if (vmf->pte)
2913  		pte_unmap_unlock(vmf->pte, vmf->ptl);
2914  	kunmap_atomic(kaddr);
2915  	flush_dcache_page(dst);
2916  
2917  	return ret;
2918  }
2919  
__get_fault_gfp_mask(struct vm_area_struct * vma)2920  static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma)
2921  {
2922  	struct file *vm_file = vma->vm_file;
2923  
2924  	if (vm_file)
2925  		return mapping_gfp_mask(vm_file->f_mapping) | __GFP_FS | __GFP_IO;
2926  
2927  	/*
2928  	 * Special mappings (e.g. VDSO) do not have any file so fake
2929  	 * a default GFP_KERNEL for them.
2930  	 */
2931  	return GFP_KERNEL;
2932  }
2933  
2934  /*
2935   * Notify the address space that the page is about to become writable so that
2936   * it can prohibit this or wait for the page to get into an appropriate state.
2937   *
2938   * We do this without the lock held, so that it can sleep if it needs to.
2939   */
do_page_mkwrite(struct vm_fault * vmf,struct folio * folio)2940  static vm_fault_t do_page_mkwrite(struct vm_fault *vmf, struct folio *folio)
2941  {
2942  	vm_fault_t ret;
2943  	unsigned int old_flags = vmf->flags;
2944  
2945  	vmf->flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE;
2946  
2947  	if (vmf->vma->vm_file &&
2948  	    IS_SWAPFILE(vmf->vma->vm_file->f_mapping->host))
2949  		return VM_FAULT_SIGBUS;
2950  
2951  	ret = vmf->vma->vm_ops->page_mkwrite(vmf);
2952  	/* Restore original flags so that caller is not surprised */
2953  	vmf->flags = old_flags;
2954  	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))
2955  		return ret;
2956  	if (unlikely(!(ret & VM_FAULT_LOCKED))) {
2957  		folio_lock(folio);
2958  		if (!folio->mapping) {
2959  			folio_unlock(folio);
2960  			return 0; /* retry */
2961  		}
2962  		ret |= VM_FAULT_LOCKED;
2963  	} else
2964  		VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
2965  	return ret;
2966  }
2967  
2968  /*
2969   * Handle dirtying of a page in shared file mapping on a write fault.
2970   *
2971   * The function expects the page to be locked and unlocks it.
2972   */
fault_dirty_shared_page(struct vm_fault * vmf)2973  static vm_fault_t fault_dirty_shared_page(struct vm_fault *vmf)
2974  {
2975  	struct vm_area_struct *vma = vmf->vma;
2976  	struct address_space *mapping;
2977  	struct folio *folio = page_folio(vmf->page);
2978  	bool dirtied;
2979  	bool page_mkwrite = vma->vm_ops && vma->vm_ops->page_mkwrite;
2980  
2981  	dirtied = folio_mark_dirty(folio);
2982  	VM_BUG_ON_FOLIO(folio_test_anon(folio), folio);
2983  	/*
2984  	 * Take a local copy of the address_space - folio.mapping may be zeroed
2985  	 * by truncate after folio_unlock().   The address_space itself remains
2986  	 * pinned by vma->vm_file's reference.  We rely on folio_unlock()'s
2987  	 * release semantics to prevent the compiler from undoing this copying.
2988  	 */
2989  	mapping = folio_raw_mapping(folio);
2990  	folio_unlock(folio);
2991  
2992  	if (!page_mkwrite)
2993  		file_update_time(vma->vm_file);
2994  
2995  	/*
2996  	 * Throttle page dirtying rate down to writeback speed.
2997  	 *
2998  	 * mapping may be NULL here because some device drivers do not
2999  	 * set page.mapping but still dirty their pages
3000  	 *
3001  	 * Drop the mmap_lock before waiting on IO, if we can. The file
3002  	 * is pinning the mapping, as per above.
3003  	 */
3004  	if ((dirtied || page_mkwrite) && mapping) {
3005  		struct file *fpin;
3006  
3007  		fpin = maybe_unlock_mmap_for_io(vmf, NULL);
3008  		balance_dirty_pages_ratelimited(mapping);
3009  		if (fpin) {
3010  			fput(fpin);
3011  			return VM_FAULT_COMPLETED;
3012  		}
3013  	}
3014  
3015  	return 0;
3016  }
3017  
3018  /*
3019   * Handle write page faults for pages that can be reused in the current vma
3020   *
3021   * This can happen either due to the mapping being with the VM_SHARED flag,
3022   * or due to us being the last reference standing to the page. In either
3023   * case, all we need to do here is to mark the page as writable and update
3024   * any related book-keeping.
3025   */
wp_page_reuse(struct vm_fault * vmf)3026  static inline void wp_page_reuse(struct vm_fault *vmf)
3027  	__releases(vmf->ptl)
3028  {
3029  	struct vm_area_struct *vma = vmf->vma;
3030  	struct page *page = vmf->page;
3031  	pte_t entry;
3032  
3033  	VM_BUG_ON(!(vmf->flags & FAULT_FLAG_WRITE));
3034  	VM_BUG_ON(page && PageAnon(page) && !PageAnonExclusive(page));
3035  
3036  	/*
3037  	 * Clear the pages cpupid information as the existing
3038  	 * information potentially belongs to a now completely
3039  	 * unrelated process.
3040  	 */
3041  	if (page)
3042  		page_cpupid_xchg_last(page, (1 << LAST_CPUPID_SHIFT) - 1);
3043  
3044  	flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
3045  	entry = pte_mkyoung(vmf->orig_pte);
3046  	entry = maybe_mkwrite(pte_mkdirty(entry), vma);
3047  	if (ptep_set_access_flags(vma, vmf->address, vmf->pte, entry, 1))
3048  		update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
3049  	pte_unmap_unlock(vmf->pte, vmf->ptl);
3050  	count_vm_event(PGREUSE);
3051  }
3052  
3053  /*
3054   * Handle the case of a page which we actually need to copy to a new page,
3055   * either due to COW or unsharing.
3056   *
3057   * Called with mmap_lock locked and the old page referenced, but
3058   * without the ptl held.
3059   *
3060   * High level logic flow:
3061   *
3062   * - Allocate a page, copy the content of the old page to the new one.
3063   * - Handle book keeping and accounting - cgroups, mmu-notifiers, etc.
3064   * - Take the PTL. If the pte changed, bail out and release the allocated page
3065   * - If the pte is still the way we remember it, update the page table and all
3066   *   relevant references. This includes dropping the reference the page-table
3067   *   held to the old page, as well as updating the rmap.
3068   * - In any case, unlock the PTL and drop the reference we took to the old page.
3069   */
wp_page_copy(struct vm_fault * vmf)3070  static vm_fault_t wp_page_copy(struct vm_fault *vmf)
3071  {
3072  	const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
3073  	struct vm_area_struct *vma = vmf->vma;
3074  	struct mm_struct *mm = vma->vm_mm;
3075  	struct folio *old_folio = NULL;
3076  	struct folio *new_folio = NULL;
3077  	pte_t entry;
3078  	int page_copied = 0;
3079  	struct mmu_notifier_range range;
3080  	int ret;
3081  
3082  	delayacct_wpcopy_start();
3083  
3084  	if (vmf->page)
3085  		old_folio = page_folio(vmf->page);
3086  	if (unlikely(anon_vma_prepare(vma)))
3087  		goto oom;
3088  
3089  	if (is_zero_pfn(pte_pfn(vmf->orig_pte))) {
3090  		new_folio = vma_alloc_zeroed_movable_folio(vma, vmf->address);
3091  		if (!new_folio)
3092  			goto oom;
3093  	} else {
3094  		new_folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma,
3095  				vmf->address, false);
3096  		if (!new_folio)
3097  			goto oom;
3098  
3099  		ret = __wp_page_copy_user(&new_folio->page, vmf->page, vmf);
3100  		if (ret) {
3101  			/*
3102  			 * COW failed, if the fault was solved by other,
3103  			 * it's fine. If not, userspace would re-fault on
3104  			 * the same address and we will handle the fault
3105  			 * from the second attempt.
3106  			 * The -EHWPOISON case will not be retried.
3107  			 */
3108  			folio_put(new_folio);
3109  			if (old_folio)
3110  				folio_put(old_folio);
3111  
3112  			delayacct_wpcopy_end();
3113  			return ret == -EHWPOISON ? VM_FAULT_HWPOISON : 0;
3114  		}
3115  		kmsan_copy_page_meta(&new_folio->page, vmf->page);
3116  	}
3117  
3118  	if (mem_cgroup_charge(new_folio, mm, GFP_KERNEL))
3119  		goto oom_free_new;
3120  	folio_throttle_swaprate(new_folio, GFP_KERNEL);
3121  
3122  	__folio_mark_uptodate(new_folio);
3123  
3124  	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm,
3125  				vmf->address & PAGE_MASK,
3126  				(vmf->address & PAGE_MASK) + PAGE_SIZE);
3127  	mmu_notifier_invalidate_range_start(&range);
3128  
3129  	/*
3130  	 * Re-check the pte - we dropped the lock
3131  	 */
3132  	vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl);
3133  	if (likely(vmf->pte && pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
3134  		if (old_folio) {
3135  			if (!folio_test_anon(old_folio)) {
3136  				dec_mm_counter(mm, mm_counter_file(&old_folio->page));
3137  				inc_mm_counter(mm, MM_ANONPAGES);
3138  			}
3139  		} else {
3140  			ksm_might_unmap_zero_page(mm, vmf->orig_pte);
3141  			inc_mm_counter(mm, MM_ANONPAGES);
3142  		}
3143  		flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
3144  		entry = mk_pte(&new_folio->page, vma->vm_page_prot);
3145  		entry = pte_sw_mkyoung(entry);
3146  		if (unlikely(unshare)) {
3147  			if (pte_soft_dirty(vmf->orig_pte))
3148  				entry = pte_mksoft_dirty(entry);
3149  			if (pte_uffd_wp(vmf->orig_pte))
3150  				entry = pte_mkuffd_wp(entry);
3151  		} else {
3152  			entry = maybe_mkwrite(pte_mkdirty(entry), vma);
3153  		}
3154  
3155  		/*
3156  		 * Clear the pte entry and flush it first, before updating the
3157  		 * pte with the new entry, to keep TLBs on different CPUs in
3158  		 * sync. This code used to set the new PTE then flush TLBs, but
3159  		 * that left a window where the new PTE could be loaded into
3160  		 * some TLBs while the old PTE remains in others.
3161  		 */
3162  		ptep_clear_flush(vma, vmf->address, vmf->pte);
3163  		folio_add_new_anon_rmap(new_folio, vma, vmf->address);
3164  		folio_add_lru_vma(new_folio, vma);
3165  		/*
3166  		 * We call the notify macro here because, when using secondary
3167  		 * mmu page tables (such as kvm shadow page tables), we want the
3168  		 * new page to be mapped directly into the secondary page table.
3169  		 */
3170  		BUG_ON(unshare && pte_write(entry));
3171  		set_pte_at_notify(mm, vmf->address, vmf->pte, entry);
3172  		update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
3173  		if (old_folio) {
3174  			/*
3175  			 * Only after switching the pte to the new page may
3176  			 * we remove the mapcount here. Otherwise another
3177  			 * process may come and find the rmap count decremented
3178  			 * before the pte is switched to the new page, and
3179  			 * "reuse" the old page writing into it while our pte
3180  			 * here still points into it and can be read by other
3181  			 * threads.
3182  			 *
3183  			 * The critical issue is to order this
3184  			 * page_remove_rmap with the ptp_clear_flush above.
3185  			 * Those stores are ordered by (if nothing else,)
3186  			 * the barrier present in the atomic_add_negative
3187  			 * in page_remove_rmap.
3188  			 *
3189  			 * Then the TLB flush in ptep_clear_flush ensures that
3190  			 * no process can access the old page before the
3191  			 * decremented mapcount is visible. And the old page
3192  			 * cannot be reused until after the decremented
3193  			 * mapcount is visible. So transitively, TLBs to
3194  			 * old page will be flushed before it can be reused.
3195  			 */
3196  			page_remove_rmap(vmf->page, vma, false);
3197  		}
3198  
3199  		/* Free the old page.. */
3200  		new_folio = old_folio;
3201  		page_copied = 1;
3202  		pte_unmap_unlock(vmf->pte, vmf->ptl);
3203  	} else if (vmf->pte) {
3204  		update_mmu_tlb(vma, vmf->address, vmf->pte);
3205  		pte_unmap_unlock(vmf->pte, vmf->ptl);
3206  	}
3207  
3208  	mmu_notifier_invalidate_range_end(&range);
3209  
3210  	if (new_folio)
3211  		folio_put(new_folio);
3212  	if (old_folio) {
3213  		if (page_copied)
3214  			free_swap_cache(&old_folio->page);
3215  		folio_put(old_folio);
3216  	}
3217  
3218  	delayacct_wpcopy_end();
3219  	return 0;
3220  oom_free_new:
3221  	folio_put(new_folio);
3222  oom:
3223  	if (old_folio)
3224  		folio_put(old_folio);
3225  
3226  	delayacct_wpcopy_end();
3227  	return VM_FAULT_OOM;
3228  }
3229  
3230  /**
3231   * finish_mkwrite_fault - finish page fault for a shared mapping, making PTE
3232   *			  writeable once the page is prepared
3233   *
3234   * @vmf: structure describing the fault
3235   *
3236   * This function handles all that is needed to finish a write page fault in a
3237   * shared mapping due to PTE being read-only once the mapped page is prepared.
3238   * It handles locking of PTE and modifying it.
3239   *
3240   * The function expects the page to be locked or other protection against
3241   * concurrent faults / writeback (such as DAX radix tree locks).
3242   *
3243   * Return: %0 on success, %VM_FAULT_NOPAGE when PTE got changed before
3244   * we acquired PTE lock.
3245   */
finish_mkwrite_fault(struct vm_fault * vmf)3246  vm_fault_t finish_mkwrite_fault(struct vm_fault *vmf)
3247  {
3248  	WARN_ON_ONCE(!(vmf->vma->vm_flags & VM_SHARED));
3249  	vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, vmf->address,
3250  				       &vmf->ptl);
3251  	if (!vmf->pte)
3252  		return VM_FAULT_NOPAGE;
3253  	/*
3254  	 * We might have raced with another page fault while we released the
3255  	 * pte_offset_map_lock.
3256  	 */
3257  	if (!pte_same(ptep_get(vmf->pte), vmf->orig_pte)) {
3258  		update_mmu_tlb(vmf->vma, vmf->address, vmf->pte);
3259  		pte_unmap_unlock(vmf->pte, vmf->ptl);
3260  		return VM_FAULT_NOPAGE;
3261  	}
3262  	wp_page_reuse(vmf);
3263  	return 0;
3264  }
3265  
3266  /*
3267   * Handle write page faults for VM_MIXEDMAP or VM_PFNMAP for a VM_SHARED
3268   * mapping
3269   */
wp_pfn_shared(struct vm_fault * vmf)3270  static vm_fault_t wp_pfn_shared(struct vm_fault *vmf)
3271  {
3272  	struct vm_area_struct *vma = vmf->vma;
3273  
3274  	if (vma->vm_ops && vma->vm_ops->pfn_mkwrite) {
3275  		vm_fault_t ret;
3276  
3277  		pte_unmap_unlock(vmf->pte, vmf->ptl);
3278  		if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
3279  			vma_end_read(vmf->vma);
3280  			return VM_FAULT_RETRY;
3281  		}
3282  
3283  		vmf->flags |= FAULT_FLAG_MKWRITE;
3284  		ret = vma->vm_ops->pfn_mkwrite(vmf);
3285  		if (ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))
3286  			return ret;
3287  		return finish_mkwrite_fault(vmf);
3288  	}
3289  	wp_page_reuse(vmf);
3290  	return 0;
3291  }
3292  
wp_page_shared(struct vm_fault * vmf,struct folio * folio)3293  static vm_fault_t wp_page_shared(struct vm_fault *vmf, struct folio *folio)
3294  	__releases(vmf->ptl)
3295  {
3296  	struct vm_area_struct *vma = vmf->vma;
3297  	vm_fault_t ret = 0;
3298  
3299  	folio_get(folio);
3300  
3301  	if (vma->vm_ops && vma->vm_ops->page_mkwrite) {
3302  		vm_fault_t tmp;
3303  
3304  		pte_unmap_unlock(vmf->pte, vmf->ptl);
3305  		if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
3306  			folio_put(folio);
3307  			vma_end_read(vmf->vma);
3308  			return VM_FAULT_RETRY;
3309  		}
3310  
3311  		tmp = do_page_mkwrite(vmf, folio);
3312  		if (unlikely(!tmp || (tmp &
3313  				      (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
3314  			folio_put(folio);
3315  			return tmp;
3316  		}
3317  		tmp = finish_mkwrite_fault(vmf);
3318  		if (unlikely(tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) {
3319  			folio_unlock(folio);
3320  			folio_put(folio);
3321  			return tmp;
3322  		}
3323  	} else {
3324  		wp_page_reuse(vmf);
3325  		folio_lock(folio);
3326  	}
3327  	ret |= fault_dirty_shared_page(vmf);
3328  	folio_put(folio);
3329  
3330  	return ret;
3331  }
3332  
3333  /*
3334   * This routine handles present pages, when
3335   * * users try to write to a shared page (FAULT_FLAG_WRITE)
3336   * * GUP wants to take a R/O pin on a possibly shared anonymous page
3337   *   (FAULT_FLAG_UNSHARE)
3338   *
3339   * It is done by copying the page to a new address and decrementing the
3340   * shared-page counter for the old page.
3341   *
3342   * Note that this routine assumes that the protection checks have been
3343   * done by the caller (the low-level page fault routine in most cases).
3344   * Thus, with FAULT_FLAG_WRITE, we can safely just mark it writable once we've
3345   * done any necessary COW.
3346   *
3347   * In case of FAULT_FLAG_WRITE, we also mark the page dirty at this point even
3348   * though the page will change only once the write actually happens. This
3349   * avoids a few races, and potentially makes it more efficient.
3350   *
3351   * We enter with non-exclusive mmap_lock (to exclude vma changes,
3352   * but allow concurrent faults), with pte both mapped and locked.
3353   * We return with mmap_lock still held, but pte unmapped and unlocked.
3354   */
do_wp_page(struct vm_fault * vmf)3355  static vm_fault_t do_wp_page(struct vm_fault *vmf)
3356  	__releases(vmf->ptl)
3357  {
3358  	const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
3359  	struct vm_area_struct *vma = vmf->vma;
3360  	struct folio *folio = NULL;
3361  
3362  	if (likely(!unshare)) {
3363  		if (userfaultfd_pte_wp(vma, ptep_get(vmf->pte))) {
3364  			pte_unmap_unlock(vmf->pte, vmf->ptl);
3365  			return handle_userfault(vmf, VM_UFFD_WP);
3366  		}
3367  
3368  		/*
3369  		 * Userfaultfd write-protect can defer flushes. Ensure the TLB
3370  		 * is flushed in this case before copying.
3371  		 */
3372  		if (unlikely(userfaultfd_wp(vmf->vma) &&
3373  			     mm_tlb_flush_pending(vmf->vma->vm_mm)))
3374  			flush_tlb_page(vmf->vma, vmf->address);
3375  	}
3376  
3377  	vmf->page = vm_normal_page(vma, vmf->address, vmf->orig_pte);
3378  
3379  	if (vmf->page)
3380  		folio = page_folio(vmf->page);
3381  
3382  	/*
3383  	 * Shared mapping: we are guaranteed to have VM_WRITE and
3384  	 * FAULT_FLAG_WRITE set at this point.
3385  	 */
3386  	if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
3387  		/*
3388  		 * VM_MIXEDMAP !pfn_valid() case, or VM_SOFTDIRTY clear on a
3389  		 * VM_PFNMAP VMA.
3390  		 *
3391  		 * We should not cow pages in a shared writeable mapping.
3392  		 * Just mark the pages writable and/or call ops->pfn_mkwrite.
3393  		 */
3394  		if (!vmf->page)
3395  			return wp_pfn_shared(vmf);
3396  		return wp_page_shared(vmf, folio);
3397  	}
3398  
3399  	/*
3400  	 * Private mapping: create an exclusive anonymous page copy if reuse
3401  	 * is impossible. We might miss VM_WRITE for FOLL_FORCE handling.
3402  	 */
3403  	if (folio && folio_test_anon(folio)) {
3404  		/*
3405  		 * If the page is exclusive to this process we must reuse the
3406  		 * page without further checks.
3407  		 */
3408  		if (PageAnonExclusive(vmf->page))
3409  			goto reuse;
3410  
3411  		/*
3412  		 * We have to verify under folio lock: these early checks are
3413  		 * just an optimization to avoid locking the folio and freeing
3414  		 * the swapcache if there is little hope that we can reuse.
3415  		 *
3416  		 * KSM doesn't necessarily raise the folio refcount.
3417  		 */
3418  		if (folio_test_ksm(folio) || folio_ref_count(folio) > 3)
3419  			goto copy;
3420  		if (!folio_test_lru(folio))
3421  			/*
3422  			 * We cannot easily detect+handle references from
3423  			 * remote LRU caches or references to LRU folios.
3424  			 */
3425  			lru_add_drain();
3426  		if (folio_ref_count(folio) > 1 + folio_test_swapcache(folio))
3427  			goto copy;
3428  		if (!folio_trylock(folio))
3429  			goto copy;
3430  		if (folio_test_swapcache(folio))
3431  			folio_free_swap(folio);
3432  		if (folio_test_ksm(folio) || folio_ref_count(folio) != 1) {
3433  			folio_unlock(folio);
3434  			goto copy;
3435  		}
3436  		/*
3437  		 * Ok, we've got the only folio reference from our mapping
3438  		 * and the folio is locked, it's dark out, and we're wearing
3439  		 * sunglasses. Hit it.
3440  		 */
3441  		page_move_anon_rmap(vmf->page, vma);
3442  		folio_unlock(folio);
3443  reuse:
3444  		if (unlikely(unshare)) {
3445  			pte_unmap_unlock(vmf->pte, vmf->ptl);
3446  			return 0;
3447  		}
3448  		wp_page_reuse(vmf);
3449  		return 0;
3450  	}
3451  copy:
3452  	if ((vmf->flags & FAULT_FLAG_VMA_LOCK) && !vma->anon_vma) {
3453  		pte_unmap_unlock(vmf->pte, vmf->ptl);
3454  		vma_end_read(vmf->vma);
3455  		return VM_FAULT_RETRY;
3456  	}
3457  
3458  	/*
3459  	 * Ok, we need to copy. Oh, well..
3460  	 */
3461  	if (folio)
3462  		folio_get(folio);
3463  
3464  	pte_unmap_unlock(vmf->pte, vmf->ptl);
3465  #ifdef CONFIG_KSM
3466  	if (folio && folio_test_ksm(folio))
3467  		count_vm_event(COW_KSM);
3468  #endif
3469  	return wp_page_copy(vmf);
3470  }
3471  
unmap_mapping_range_vma(struct vm_area_struct * vma,unsigned long start_addr,unsigned long end_addr,struct zap_details * details)3472  static void unmap_mapping_range_vma(struct vm_area_struct *vma,
3473  		unsigned long start_addr, unsigned long end_addr,
3474  		struct zap_details *details)
3475  {
3476  	zap_page_range_single(vma, start_addr, end_addr - start_addr, details);
3477  }
3478  
unmap_mapping_range_tree(struct rb_root_cached * root,pgoff_t first_index,pgoff_t last_index,struct zap_details * details)3479  static inline void unmap_mapping_range_tree(struct rb_root_cached *root,
3480  					    pgoff_t first_index,
3481  					    pgoff_t last_index,
3482  					    struct zap_details *details)
3483  {
3484  	struct vm_area_struct *vma;
3485  	pgoff_t vba, vea, zba, zea;
3486  
3487  	vma_interval_tree_foreach(vma, root, first_index, last_index) {
3488  		vba = vma->vm_pgoff;
3489  		vea = vba + vma_pages(vma) - 1;
3490  		zba = max(first_index, vba);
3491  		zea = min(last_index, vea);
3492  
3493  		unmap_mapping_range_vma(vma,
3494  			((zba - vba) << PAGE_SHIFT) + vma->vm_start,
3495  			((zea - vba + 1) << PAGE_SHIFT) + vma->vm_start,
3496  				details);
3497  	}
3498  }
3499  
3500  /**
3501   * unmap_mapping_folio() - Unmap single folio from processes.
3502   * @folio: The locked folio to be unmapped.
3503   *
3504   * Unmap this folio from any userspace process which still has it mmaped.
3505   * Typically, for efficiency, the range of nearby pages has already been
3506   * unmapped by unmap_mapping_pages() or unmap_mapping_range().  But once
3507   * truncation or invalidation holds the lock on a folio, it may find that
3508   * the page has been remapped again: and then uses unmap_mapping_folio()
3509   * to unmap it finally.
3510   */
unmap_mapping_folio(struct folio * folio)3511  void unmap_mapping_folio(struct folio *folio)
3512  {
3513  	struct address_space *mapping = folio->mapping;
3514  	struct zap_details details = { };
3515  	pgoff_t	first_index;
3516  	pgoff_t	last_index;
3517  
3518  	VM_BUG_ON(!folio_test_locked(folio));
3519  
3520  	first_index = folio->index;
3521  	last_index = folio_next_index(folio) - 1;
3522  
3523  	details.even_cows = false;
3524  	details.single_folio = folio;
3525  	details.zap_flags = ZAP_FLAG_DROP_MARKER;
3526  
3527  	i_mmap_lock_read(mapping);
3528  	if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root)))
3529  		unmap_mapping_range_tree(&mapping->i_mmap, first_index,
3530  					 last_index, &details);
3531  	i_mmap_unlock_read(mapping);
3532  }
3533  
3534  /**
3535   * unmap_mapping_pages() - Unmap pages from processes.
3536   * @mapping: The address space containing pages to be unmapped.
3537   * @start: Index of first page to be unmapped.
3538   * @nr: Number of pages to be unmapped.  0 to unmap to end of file.
3539   * @even_cows: Whether to unmap even private COWed pages.
3540   *
3541   * Unmap the pages in this address space from any userspace process which
3542   * has them mmaped.  Generally, you want to remove COWed pages as well when
3543   * a file is being truncated, but not when invalidating pages from the page
3544   * cache.
3545   */
unmap_mapping_pages(struct address_space * mapping,pgoff_t start,pgoff_t nr,bool even_cows)3546  void unmap_mapping_pages(struct address_space *mapping, pgoff_t start,
3547  		pgoff_t nr, bool even_cows)
3548  {
3549  	struct zap_details details = { };
3550  	pgoff_t	first_index = start;
3551  	pgoff_t	last_index = start + nr - 1;
3552  
3553  	details.even_cows = even_cows;
3554  	if (last_index < first_index)
3555  		last_index = ULONG_MAX;
3556  
3557  	i_mmap_lock_read(mapping);
3558  	if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root)))
3559  		unmap_mapping_range_tree(&mapping->i_mmap, first_index,
3560  					 last_index, &details);
3561  	i_mmap_unlock_read(mapping);
3562  }
3563  EXPORT_SYMBOL_GPL(unmap_mapping_pages);
3564  
3565  /**
3566   * unmap_mapping_range - unmap the portion of all mmaps in the specified
3567   * address_space corresponding to the specified byte range in the underlying
3568   * file.
3569   *
3570   * @mapping: the address space containing mmaps to be unmapped.
3571   * @holebegin: byte in first page to unmap, relative to the start of
3572   * the underlying file.  This will be rounded down to a PAGE_SIZE
3573   * boundary.  Note that this is different from truncate_pagecache(), which
3574   * must keep the partial page.  In contrast, we must get rid of
3575   * partial pages.
3576   * @holelen: size of prospective hole in bytes.  This will be rounded
3577   * up to a PAGE_SIZE boundary.  A holelen of zero truncates to the
3578   * end of the file.
3579   * @even_cows: 1 when truncating a file, unmap even private COWed pages;
3580   * but 0 when invalidating pagecache, don't throw away private data.
3581   */
unmap_mapping_range(struct address_space * mapping,loff_t const holebegin,loff_t const holelen,int even_cows)3582  void unmap_mapping_range(struct address_space *mapping,
3583  		loff_t const holebegin, loff_t const holelen, int even_cows)
3584  {
3585  	pgoff_t hba = (pgoff_t)(holebegin) >> PAGE_SHIFT;
3586  	pgoff_t hlen = ((pgoff_t)(holelen) + PAGE_SIZE - 1) >> PAGE_SHIFT;
3587  
3588  	/* Check for overflow. */
3589  	if (sizeof(holelen) > sizeof(hlen)) {
3590  		long long holeend =
3591  			(holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
3592  		if (holeend & ~(long long)ULONG_MAX)
3593  			hlen = ULONG_MAX - hba + 1;
3594  	}
3595  
3596  	unmap_mapping_pages(mapping, hba, hlen, even_cows);
3597  }
3598  EXPORT_SYMBOL(unmap_mapping_range);
3599  
3600  /*
3601   * Restore a potential device exclusive pte to a working pte entry
3602   */
remove_device_exclusive_entry(struct vm_fault * vmf)3603  static vm_fault_t remove_device_exclusive_entry(struct vm_fault *vmf)
3604  {
3605  	struct folio *folio = page_folio(vmf->page);
3606  	struct vm_area_struct *vma = vmf->vma;
3607  	struct mmu_notifier_range range;
3608  	vm_fault_t ret;
3609  
3610  	/*
3611  	 * We need a reference to lock the folio because we don't hold
3612  	 * the PTL so a racing thread can remove the device-exclusive
3613  	 * entry and unmap it. If the folio is free the entry must
3614  	 * have been removed already. If it happens to have already
3615  	 * been re-allocated after being freed all we do is lock and
3616  	 * unlock it.
3617  	 */
3618  	if (!folio_try_get(folio))
3619  		return 0;
3620  
3621  	ret = folio_lock_or_retry(folio, vmf);
3622  	if (ret) {
3623  		folio_put(folio);
3624  		return ret;
3625  	}
3626  	mmu_notifier_range_init_owner(&range, MMU_NOTIFY_EXCLUSIVE, 0,
3627  				vma->vm_mm, vmf->address & PAGE_MASK,
3628  				(vmf->address & PAGE_MASK) + PAGE_SIZE, NULL);
3629  	mmu_notifier_invalidate_range_start(&range);
3630  
3631  	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
3632  				&vmf->ptl);
3633  	if (likely(vmf->pte && pte_same(ptep_get(vmf->pte), vmf->orig_pte)))
3634  		restore_exclusive_pte(vma, vmf->page, vmf->address, vmf->pte);
3635  
3636  	if (vmf->pte)
3637  		pte_unmap_unlock(vmf->pte, vmf->ptl);
3638  	folio_unlock(folio);
3639  	folio_put(folio);
3640  
3641  	mmu_notifier_invalidate_range_end(&range);
3642  	return 0;
3643  }
3644  
should_try_to_free_swap(struct folio * folio,struct vm_area_struct * vma,unsigned int fault_flags)3645  static inline bool should_try_to_free_swap(struct folio *folio,
3646  					   struct vm_area_struct *vma,
3647  					   unsigned int fault_flags)
3648  {
3649  	if (!folio_test_swapcache(folio))
3650  		return false;
3651  	if (mem_cgroup_swap_full(folio) || (vma->vm_flags & VM_LOCKED) ||
3652  	    folio_test_mlocked(folio))
3653  		return true;
3654  	/*
3655  	 * If we want to map a page that's in the swapcache writable, we
3656  	 * have to detect via the refcount if we're really the exclusive
3657  	 * user. Try freeing the swapcache to get rid of the swapcache
3658  	 * reference only in case it's likely that we'll be the exlusive user.
3659  	 */
3660  	return (fault_flags & FAULT_FLAG_WRITE) && !folio_test_ksm(folio) &&
3661  		folio_ref_count(folio) == 2;
3662  }
3663  
pte_marker_clear(struct vm_fault * vmf)3664  static vm_fault_t pte_marker_clear(struct vm_fault *vmf)
3665  {
3666  	vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd,
3667  				       vmf->address, &vmf->ptl);
3668  	if (!vmf->pte)
3669  		return 0;
3670  	/*
3671  	 * Be careful so that we will only recover a special uffd-wp pte into a
3672  	 * none pte.  Otherwise it means the pte could have changed, so retry.
3673  	 *
3674  	 * This should also cover the case where e.g. the pte changed
3675  	 * quickly from a PTE_MARKER_UFFD_WP into PTE_MARKER_POISONED.
3676  	 * So is_pte_marker() check is not enough to safely drop the pte.
3677  	 */
3678  	if (pte_same(vmf->orig_pte, ptep_get(vmf->pte)))
3679  		pte_clear(vmf->vma->vm_mm, vmf->address, vmf->pte);
3680  	pte_unmap_unlock(vmf->pte, vmf->ptl);
3681  	return 0;
3682  }
3683  
do_pte_missing(struct vm_fault * vmf)3684  static vm_fault_t do_pte_missing(struct vm_fault *vmf)
3685  {
3686  	if (vma_is_anonymous(vmf->vma))
3687  		return do_anonymous_page(vmf);
3688  	else
3689  		return do_fault(vmf);
3690  }
3691  
3692  /*
3693   * This is actually a page-missing access, but with uffd-wp special pte
3694   * installed.  It means this pte was wr-protected before being unmapped.
3695   */
pte_marker_handle_uffd_wp(struct vm_fault * vmf)3696  static vm_fault_t pte_marker_handle_uffd_wp(struct vm_fault *vmf)
3697  {
3698  	/*
3699  	 * Just in case there're leftover special ptes even after the region
3700  	 * got unregistered - we can simply clear them.
3701  	 */
3702  	if (unlikely(!userfaultfd_wp(vmf->vma)))
3703  		return pte_marker_clear(vmf);
3704  
3705  	return do_pte_missing(vmf);
3706  }
3707  
handle_pte_marker(struct vm_fault * vmf)3708  static vm_fault_t handle_pte_marker(struct vm_fault *vmf)
3709  {
3710  	swp_entry_t entry = pte_to_swp_entry(vmf->orig_pte);
3711  	unsigned long marker = pte_marker_get(entry);
3712  
3713  	/*
3714  	 * PTE markers should never be empty.  If anything weird happened,
3715  	 * the best thing to do is to kill the process along with its mm.
3716  	 */
3717  	if (WARN_ON_ONCE(!marker))
3718  		return VM_FAULT_SIGBUS;
3719  
3720  	/* Higher priority than uffd-wp when data corrupted */
3721  	if (marker & PTE_MARKER_POISONED)
3722  		return VM_FAULT_HWPOISON;
3723  
3724  	if (pte_marker_entry_uffd_wp(entry))
3725  		return pte_marker_handle_uffd_wp(vmf);
3726  
3727  	/* This is an unknown pte marker */
3728  	return VM_FAULT_SIGBUS;
3729  }
3730  
3731  /*
3732   * We enter with non-exclusive mmap_lock (to exclude vma changes,
3733   * but allow concurrent faults), and pte mapped but not yet locked.
3734   * We return with pte unmapped and unlocked.
3735   *
3736   * We return with the mmap_lock locked or unlocked in the same cases
3737   * as does filemap_fault().
3738   */
do_swap_page(struct vm_fault * vmf)3739  vm_fault_t do_swap_page(struct vm_fault *vmf)
3740  {
3741  	struct vm_area_struct *vma = vmf->vma;
3742  	struct folio *swapcache, *folio = NULL;
3743  	struct page *page;
3744  	struct swap_info_struct *si = NULL;
3745  	rmap_t rmap_flags = RMAP_NONE;
3746  	bool need_clear_cache = false;
3747  	bool exclusive = false;
3748  	swp_entry_t entry;
3749  	pte_t pte;
3750  	vm_fault_t ret = 0;
3751  	void *shadow = NULL;
3752  
3753  	if (!pte_unmap_same(vmf))
3754  		goto out;
3755  
3756  	entry = pte_to_swp_entry(vmf->orig_pte);
3757  	if (unlikely(non_swap_entry(entry))) {
3758  		if (is_migration_entry(entry)) {
3759  			migration_entry_wait(vma->vm_mm, vmf->pmd,
3760  					     vmf->address);
3761  		} else if (is_device_exclusive_entry(entry)) {
3762  			vmf->page = pfn_swap_entry_to_page(entry);
3763  			ret = remove_device_exclusive_entry(vmf);
3764  		} else if (is_device_private_entry(entry)) {
3765  			if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
3766  				/*
3767  				 * migrate_to_ram is not yet ready to operate
3768  				 * under VMA lock.
3769  				 */
3770  				vma_end_read(vma);
3771  				ret = VM_FAULT_RETRY;
3772  				goto out;
3773  			}
3774  
3775  			vmf->page = pfn_swap_entry_to_page(entry);
3776  			vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
3777  					vmf->address, &vmf->ptl);
3778  			if (unlikely(!vmf->pte ||
3779  				     !pte_same(ptep_get(vmf->pte),
3780  							vmf->orig_pte)))
3781  				goto unlock;
3782  
3783  			/*
3784  			 * Get a page reference while we know the page can't be
3785  			 * freed.
3786  			 */
3787  			get_page(vmf->page);
3788  			pte_unmap_unlock(vmf->pte, vmf->ptl);
3789  			ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
3790  			put_page(vmf->page);
3791  		} else if (is_hwpoison_entry(entry)) {
3792  			ret = VM_FAULT_HWPOISON;
3793  		} else if (is_pte_marker_entry(entry)) {
3794  			ret = handle_pte_marker(vmf);
3795  		} else {
3796  			print_bad_pte(vma, vmf->address, vmf->orig_pte, NULL);
3797  			ret = VM_FAULT_SIGBUS;
3798  		}
3799  		goto out;
3800  	}
3801  
3802  	/* Prevent swapoff from happening to us. */
3803  	si = get_swap_device(entry);
3804  	if (unlikely(!si))
3805  		goto out;
3806  
3807  	folio = swap_cache_get_folio(entry, vma, vmf->address);
3808  	if (folio)
3809  		page = folio_file_page(folio, swp_offset(entry));
3810  	swapcache = folio;
3811  
3812  	if (!folio) {
3813  		if (data_race(si->flags & SWP_SYNCHRONOUS_IO) &&
3814  		    __swap_count(entry) == 1) {
3815  			/*
3816  			 * Prevent parallel swapin from proceeding with
3817  			 * the cache flag. Otherwise, another thread may
3818  			 * finish swapin first, free the entry, and swapout
3819  			 * reusing the same entry. It's undetectable as
3820  			 * pte_same() returns true due to entry reuse.
3821  			 */
3822  			if (swapcache_prepare(entry)) {
3823  				/* Relax a bit to prevent rapid repeated page faults */
3824  				schedule_timeout_uninterruptible(1);
3825  				goto out;
3826  			}
3827  			need_clear_cache = true;
3828  
3829  			/* skip swapcache */
3830  			folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0,
3831  						vma, vmf->address, false);
3832  			page = &folio->page;
3833  			if (folio) {
3834  				__folio_set_locked(folio);
3835  				__folio_set_swapbacked(folio);
3836  
3837  				if (mem_cgroup_swapin_charge_folio(folio,
3838  							vma->vm_mm, GFP_KERNEL,
3839  							entry)) {
3840  					ret = VM_FAULT_OOM;
3841  					goto out_page;
3842  				}
3843  				mem_cgroup_swapin_uncharge_swap(entry);
3844  
3845  				shadow = get_shadow_from_swap_cache(entry);
3846  				if (shadow)
3847  					workingset_refault(folio, shadow);
3848  
3849  				folio_add_lru(folio);
3850  
3851  				/* To provide entry to swap_readpage() */
3852  				folio->swap = entry;
3853  				swap_readpage(page, true, NULL);
3854  				folio->private = NULL;
3855  			}
3856  		} else {
3857  			page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE,
3858  						vmf);
3859  			if (page)
3860  				folio = page_folio(page);
3861  			swapcache = folio;
3862  		}
3863  
3864  		if (!folio) {
3865  			/*
3866  			 * Back out if somebody else faulted in this pte
3867  			 * while we released the pte lock.
3868  			 */
3869  			vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
3870  					vmf->address, &vmf->ptl);
3871  			if (likely(vmf->pte &&
3872  				   pte_same(ptep_get(vmf->pte), vmf->orig_pte)))
3873  				ret = VM_FAULT_OOM;
3874  			goto unlock;
3875  		}
3876  
3877  		/* Had to read the page from swap area: Major fault */
3878  		ret = VM_FAULT_MAJOR;
3879  		count_vm_event(PGMAJFAULT);
3880  		count_memcg_event_mm(vma->vm_mm, PGMAJFAULT);
3881  	} else if (PageHWPoison(page)) {
3882  		/*
3883  		 * hwpoisoned dirty swapcache pages are kept for killing
3884  		 * owner processes (which may be unknown at hwpoison time)
3885  		 */
3886  		ret = VM_FAULT_HWPOISON;
3887  		goto out_release;
3888  	}
3889  
3890  	ret |= folio_lock_or_retry(folio, vmf);
3891  	if (ret & VM_FAULT_RETRY)
3892  		goto out_release;
3893  
3894  	if (swapcache) {
3895  		/*
3896  		 * Make sure folio_free_swap() or swapoff did not release the
3897  		 * swapcache from under us.  The page pin, and pte_same test
3898  		 * below, are not enough to exclude that.  Even if it is still
3899  		 * swapcache, we need to check that the page's swap has not
3900  		 * changed.
3901  		 */
3902  		if (unlikely(!folio_test_swapcache(folio) ||
3903  			     page_swap_entry(page).val != entry.val))
3904  			goto out_page;
3905  
3906  		/*
3907  		 * KSM sometimes has to copy on read faults, for example, if
3908  		 * page->index of !PageKSM() pages would be nonlinear inside the
3909  		 * anon VMA -- PageKSM() is lost on actual swapout.
3910  		 */
3911  		page = ksm_might_need_to_copy(page, vma, vmf->address);
3912  		if (unlikely(!page)) {
3913  			ret = VM_FAULT_OOM;
3914  			goto out_page;
3915  		} else if (unlikely(PTR_ERR(page) == -EHWPOISON)) {
3916  			ret = VM_FAULT_HWPOISON;
3917  			goto out_page;
3918  		}
3919  		folio = page_folio(page);
3920  
3921  		/*
3922  		 * If we want to map a page that's in the swapcache writable, we
3923  		 * have to detect via the refcount if we're really the exclusive
3924  		 * owner. Try removing the extra reference from the local LRU
3925  		 * caches if required.
3926  		 */
3927  		if ((vmf->flags & FAULT_FLAG_WRITE) && folio == swapcache &&
3928  		    !folio_test_ksm(folio) && !folio_test_lru(folio))
3929  			lru_add_drain();
3930  	}
3931  
3932  	folio_throttle_swaprate(folio, GFP_KERNEL);
3933  
3934  	/*
3935  	 * Back out if somebody else already faulted in this pte.
3936  	 */
3937  	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
3938  			&vmf->ptl);
3939  	if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte)))
3940  		goto out_nomap;
3941  
3942  	if (unlikely(!folio_test_uptodate(folio))) {
3943  		ret = VM_FAULT_SIGBUS;
3944  		goto out_nomap;
3945  	}
3946  
3947  	/*
3948  	 * PG_anon_exclusive reuses PG_mappedtodisk for anon pages. A swap pte
3949  	 * must never point at an anonymous page in the swapcache that is
3950  	 * PG_anon_exclusive. Sanity check that this holds and especially, that
3951  	 * no filesystem set PG_mappedtodisk on a page in the swapcache. Sanity
3952  	 * check after taking the PT lock and making sure that nobody
3953  	 * concurrently faulted in this page and set PG_anon_exclusive.
3954  	 */
3955  	BUG_ON(!folio_test_anon(folio) && folio_test_mappedtodisk(folio));
3956  	BUG_ON(folio_test_anon(folio) && PageAnonExclusive(page));
3957  
3958  	/*
3959  	 * Check under PT lock (to protect against concurrent fork() sharing
3960  	 * the swap entry concurrently) for certainly exclusive pages.
3961  	 */
3962  	if (!folio_test_ksm(folio)) {
3963  		exclusive = pte_swp_exclusive(vmf->orig_pte);
3964  		if (folio != swapcache) {
3965  			/*
3966  			 * We have a fresh page that is not exposed to the
3967  			 * swapcache -> certainly exclusive.
3968  			 */
3969  			exclusive = true;
3970  		} else if (exclusive && folio_test_writeback(folio) &&
3971  			  data_race(si->flags & SWP_STABLE_WRITES)) {
3972  			/*
3973  			 * This is tricky: not all swap backends support
3974  			 * concurrent page modifications while under writeback.
3975  			 *
3976  			 * So if we stumble over such a page in the swapcache
3977  			 * we must not set the page exclusive, otherwise we can
3978  			 * map it writable without further checks and modify it
3979  			 * while still under writeback.
3980  			 *
3981  			 * For these problematic swap backends, simply drop the
3982  			 * exclusive marker: this is perfectly fine as we start
3983  			 * writeback only if we fully unmapped the page and
3984  			 * there are no unexpected references on the page after
3985  			 * unmapping succeeded. After fully unmapped, no
3986  			 * further GUP references (FOLL_GET and FOLL_PIN) can
3987  			 * appear, so dropping the exclusive marker and mapping
3988  			 * it only R/O is fine.
3989  			 */
3990  			exclusive = false;
3991  		}
3992  	}
3993  
3994  	/*
3995  	 * Some architectures may have to restore extra metadata to the page
3996  	 * when reading from swap. This metadata may be indexed by swap entry
3997  	 * so this must be called before swap_free().
3998  	 */
3999  	arch_swap_restore(entry, folio);
4000  
4001  	/*
4002  	 * Remove the swap entry and conditionally try to free up the swapcache.
4003  	 * We're already holding a reference on the page but haven't mapped it
4004  	 * yet.
4005  	 */
4006  	swap_free(entry);
4007  	if (should_try_to_free_swap(folio, vma, vmf->flags))
4008  		folio_free_swap(folio);
4009  
4010  	inc_mm_counter(vma->vm_mm, MM_ANONPAGES);
4011  	dec_mm_counter(vma->vm_mm, MM_SWAPENTS);
4012  	pte = mk_pte(page, vma->vm_page_prot);
4013  
4014  	/*
4015  	 * Same logic as in do_wp_page(); however, optimize for pages that are
4016  	 * certainly not shared either because we just allocated them without
4017  	 * exposing them to the swapcache or because the swap entry indicates
4018  	 * exclusivity.
4019  	 */
4020  	if (!folio_test_ksm(folio) &&
4021  	    (exclusive || folio_ref_count(folio) == 1)) {
4022  		if (vmf->flags & FAULT_FLAG_WRITE) {
4023  			pte = maybe_mkwrite(pte_mkdirty(pte), vma);
4024  			vmf->flags &= ~FAULT_FLAG_WRITE;
4025  		}
4026  		rmap_flags |= RMAP_EXCLUSIVE;
4027  	}
4028  	flush_icache_page(vma, page);
4029  	if (pte_swp_soft_dirty(vmf->orig_pte))
4030  		pte = pte_mksoft_dirty(pte);
4031  	if (pte_swp_uffd_wp(vmf->orig_pte))
4032  		pte = pte_mkuffd_wp(pte);
4033  	vmf->orig_pte = pte;
4034  
4035  	/* ksm created a completely new copy */
4036  	if (unlikely(folio != swapcache && swapcache)) {
4037  		page_add_new_anon_rmap(page, vma, vmf->address);
4038  		folio_add_lru_vma(folio, vma);
4039  	} else {
4040  		page_add_anon_rmap(page, vma, vmf->address, rmap_flags);
4041  	}
4042  
4043  	VM_BUG_ON(!folio_test_anon(folio) ||
4044  			(pte_write(pte) && !PageAnonExclusive(page)));
4045  	set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte);
4046  	arch_do_swap_page(vma->vm_mm, vma, vmf->address, pte, vmf->orig_pte);
4047  
4048  	folio_unlock(folio);
4049  	if (folio != swapcache && swapcache) {
4050  		/*
4051  		 * Hold the lock to avoid the swap entry to be reused
4052  		 * until we take the PT lock for the pte_same() check
4053  		 * (to avoid false positives from pte_same). For
4054  		 * further safety release the lock after the swap_free
4055  		 * so that the swap count won't change under a
4056  		 * parallel locked swapcache.
4057  		 */
4058  		folio_unlock(swapcache);
4059  		folio_put(swapcache);
4060  	}
4061  
4062  	if (vmf->flags & FAULT_FLAG_WRITE) {
4063  		ret |= do_wp_page(vmf);
4064  		if (ret & VM_FAULT_ERROR)
4065  			ret &= VM_FAULT_ERROR;
4066  		goto out;
4067  	}
4068  
4069  	/* No need to invalidate - it was non-present before */
4070  	update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
4071  unlock:
4072  	if (vmf->pte)
4073  		pte_unmap_unlock(vmf->pte, vmf->ptl);
4074  out:
4075  	/* Clear the swap cache pin for direct swapin after PTL unlock */
4076  	if (need_clear_cache)
4077  		swapcache_clear(si, entry);
4078  	if (si)
4079  		put_swap_device(si);
4080  	return ret;
4081  out_nomap:
4082  	if (vmf->pte)
4083  		pte_unmap_unlock(vmf->pte, vmf->ptl);
4084  out_page:
4085  	folio_unlock(folio);
4086  out_release:
4087  	folio_put(folio);
4088  	if (folio != swapcache && swapcache) {
4089  		folio_unlock(swapcache);
4090  		folio_put(swapcache);
4091  	}
4092  	if (need_clear_cache)
4093  		swapcache_clear(si, entry);
4094  	if (si)
4095  		put_swap_device(si);
4096  	return ret;
4097  }
4098  
4099  /*
4100   * We enter with non-exclusive mmap_lock (to exclude vma changes,
4101   * but allow concurrent faults), and pte mapped but not yet locked.
4102   * We return with mmap_lock still held, but pte unmapped and unlocked.
4103   */
do_anonymous_page(struct vm_fault * vmf)4104  static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
4105  {
4106  	bool uffd_wp = vmf_orig_pte_uffd_wp(vmf);
4107  	struct vm_area_struct *vma = vmf->vma;
4108  	struct folio *folio;
4109  	vm_fault_t ret = 0;
4110  	pte_t entry;
4111  
4112  	/* File mapping without ->vm_ops ? */
4113  	if (vma->vm_flags & VM_SHARED)
4114  		return VM_FAULT_SIGBUS;
4115  
4116  	/*
4117  	 * Use pte_alloc() instead of pte_alloc_map(), so that OOM can
4118  	 * be distinguished from a transient failure of pte_offset_map().
4119  	 */
4120  	if (pte_alloc(vma->vm_mm, vmf->pmd))
4121  		return VM_FAULT_OOM;
4122  
4123  	/* Use the zero-page for reads */
4124  	if (!(vmf->flags & FAULT_FLAG_WRITE) &&
4125  			!mm_forbids_zeropage(vma->vm_mm)) {
4126  		entry = pte_mkspecial(pfn_pte(my_zero_pfn(vmf->address),
4127  						vma->vm_page_prot));
4128  		vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
4129  				vmf->address, &vmf->ptl);
4130  		if (!vmf->pte)
4131  			goto unlock;
4132  		if (vmf_pte_changed(vmf)) {
4133  			update_mmu_tlb(vma, vmf->address, vmf->pte);
4134  			goto unlock;
4135  		}
4136  		ret = check_stable_address_space(vma->vm_mm);
4137  		if (ret)
4138  			goto unlock;
4139  		/* Deliver the page fault to userland, check inside PT lock */
4140  		if (userfaultfd_missing(vma)) {
4141  			pte_unmap_unlock(vmf->pte, vmf->ptl);
4142  			return handle_userfault(vmf, VM_UFFD_MISSING);
4143  		}
4144  		goto setpte;
4145  	}
4146  
4147  	/* Allocate our own private page. */
4148  	if (unlikely(anon_vma_prepare(vma)))
4149  		goto oom;
4150  	folio = vma_alloc_zeroed_movable_folio(vma, vmf->address);
4151  	if (!folio)
4152  		goto oom;
4153  
4154  	if (mem_cgroup_charge(folio, vma->vm_mm, GFP_KERNEL))
4155  		goto oom_free_page;
4156  	folio_throttle_swaprate(folio, GFP_KERNEL);
4157  
4158  	/*
4159  	 * The memory barrier inside __folio_mark_uptodate makes sure that
4160  	 * preceding stores to the page contents become visible before
4161  	 * the set_pte_at() write.
4162  	 */
4163  	__folio_mark_uptodate(folio);
4164  
4165  	entry = mk_pte(&folio->page, vma->vm_page_prot);
4166  	entry = pte_sw_mkyoung(entry);
4167  	if (vma->vm_flags & VM_WRITE)
4168  		entry = pte_mkwrite(pte_mkdirty(entry), vma);
4169  
4170  	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
4171  			&vmf->ptl);
4172  	if (!vmf->pte)
4173  		goto release;
4174  	if (vmf_pte_changed(vmf)) {
4175  		update_mmu_tlb(vma, vmf->address, vmf->pte);
4176  		goto release;
4177  	}
4178  
4179  	ret = check_stable_address_space(vma->vm_mm);
4180  	if (ret)
4181  		goto release;
4182  
4183  	/* Deliver the page fault to userland, check inside PT lock */
4184  	if (userfaultfd_missing(vma)) {
4185  		pte_unmap_unlock(vmf->pte, vmf->ptl);
4186  		folio_put(folio);
4187  		return handle_userfault(vmf, VM_UFFD_MISSING);
4188  	}
4189  
4190  	inc_mm_counter(vma->vm_mm, MM_ANONPAGES);
4191  	folio_add_new_anon_rmap(folio, vma, vmf->address);
4192  	folio_add_lru_vma(folio, vma);
4193  setpte:
4194  	if (uffd_wp)
4195  		entry = pte_mkuffd_wp(entry);
4196  	set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry);
4197  
4198  	/* No need to invalidate - it was non-present before */
4199  	update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
4200  unlock:
4201  	if (vmf->pte)
4202  		pte_unmap_unlock(vmf->pte, vmf->ptl);
4203  	return ret;
4204  release:
4205  	folio_put(folio);
4206  	goto unlock;
4207  oom_free_page:
4208  	folio_put(folio);
4209  oom:
4210  	return VM_FAULT_OOM;
4211  }
4212  
4213  /*
4214   * The mmap_lock must have been held on entry, and may have been
4215   * released depending on flags and vma->vm_ops->fault() return value.
4216   * See filemap_fault() and __lock_page_retry().
4217   */
__do_fault(struct vm_fault * vmf)4218  static vm_fault_t __do_fault(struct vm_fault *vmf)
4219  {
4220  	struct vm_area_struct *vma = vmf->vma;
4221  	vm_fault_t ret;
4222  
4223  	/*
4224  	 * Preallocate pte before we take page_lock because this might lead to
4225  	 * deadlocks for memcg reclaim which waits for pages under writeback:
4226  	 *				lock_page(A)
4227  	 *				SetPageWriteback(A)
4228  	 *				unlock_page(A)
4229  	 * lock_page(B)
4230  	 *				lock_page(B)
4231  	 * pte_alloc_one
4232  	 *   shrink_page_list
4233  	 *     wait_on_page_writeback(A)
4234  	 *				SetPageWriteback(B)
4235  	 *				unlock_page(B)
4236  	 *				# flush A, B to clear the writeback
4237  	 */
4238  	if (pmd_none(*vmf->pmd) && !vmf->prealloc_pte) {
4239  		vmf->prealloc_pte = pte_alloc_one(vma->vm_mm);
4240  		if (!vmf->prealloc_pte)
4241  			return VM_FAULT_OOM;
4242  	}
4243  
4244  	ret = vma->vm_ops->fault(vmf);
4245  	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY |
4246  			    VM_FAULT_DONE_COW)))
4247  		return ret;
4248  
4249  	if (unlikely(PageHWPoison(vmf->page))) {
4250  		struct page *page = vmf->page;
4251  		vm_fault_t poisonret = VM_FAULT_HWPOISON;
4252  		if (ret & VM_FAULT_LOCKED) {
4253  			if (page_mapped(page))
4254  				unmap_mapping_pages(page_mapping(page),
4255  						    page->index, 1, false);
4256  			/* Retry if a clean page was removed from the cache. */
4257  			if (invalidate_inode_page(page))
4258  				poisonret = VM_FAULT_NOPAGE;
4259  			unlock_page(page);
4260  		}
4261  		put_page(page);
4262  		vmf->page = NULL;
4263  		return poisonret;
4264  	}
4265  
4266  	if (unlikely(!(ret & VM_FAULT_LOCKED)))
4267  		lock_page(vmf->page);
4268  	else
4269  		VM_BUG_ON_PAGE(!PageLocked(vmf->page), vmf->page);
4270  
4271  	return ret;
4272  }
4273  
4274  #ifdef CONFIG_TRANSPARENT_HUGEPAGE
deposit_prealloc_pte(struct vm_fault * vmf)4275  static void deposit_prealloc_pte(struct vm_fault *vmf)
4276  {
4277  	struct vm_area_struct *vma = vmf->vma;
4278  
4279  	pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, vmf->prealloc_pte);
4280  	/*
4281  	 * We are going to consume the prealloc table,
4282  	 * count that as nr_ptes.
4283  	 */
4284  	mm_inc_nr_ptes(vma->vm_mm);
4285  	vmf->prealloc_pte = NULL;
4286  }
4287  
do_set_pmd(struct vm_fault * vmf,struct page * page)4288  vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page)
4289  {
4290  	struct vm_area_struct *vma = vmf->vma;
4291  	bool write = vmf->flags & FAULT_FLAG_WRITE;
4292  	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
4293  	pmd_t entry;
4294  	vm_fault_t ret = VM_FAULT_FALLBACK;
4295  
4296  	/*
4297  	 * It is too late to allocate a small folio, we already have a large
4298  	 * folio in the pagecache: especially s390 KVM cannot tolerate any
4299  	 * PMD mappings, but PTE-mapped THP are fine. So let's simply refuse any
4300  	 * PMD mappings if THPs are disabled.
4301  	 */
4302  	if (thp_disabled_by_hw() || vma_thp_disabled(vma, vma->vm_flags))
4303  		return ret;
4304  
4305  	if (!transhuge_vma_suitable(vma, haddr))
4306  		return ret;
4307  
4308  	page = compound_head(page);
4309  	if (compound_order(page) != HPAGE_PMD_ORDER)
4310  		return ret;
4311  
4312  	/*
4313  	 * Just backoff if any subpage of a THP is corrupted otherwise
4314  	 * the corrupted page may mapped by PMD silently to escape the
4315  	 * check.  This kind of THP just can be PTE mapped.  Access to
4316  	 * the corrupted subpage should trigger SIGBUS as expected.
4317  	 */
4318  	if (unlikely(PageHasHWPoisoned(page)))
4319  		return ret;
4320  
4321  	/*
4322  	 * Archs like ppc64 need additional space to store information
4323  	 * related to pte entry. Use the preallocated table for that.
4324  	 */
4325  	if (arch_needs_pgtable_deposit() && !vmf->prealloc_pte) {
4326  		vmf->prealloc_pte = pte_alloc_one(vma->vm_mm);
4327  		if (!vmf->prealloc_pte)
4328  			return VM_FAULT_OOM;
4329  	}
4330  
4331  	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
4332  	if (unlikely(!pmd_none(*vmf->pmd)))
4333  		goto out;
4334  
4335  	flush_icache_pages(vma, page, HPAGE_PMD_NR);
4336  
4337  	entry = mk_huge_pmd(page, vma->vm_page_prot);
4338  	if (write)
4339  		entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
4340  
4341  	add_mm_counter(vma->vm_mm, mm_counter_file(page), HPAGE_PMD_NR);
4342  	page_add_file_rmap(page, vma, true);
4343  
4344  	/*
4345  	 * deposit and withdraw with pmd lock held
4346  	 */
4347  	if (arch_needs_pgtable_deposit())
4348  		deposit_prealloc_pte(vmf);
4349  
4350  	set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
4351  
4352  	update_mmu_cache_pmd(vma, haddr, vmf->pmd);
4353  
4354  	/* fault is handled */
4355  	ret = 0;
4356  	count_vm_event(THP_FILE_MAPPED);
4357  out:
4358  	spin_unlock(vmf->ptl);
4359  	return ret;
4360  }
4361  #else
do_set_pmd(struct vm_fault * vmf,struct page * page)4362  vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page)
4363  {
4364  	return VM_FAULT_FALLBACK;
4365  }
4366  #endif
4367  
4368  /**
4369   * set_pte_range - Set a range of PTEs to point to pages in a folio.
4370   * @vmf: Fault decription.
4371   * @folio: The folio that contains @page.
4372   * @page: The first page to create a PTE for.
4373   * @nr: The number of PTEs to create.
4374   * @addr: The first address to create a PTE for.
4375   */
set_pte_range(struct vm_fault * vmf,struct folio * folio,struct page * page,unsigned int nr,unsigned long addr)4376  void set_pte_range(struct vm_fault *vmf, struct folio *folio,
4377  		struct page *page, unsigned int nr, unsigned long addr)
4378  {
4379  	struct vm_area_struct *vma = vmf->vma;
4380  	bool uffd_wp = vmf_orig_pte_uffd_wp(vmf);
4381  	bool write = vmf->flags & FAULT_FLAG_WRITE;
4382  	bool prefault = !in_range(vmf->address, addr, nr * PAGE_SIZE);
4383  	pte_t entry;
4384  
4385  	flush_icache_pages(vma, page, nr);
4386  	entry = mk_pte(page, vma->vm_page_prot);
4387  
4388  	if (prefault && arch_wants_old_prefaulted_pte())
4389  		entry = pte_mkold(entry);
4390  	else
4391  		entry = pte_sw_mkyoung(entry);
4392  
4393  	if (write)
4394  		entry = maybe_mkwrite(pte_mkdirty(entry), vma);
4395  	if (unlikely(uffd_wp))
4396  		entry = pte_mkuffd_wp(entry);
4397  	/* copy-on-write page */
4398  	if (write && !(vma->vm_flags & VM_SHARED)) {
4399  		add_mm_counter(vma->vm_mm, MM_ANONPAGES, nr);
4400  		VM_BUG_ON_FOLIO(nr != 1, folio);
4401  		folio_add_new_anon_rmap(folio, vma, addr);
4402  		folio_add_lru_vma(folio, vma);
4403  	} else {
4404  		add_mm_counter(vma->vm_mm, mm_counter_file(page), nr);
4405  		folio_add_file_rmap_range(folio, page, nr, vma, false);
4406  	}
4407  	set_ptes(vma->vm_mm, addr, vmf->pte, entry, nr);
4408  
4409  	/* no need to invalidate: a not-present page won't be cached */
4410  	update_mmu_cache_range(vmf, vma, addr, vmf->pte, nr);
4411  }
4412  
vmf_pte_changed(struct vm_fault * vmf)4413  static bool vmf_pte_changed(struct vm_fault *vmf)
4414  {
4415  	if (vmf->flags & FAULT_FLAG_ORIG_PTE_VALID)
4416  		return !pte_same(ptep_get(vmf->pte), vmf->orig_pte);
4417  
4418  	return !pte_none(ptep_get(vmf->pte));
4419  }
4420  
4421  /**
4422   * finish_fault - finish page fault once we have prepared the page to fault
4423   *
4424   * @vmf: structure describing the fault
4425   *
4426   * This function handles all that is needed to finish a page fault once the
4427   * page to fault in is prepared. It handles locking of PTEs, inserts PTE for
4428   * given page, adds reverse page mapping, handles memcg charges and LRU
4429   * addition.
4430   *
4431   * The function expects the page to be locked and on success it consumes a
4432   * reference of a page being mapped (for the PTE which maps it).
4433   *
4434   * Return: %0 on success, %VM_FAULT_ code in case of error.
4435   */
finish_fault(struct vm_fault * vmf)4436  vm_fault_t finish_fault(struct vm_fault *vmf)
4437  {
4438  	struct vm_area_struct *vma = vmf->vma;
4439  	struct page *page;
4440  	vm_fault_t ret;
4441  
4442  	/* Did we COW the page? */
4443  	if ((vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED))
4444  		page = vmf->cow_page;
4445  	else
4446  		page = vmf->page;
4447  
4448  	/*
4449  	 * check even for read faults because we might have lost our CoWed
4450  	 * page
4451  	 */
4452  	if (!(vma->vm_flags & VM_SHARED)) {
4453  		ret = check_stable_address_space(vma->vm_mm);
4454  		if (ret)
4455  			return ret;
4456  	}
4457  
4458  	if (pmd_none(*vmf->pmd)) {
4459  		if (PageTransCompound(page)) {
4460  			ret = do_set_pmd(vmf, page);
4461  			if (ret != VM_FAULT_FALLBACK)
4462  				return ret;
4463  		}
4464  
4465  		if (vmf->prealloc_pte)
4466  			pmd_install(vma->vm_mm, vmf->pmd, &vmf->prealloc_pte);
4467  		else if (unlikely(pte_alloc(vma->vm_mm, vmf->pmd)))
4468  			return VM_FAULT_OOM;
4469  	}
4470  
4471  	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
4472  				      vmf->address, &vmf->ptl);
4473  	if (!vmf->pte)
4474  		return VM_FAULT_NOPAGE;
4475  
4476  	/* Re-check under ptl */
4477  	if (likely(!vmf_pte_changed(vmf))) {
4478  		struct folio *folio = page_folio(page);
4479  
4480  		set_pte_range(vmf, folio, page, 1, vmf->address);
4481  		ret = 0;
4482  	} else {
4483  		update_mmu_tlb(vma, vmf->address, vmf->pte);
4484  		ret = VM_FAULT_NOPAGE;
4485  	}
4486  
4487  	pte_unmap_unlock(vmf->pte, vmf->ptl);
4488  	return ret;
4489  }
4490  
4491  static unsigned long fault_around_pages __read_mostly =
4492  	65536 >> PAGE_SHIFT;
4493  
4494  #ifdef CONFIG_DEBUG_FS
fault_around_bytes_get(void * data,u64 * val)4495  static int fault_around_bytes_get(void *data, u64 *val)
4496  {
4497  	*val = fault_around_pages << PAGE_SHIFT;
4498  	return 0;
4499  }
4500  
4501  /*
4502   * fault_around_bytes must be rounded down to the nearest page order as it's
4503   * what do_fault_around() expects to see.
4504   */
fault_around_bytes_set(void * data,u64 val)4505  static int fault_around_bytes_set(void *data, u64 val)
4506  {
4507  	if (val / PAGE_SIZE > PTRS_PER_PTE)
4508  		return -EINVAL;
4509  
4510  	/*
4511  	 * The minimum value is 1 page, however this results in no fault-around
4512  	 * at all. See should_fault_around().
4513  	 */
4514  	fault_around_pages = max(rounddown_pow_of_two(val) >> PAGE_SHIFT, 1UL);
4515  
4516  	return 0;
4517  }
4518  DEFINE_DEBUGFS_ATTRIBUTE(fault_around_bytes_fops,
4519  		fault_around_bytes_get, fault_around_bytes_set, "%llu\n");
4520  
fault_around_debugfs(void)4521  static int __init fault_around_debugfs(void)
4522  {
4523  	debugfs_create_file_unsafe("fault_around_bytes", 0644, NULL, NULL,
4524  				   &fault_around_bytes_fops);
4525  	return 0;
4526  }
4527  late_initcall(fault_around_debugfs);
4528  #endif
4529  
4530  /*
4531   * do_fault_around() tries to map few pages around the fault address. The hope
4532   * is that the pages will be needed soon and this will lower the number of
4533   * faults to handle.
4534   *
4535   * It uses vm_ops->map_pages() to map the pages, which skips the page if it's
4536   * not ready to be mapped: not up-to-date, locked, etc.
4537   *
4538   * This function doesn't cross VMA or page table boundaries, in order to call
4539   * map_pages() and acquire a PTE lock only once.
4540   *
4541   * fault_around_pages defines how many pages we'll try to map.
4542   * do_fault_around() expects it to be set to a power of two less than or equal
4543   * to PTRS_PER_PTE.
4544   *
4545   * The virtual address of the area that we map is naturally aligned to
4546   * fault_around_pages * PAGE_SIZE rounded down to the machine page size
4547   * (and therefore to page order).  This way it's easier to guarantee
4548   * that we don't cross page table boundaries.
4549   */
do_fault_around(struct vm_fault * vmf)4550  static vm_fault_t do_fault_around(struct vm_fault *vmf)
4551  {
4552  	pgoff_t nr_pages = READ_ONCE(fault_around_pages);
4553  	pgoff_t pte_off = pte_index(vmf->address);
4554  	/* The page offset of vmf->address within the VMA. */
4555  	pgoff_t vma_off = vmf->pgoff - vmf->vma->vm_pgoff;
4556  	pgoff_t from_pte, to_pte;
4557  	vm_fault_t ret;
4558  
4559  	/* The PTE offset of the start address, clamped to the VMA. */
4560  	from_pte = max(ALIGN_DOWN(pte_off, nr_pages),
4561  		       pte_off - min(pte_off, vma_off));
4562  
4563  	/* The PTE offset of the end address, clamped to the VMA and PTE. */
4564  	to_pte = min3(from_pte + nr_pages, (pgoff_t)PTRS_PER_PTE,
4565  		      pte_off + vma_pages(vmf->vma) - vma_off) - 1;
4566  
4567  	if (pmd_none(*vmf->pmd)) {
4568  		vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm);
4569  		if (!vmf->prealloc_pte)
4570  			return VM_FAULT_OOM;
4571  	}
4572  
4573  	rcu_read_lock();
4574  	ret = vmf->vma->vm_ops->map_pages(vmf,
4575  			vmf->pgoff + from_pte - pte_off,
4576  			vmf->pgoff + to_pte - pte_off);
4577  	rcu_read_unlock();
4578  
4579  	return ret;
4580  }
4581  
4582  /* Return true if we should do read fault-around, false otherwise */
should_fault_around(struct vm_fault * vmf)4583  static inline bool should_fault_around(struct vm_fault *vmf)
4584  {
4585  	/* No ->map_pages?  No way to fault around... */
4586  	if (!vmf->vma->vm_ops->map_pages)
4587  		return false;
4588  
4589  	if (uffd_disable_fault_around(vmf->vma))
4590  		return false;
4591  
4592  	/* A single page implies no faulting 'around' at all. */
4593  	return fault_around_pages > 1;
4594  }
4595  
do_read_fault(struct vm_fault * vmf)4596  static vm_fault_t do_read_fault(struct vm_fault *vmf)
4597  {
4598  	vm_fault_t ret = 0;
4599  	struct folio *folio;
4600  
4601  	/*
4602  	 * Let's call ->map_pages() first and use ->fault() as fallback
4603  	 * if page by the offset is not ready to be mapped (cold cache or
4604  	 * something).
4605  	 */
4606  	if (should_fault_around(vmf)) {
4607  		ret = do_fault_around(vmf);
4608  		if (ret)
4609  			return ret;
4610  	}
4611  
4612  	if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
4613  		vma_end_read(vmf->vma);
4614  		return VM_FAULT_RETRY;
4615  	}
4616  
4617  	ret = __do_fault(vmf);
4618  	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
4619  		return ret;
4620  
4621  	ret |= finish_fault(vmf);
4622  	folio = page_folio(vmf->page);
4623  	folio_unlock(folio);
4624  	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
4625  		folio_put(folio);
4626  	return ret;
4627  }
4628  
do_cow_fault(struct vm_fault * vmf)4629  static vm_fault_t do_cow_fault(struct vm_fault *vmf)
4630  {
4631  	struct vm_area_struct *vma = vmf->vma;
4632  	vm_fault_t ret;
4633  
4634  	if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
4635  		vma_end_read(vma);
4636  		return VM_FAULT_RETRY;
4637  	}
4638  
4639  	if (unlikely(anon_vma_prepare(vma)))
4640  		return VM_FAULT_OOM;
4641  
4642  	vmf->cow_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vmf->address);
4643  	if (!vmf->cow_page)
4644  		return VM_FAULT_OOM;
4645  
4646  	if (mem_cgroup_charge(page_folio(vmf->cow_page), vma->vm_mm,
4647  				GFP_KERNEL)) {
4648  		put_page(vmf->cow_page);
4649  		return VM_FAULT_OOM;
4650  	}
4651  	folio_throttle_swaprate(page_folio(vmf->cow_page), GFP_KERNEL);
4652  
4653  	ret = __do_fault(vmf);
4654  	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
4655  		goto uncharge_out;
4656  	if (ret & VM_FAULT_DONE_COW)
4657  		return ret;
4658  
4659  	copy_user_highpage(vmf->cow_page, vmf->page, vmf->address, vma);
4660  	__SetPageUptodate(vmf->cow_page);
4661  
4662  	ret |= finish_fault(vmf);
4663  	unlock_page(vmf->page);
4664  	put_page(vmf->page);
4665  	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
4666  		goto uncharge_out;
4667  	return ret;
4668  uncharge_out:
4669  	put_page(vmf->cow_page);
4670  	return ret;
4671  }
4672  
do_shared_fault(struct vm_fault * vmf)4673  static vm_fault_t do_shared_fault(struct vm_fault *vmf)
4674  {
4675  	struct vm_area_struct *vma = vmf->vma;
4676  	vm_fault_t ret, tmp;
4677  	struct folio *folio;
4678  
4679  	if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
4680  		vma_end_read(vma);
4681  		return VM_FAULT_RETRY;
4682  	}
4683  
4684  	ret = __do_fault(vmf);
4685  	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
4686  		return ret;
4687  
4688  	folio = page_folio(vmf->page);
4689  
4690  	/*
4691  	 * Check if the backing address space wants to know that the page is
4692  	 * about to become writable
4693  	 */
4694  	if (vma->vm_ops->page_mkwrite) {
4695  		folio_unlock(folio);
4696  		tmp = do_page_mkwrite(vmf, folio);
4697  		if (unlikely(!tmp ||
4698  				(tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
4699  			folio_put(folio);
4700  			return tmp;
4701  		}
4702  	}
4703  
4704  	ret |= finish_fault(vmf);
4705  	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE |
4706  					VM_FAULT_RETRY))) {
4707  		folio_unlock(folio);
4708  		folio_put(folio);
4709  		return ret;
4710  	}
4711  
4712  	ret |= fault_dirty_shared_page(vmf);
4713  	return ret;
4714  }
4715  
4716  /*
4717   * We enter with non-exclusive mmap_lock (to exclude vma changes,
4718   * but allow concurrent faults).
4719   * The mmap_lock may have been released depending on flags and our
4720   * return value.  See filemap_fault() and __folio_lock_or_retry().
4721   * If mmap_lock is released, vma may become invalid (for example
4722   * by other thread calling munmap()).
4723   */
do_fault(struct vm_fault * vmf)4724  static vm_fault_t do_fault(struct vm_fault *vmf)
4725  {
4726  	struct vm_area_struct *vma = vmf->vma;
4727  	struct mm_struct *vm_mm = vma->vm_mm;
4728  	vm_fault_t ret;
4729  
4730  	/*
4731  	 * The VMA was not fully populated on mmap() or missing VM_DONTEXPAND
4732  	 */
4733  	if (!vma->vm_ops->fault) {
4734  		vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd,
4735  					       vmf->address, &vmf->ptl);
4736  		if (unlikely(!vmf->pte))
4737  			ret = VM_FAULT_SIGBUS;
4738  		else {
4739  			/*
4740  			 * Make sure this is not a temporary clearing of pte
4741  			 * by holding ptl and checking again. A R/M/W update
4742  			 * of pte involves: take ptl, clearing the pte so that
4743  			 * we don't have concurrent modification by hardware
4744  			 * followed by an update.
4745  			 */
4746  			if (unlikely(pte_none(ptep_get(vmf->pte))))
4747  				ret = VM_FAULT_SIGBUS;
4748  			else
4749  				ret = VM_FAULT_NOPAGE;
4750  
4751  			pte_unmap_unlock(vmf->pte, vmf->ptl);
4752  		}
4753  	} else if (!(vmf->flags & FAULT_FLAG_WRITE))
4754  		ret = do_read_fault(vmf);
4755  	else if (!(vma->vm_flags & VM_SHARED))
4756  		ret = do_cow_fault(vmf);
4757  	else
4758  		ret = do_shared_fault(vmf);
4759  
4760  	/* preallocated pagetable is unused: free it */
4761  	if (vmf->prealloc_pte) {
4762  		pte_free(vm_mm, vmf->prealloc_pte);
4763  		vmf->prealloc_pte = NULL;
4764  	}
4765  	return ret;
4766  }
4767  
numa_migrate_prep(struct page * page,struct vm_area_struct * vma,unsigned long addr,int page_nid,int * flags)4768  int numa_migrate_prep(struct page *page, struct vm_area_struct *vma,
4769  		      unsigned long addr, int page_nid, int *flags)
4770  {
4771  	get_page(page);
4772  
4773  	/* Record the current PID acceesing VMA */
4774  	vma_set_access_pid_bit(vma);
4775  
4776  	count_vm_numa_event(NUMA_HINT_FAULTS);
4777  	if (page_nid == numa_node_id()) {
4778  		count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
4779  		*flags |= TNF_FAULT_LOCAL;
4780  	}
4781  
4782  	return mpol_misplaced(page, vma, addr);
4783  }
4784  
do_numa_page(struct vm_fault * vmf)4785  static vm_fault_t do_numa_page(struct vm_fault *vmf)
4786  {
4787  	struct vm_area_struct *vma = vmf->vma;
4788  	struct page *page = NULL;
4789  	int page_nid = NUMA_NO_NODE;
4790  	bool writable = false;
4791  	int last_cpupid;
4792  	int target_nid;
4793  	pte_t pte, old_pte;
4794  	int flags = 0;
4795  
4796  	/*
4797  	 * The "pte" at this point cannot be used safely without
4798  	 * validation through pte_unmap_same(). It's of NUMA type but
4799  	 * the pfn may be screwed if the read is non atomic.
4800  	 */
4801  	spin_lock(vmf->ptl);
4802  	if (unlikely(!pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
4803  		pte_unmap_unlock(vmf->pte, vmf->ptl);
4804  		return 0;
4805  	}
4806  
4807  	/* Get the normal PTE  */
4808  	old_pte = ptep_get(vmf->pte);
4809  	pte = pte_modify(old_pte, vma->vm_page_prot);
4810  
4811  	/*
4812  	 * Detect now whether the PTE could be writable; this information
4813  	 * is only valid while holding the PT lock.
4814  	 */
4815  	writable = pte_write(pte);
4816  	if (!writable && vma_wants_manual_pte_write_upgrade(vma) &&
4817  	    can_change_pte_writable(vma, vmf->address, pte))
4818  		writable = true;
4819  
4820  	page = vm_normal_page(vma, vmf->address, pte);
4821  	if (!page || is_zone_device_page(page))
4822  		goto out_map;
4823  
4824  	/* TODO: handle PTE-mapped THP */
4825  	if (PageCompound(page))
4826  		goto out_map;
4827  
4828  	/*
4829  	 * Avoid grouping on RO pages in general. RO pages shouldn't hurt as
4830  	 * much anyway since they can be in shared cache state. This misses
4831  	 * the case where a mapping is writable but the process never writes
4832  	 * to it but pte_write gets cleared during protection updates and
4833  	 * pte_dirty has unpredictable behaviour between PTE scan updates,
4834  	 * background writeback, dirty balancing and application behaviour.
4835  	 */
4836  	if (!writable)
4837  		flags |= TNF_NO_GROUP;
4838  
4839  	/*
4840  	 * Flag if the page is shared between multiple address spaces. This
4841  	 * is later used when determining whether to group tasks together
4842  	 */
4843  	if (page_mapcount(page) > 1 && (vma->vm_flags & VM_SHARED))
4844  		flags |= TNF_SHARED;
4845  
4846  	page_nid = page_to_nid(page);
4847  	/*
4848  	 * For memory tiering mode, cpupid of slow memory page is used
4849  	 * to record page access time.  So use default value.
4850  	 */
4851  	if ((sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) &&
4852  	    !node_is_toptier(page_nid))
4853  		last_cpupid = (-1 & LAST_CPUPID_MASK);
4854  	else
4855  		last_cpupid = page_cpupid_last(page);
4856  	target_nid = numa_migrate_prep(page, vma, vmf->address, page_nid,
4857  			&flags);
4858  	if (target_nid == NUMA_NO_NODE) {
4859  		put_page(page);
4860  		goto out_map;
4861  	}
4862  	pte_unmap_unlock(vmf->pte, vmf->ptl);
4863  	writable = false;
4864  
4865  	/* Migrate to the requested node */
4866  	if (migrate_misplaced_page(page, vma, target_nid)) {
4867  		page_nid = target_nid;
4868  		flags |= TNF_MIGRATED;
4869  		task_numa_fault(last_cpupid, page_nid, 1, flags);
4870  		return 0;
4871  	}
4872  
4873  	flags |= TNF_MIGRATE_FAIL;
4874  	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
4875  				       vmf->address, &vmf->ptl);
4876  	if (unlikely(!vmf->pte))
4877  		return 0;
4878  	if (unlikely(!pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
4879  		pte_unmap_unlock(vmf->pte, vmf->ptl);
4880  		return 0;
4881  	}
4882  out_map:
4883  	/*
4884  	 * Make it present again, depending on how arch implements
4885  	 * non-accessible ptes, some can allow access by kernel mode.
4886  	 */
4887  	old_pte = ptep_modify_prot_start(vma, vmf->address, vmf->pte);
4888  	pte = pte_modify(old_pte, vma->vm_page_prot);
4889  	pte = pte_mkyoung(pte);
4890  	if (writable)
4891  		pte = pte_mkwrite(pte, vma);
4892  	ptep_modify_prot_commit(vma, vmf->address, vmf->pte, old_pte, pte);
4893  	update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
4894  	pte_unmap_unlock(vmf->pte, vmf->ptl);
4895  
4896  	if (page_nid != NUMA_NO_NODE)
4897  		task_numa_fault(last_cpupid, page_nid, 1, flags);
4898  	return 0;
4899  }
4900  
create_huge_pmd(struct vm_fault * vmf)4901  static inline vm_fault_t create_huge_pmd(struct vm_fault *vmf)
4902  {
4903  	struct vm_area_struct *vma = vmf->vma;
4904  	if (vma_is_anonymous(vma))
4905  		return do_huge_pmd_anonymous_page(vmf);
4906  	if (vma->vm_ops->huge_fault)
4907  		return vma->vm_ops->huge_fault(vmf, PMD_ORDER);
4908  	return VM_FAULT_FALLBACK;
4909  }
4910  
4911  /* `inline' is required to avoid gcc 4.1.2 build error */
wp_huge_pmd(struct vm_fault * vmf)4912  static inline vm_fault_t wp_huge_pmd(struct vm_fault *vmf)
4913  {
4914  	struct vm_area_struct *vma = vmf->vma;
4915  	const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
4916  	vm_fault_t ret;
4917  
4918  	if (vma_is_anonymous(vma)) {
4919  		if (likely(!unshare) &&
4920  		    userfaultfd_huge_pmd_wp(vma, vmf->orig_pmd))
4921  			return handle_userfault(vmf, VM_UFFD_WP);
4922  		return do_huge_pmd_wp_page(vmf);
4923  	}
4924  
4925  	if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
4926  		if (vma->vm_ops->huge_fault) {
4927  			ret = vma->vm_ops->huge_fault(vmf, PMD_ORDER);
4928  			if (!(ret & VM_FAULT_FALLBACK))
4929  				return ret;
4930  		}
4931  	}
4932  
4933  	/* COW or write-notify handled on pte level: split pmd. */
4934  	__split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL);
4935  
4936  	return VM_FAULT_FALLBACK;
4937  }
4938  
create_huge_pud(struct vm_fault * vmf)4939  static vm_fault_t create_huge_pud(struct vm_fault *vmf)
4940  {
4941  #if defined(CONFIG_TRANSPARENT_HUGEPAGE) &&			\
4942  	defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
4943  	struct vm_area_struct *vma = vmf->vma;
4944  	/* No support for anonymous transparent PUD pages yet */
4945  	if (vma_is_anonymous(vma))
4946  		return VM_FAULT_FALLBACK;
4947  	if (vma->vm_ops->huge_fault)
4948  		return vma->vm_ops->huge_fault(vmf, PUD_ORDER);
4949  #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
4950  	return VM_FAULT_FALLBACK;
4951  }
4952  
wp_huge_pud(struct vm_fault * vmf,pud_t orig_pud)4953  static vm_fault_t wp_huge_pud(struct vm_fault *vmf, pud_t orig_pud)
4954  {
4955  #if defined(CONFIG_TRANSPARENT_HUGEPAGE) &&			\
4956  	defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
4957  	struct vm_area_struct *vma = vmf->vma;
4958  	vm_fault_t ret;
4959  
4960  	/* No support for anonymous transparent PUD pages yet */
4961  	if (vma_is_anonymous(vma))
4962  		goto split;
4963  	if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
4964  		if (vma->vm_ops->huge_fault) {
4965  			ret = vma->vm_ops->huge_fault(vmf, PUD_ORDER);
4966  			if (!(ret & VM_FAULT_FALLBACK))
4967  				return ret;
4968  		}
4969  	}
4970  split:
4971  	/* COW or write-notify not handled on PUD level: split pud.*/
4972  	__split_huge_pud(vma, vmf->pud, vmf->address);
4973  #endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
4974  	return VM_FAULT_FALLBACK;
4975  }
4976  
4977  /*
4978   * These routines also need to handle stuff like marking pages dirty
4979   * and/or accessed for architectures that don't do it in hardware (most
4980   * RISC architectures).  The early dirtying is also good on the i386.
4981   *
4982   * There is also a hook called "update_mmu_cache()" that architectures
4983   * with external mmu caches can use to update those (ie the Sparc or
4984   * PowerPC hashed page tables that act as extended TLBs).
4985   *
4986   * We enter with non-exclusive mmap_lock (to exclude vma changes, but allow
4987   * concurrent faults).
4988   *
4989   * The mmap_lock may have been released depending on flags and our return value.
4990   * See filemap_fault() and __folio_lock_or_retry().
4991   */
handle_pte_fault(struct vm_fault * vmf)4992  static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
4993  {
4994  	pte_t entry;
4995  
4996  	if (unlikely(pmd_none(*vmf->pmd))) {
4997  		/*
4998  		 * Leave __pte_alloc() until later: because vm_ops->fault may
4999  		 * want to allocate huge page, and if we expose page table
5000  		 * for an instant, it will be difficult to retract from
5001  		 * concurrent faults and from rmap lookups.
5002  		 */
5003  		vmf->pte = NULL;
5004  		vmf->flags &= ~FAULT_FLAG_ORIG_PTE_VALID;
5005  	} else {
5006  		/*
5007  		 * A regular pmd is established and it can't morph into a huge
5008  		 * pmd by anon khugepaged, since that takes mmap_lock in write
5009  		 * mode; but shmem or file collapse to THP could still morph
5010  		 * it into a huge pmd: just retry later if so.
5011  		 */
5012  		vmf->pte = pte_offset_map_nolock(vmf->vma->vm_mm, vmf->pmd,
5013  						 vmf->address, &vmf->ptl);
5014  		if (unlikely(!vmf->pte))
5015  			return 0;
5016  		vmf->orig_pte = ptep_get_lockless(vmf->pte);
5017  		vmf->flags |= FAULT_FLAG_ORIG_PTE_VALID;
5018  
5019  		if (pte_none(vmf->orig_pte)) {
5020  			pte_unmap(vmf->pte);
5021  			vmf->pte = NULL;
5022  		}
5023  	}
5024  
5025  	if (!vmf->pte)
5026  		return do_pte_missing(vmf);
5027  
5028  	if (!pte_present(vmf->orig_pte))
5029  		return do_swap_page(vmf);
5030  
5031  	if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma))
5032  		return do_numa_page(vmf);
5033  
5034  	spin_lock(vmf->ptl);
5035  	entry = vmf->orig_pte;
5036  	if (unlikely(!pte_same(ptep_get(vmf->pte), entry))) {
5037  		update_mmu_tlb(vmf->vma, vmf->address, vmf->pte);
5038  		goto unlock;
5039  	}
5040  	if (vmf->flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) {
5041  		if (!pte_write(entry))
5042  			return do_wp_page(vmf);
5043  		else if (likely(vmf->flags & FAULT_FLAG_WRITE))
5044  			entry = pte_mkdirty(entry);
5045  	}
5046  	entry = pte_mkyoung(entry);
5047  	if (ptep_set_access_flags(vmf->vma, vmf->address, vmf->pte, entry,
5048  				vmf->flags & FAULT_FLAG_WRITE)) {
5049  		update_mmu_cache_range(vmf, vmf->vma, vmf->address,
5050  				vmf->pte, 1);
5051  	} else {
5052  		/* Skip spurious TLB flush for retried page fault */
5053  		if (vmf->flags & FAULT_FLAG_TRIED)
5054  			goto unlock;
5055  		/*
5056  		 * This is needed only for protection faults but the arch code
5057  		 * is not yet telling us if this is a protection fault or not.
5058  		 * This still avoids useless tlb flushes for .text page faults
5059  		 * with threads.
5060  		 */
5061  		if (vmf->flags & FAULT_FLAG_WRITE)
5062  			flush_tlb_fix_spurious_fault(vmf->vma, vmf->address,
5063  						     vmf->pte);
5064  	}
5065  unlock:
5066  	pte_unmap_unlock(vmf->pte, vmf->ptl);
5067  	return 0;
5068  }
5069  
5070  /*
5071   * On entry, we hold either the VMA lock or the mmap_lock
5072   * (FAULT_FLAG_VMA_LOCK tells you which).  If VM_FAULT_RETRY is set in
5073   * the result, the mmap_lock is not held on exit.  See filemap_fault()
5074   * and __folio_lock_or_retry().
5075   */
__handle_mm_fault(struct vm_area_struct * vma,unsigned long address,unsigned int flags)5076  static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
5077  		unsigned long address, unsigned int flags)
5078  {
5079  	struct vm_fault vmf = {
5080  		.vma = vma,
5081  		.address = address & PAGE_MASK,
5082  		.real_address = address,
5083  		.flags = flags,
5084  		.pgoff = linear_page_index(vma, address),
5085  		.gfp_mask = __get_fault_gfp_mask(vma),
5086  	};
5087  	struct mm_struct *mm = vma->vm_mm;
5088  	unsigned long vm_flags = vma->vm_flags;
5089  	pgd_t *pgd;
5090  	p4d_t *p4d;
5091  	vm_fault_t ret;
5092  
5093  	pgd = pgd_offset(mm, address);
5094  	p4d = p4d_alloc(mm, pgd, address);
5095  	if (!p4d)
5096  		return VM_FAULT_OOM;
5097  
5098  	vmf.pud = pud_alloc(mm, p4d, address);
5099  	if (!vmf.pud)
5100  		return VM_FAULT_OOM;
5101  retry_pud:
5102  	if (pud_none(*vmf.pud) &&
5103  	    hugepage_vma_check(vma, vm_flags, false, true, true)) {
5104  		ret = create_huge_pud(&vmf);
5105  		if (!(ret & VM_FAULT_FALLBACK))
5106  			return ret;
5107  	} else {
5108  		pud_t orig_pud = *vmf.pud;
5109  
5110  		barrier();
5111  		if (pud_trans_huge(orig_pud) || pud_devmap(orig_pud)) {
5112  
5113  			/*
5114  			 * TODO once we support anonymous PUDs: NUMA case and
5115  			 * FAULT_FLAG_UNSHARE handling.
5116  			 */
5117  			if ((flags & FAULT_FLAG_WRITE) && !pud_write(orig_pud)) {
5118  				ret = wp_huge_pud(&vmf, orig_pud);
5119  				if (!(ret & VM_FAULT_FALLBACK))
5120  					return ret;
5121  			} else {
5122  				huge_pud_set_accessed(&vmf, orig_pud);
5123  				return 0;
5124  			}
5125  		}
5126  	}
5127  
5128  	vmf.pmd = pmd_alloc(mm, vmf.pud, address);
5129  	if (!vmf.pmd)
5130  		return VM_FAULT_OOM;
5131  
5132  	/* Huge pud page fault raced with pmd_alloc? */
5133  	if (pud_trans_unstable(vmf.pud))
5134  		goto retry_pud;
5135  
5136  	if (pmd_none(*vmf.pmd) &&
5137  	    hugepage_vma_check(vma, vm_flags, false, true, true)) {
5138  		ret = create_huge_pmd(&vmf);
5139  		if (!(ret & VM_FAULT_FALLBACK))
5140  			return ret;
5141  	} else {
5142  		vmf.orig_pmd = pmdp_get_lockless(vmf.pmd);
5143  
5144  		if (unlikely(is_swap_pmd(vmf.orig_pmd))) {
5145  			VM_BUG_ON(thp_migration_supported() &&
5146  					  !is_pmd_migration_entry(vmf.orig_pmd));
5147  			if (is_pmd_migration_entry(vmf.orig_pmd))
5148  				pmd_migration_entry_wait(mm, vmf.pmd);
5149  			return 0;
5150  		}
5151  		if (pmd_trans_huge(vmf.orig_pmd) || pmd_devmap(vmf.orig_pmd)) {
5152  			if (pmd_protnone(vmf.orig_pmd) && vma_is_accessible(vma))
5153  				return do_huge_pmd_numa_page(&vmf);
5154  
5155  			if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) &&
5156  			    !pmd_write(vmf.orig_pmd)) {
5157  				ret = wp_huge_pmd(&vmf);
5158  				if (!(ret & VM_FAULT_FALLBACK))
5159  					return ret;
5160  			} else {
5161  				huge_pmd_set_accessed(&vmf);
5162  				return 0;
5163  			}
5164  		}
5165  	}
5166  
5167  	return handle_pte_fault(&vmf);
5168  }
5169  
5170  /**
5171   * mm_account_fault - Do page fault accounting
5172   * @mm: mm from which memcg should be extracted. It can be NULL.
5173   * @regs: the pt_regs struct pointer.  When set to NULL, will skip accounting
5174   *        of perf event counters, but we'll still do the per-task accounting to
5175   *        the task who triggered this page fault.
5176   * @address: the faulted address.
5177   * @flags: the fault flags.
5178   * @ret: the fault retcode.
5179   *
5180   * This will take care of most of the page fault accounting.  Meanwhile, it
5181   * will also include the PERF_COUNT_SW_PAGE_FAULTS_[MAJ|MIN] perf counter
5182   * updates.  However, note that the handling of PERF_COUNT_SW_PAGE_FAULTS should
5183   * still be in per-arch page fault handlers at the entry of page fault.
5184   */
mm_account_fault(struct mm_struct * mm,struct pt_regs * regs,unsigned long address,unsigned int flags,vm_fault_t ret)5185  static inline void mm_account_fault(struct mm_struct *mm, struct pt_regs *regs,
5186  				    unsigned long address, unsigned int flags,
5187  				    vm_fault_t ret)
5188  {
5189  	bool major;
5190  
5191  	/* Incomplete faults will be accounted upon completion. */
5192  	if (ret & VM_FAULT_RETRY)
5193  		return;
5194  
5195  	/*
5196  	 * To preserve the behavior of older kernels, PGFAULT counters record
5197  	 * both successful and failed faults, as opposed to perf counters,
5198  	 * which ignore failed cases.
5199  	 */
5200  	count_vm_event(PGFAULT);
5201  	count_memcg_event_mm(mm, PGFAULT);
5202  
5203  	/*
5204  	 * Do not account for unsuccessful faults (e.g. when the address wasn't
5205  	 * valid).  That includes arch_vma_access_permitted() failing before
5206  	 * reaching here. So this is not a "this many hardware page faults"
5207  	 * counter.  We should use the hw profiling for that.
5208  	 */
5209  	if (ret & VM_FAULT_ERROR)
5210  		return;
5211  
5212  	/*
5213  	 * We define the fault as a major fault when the final successful fault
5214  	 * is VM_FAULT_MAJOR, or if it retried (which implies that we couldn't
5215  	 * handle it immediately previously).
5216  	 */
5217  	major = (ret & VM_FAULT_MAJOR) || (flags & FAULT_FLAG_TRIED);
5218  
5219  	if (major)
5220  		current->maj_flt++;
5221  	else
5222  		current->min_flt++;
5223  
5224  	/*
5225  	 * If the fault is done for GUP, regs will be NULL.  We only do the
5226  	 * accounting for the per thread fault counters who triggered the
5227  	 * fault, and we skip the perf event updates.
5228  	 */
5229  	if (!regs)
5230  		return;
5231  
5232  	if (major)
5233  		perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address);
5234  	else
5235  		perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address);
5236  }
5237  
5238  #ifdef CONFIG_LRU_GEN
lru_gen_enter_fault(struct vm_area_struct * vma)5239  static void lru_gen_enter_fault(struct vm_area_struct *vma)
5240  {
5241  	/* the LRU algorithm only applies to accesses with recency */
5242  	current->in_lru_fault = vma_has_recency(vma);
5243  }
5244  
lru_gen_exit_fault(void)5245  static void lru_gen_exit_fault(void)
5246  {
5247  	current->in_lru_fault = false;
5248  }
5249  #else
lru_gen_enter_fault(struct vm_area_struct * vma)5250  static void lru_gen_enter_fault(struct vm_area_struct *vma)
5251  {
5252  }
5253  
lru_gen_exit_fault(void)5254  static void lru_gen_exit_fault(void)
5255  {
5256  }
5257  #endif /* CONFIG_LRU_GEN */
5258  
sanitize_fault_flags(struct vm_area_struct * vma,unsigned int * flags)5259  static vm_fault_t sanitize_fault_flags(struct vm_area_struct *vma,
5260  				       unsigned int *flags)
5261  {
5262  	if (unlikely(*flags & FAULT_FLAG_UNSHARE)) {
5263  		if (WARN_ON_ONCE(*flags & FAULT_FLAG_WRITE))
5264  			return VM_FAULT_SIGSEGV;
5265  		/*
5266  		 * FAULT_FLAG_UNSHARE only applies to COW mappings. Let's
5267  		 * just treat it like an ordinary read-fault otherwise.
5268  		 */
5269  		if (!is_cow_mapping(vma->vm_flags))
5270  			*flags &= ~FAULT_FLAG_UNSHARE;
5271  	} else if (*flags & FAULT_FLAG_WRITE) {
5272  		/* Write faults on read-only mappings are impossible ... */
5273  		if (WARN_ON_ONCE(!(vma->vm_flags & VM_MAYWRITE)))
5274  			return VM_FAULT_SIGSEGV;
5275  		/* ... and FOLL_FORCE only applies to COW mappings. */
5276  		if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE) &&
5277  				 !is_cow_mapping(vma->vm_flags)))
5278  			return VM_FAULT_SIGSEGV;
5279  	}
5280  #ifdef CONFIG_PER_VMA_LOCK
5281  	/*
5282  	 * Per-VMA locks can't be used with FAULT_FLAG_RETRY_NOWAIT because of
5283  	 * the assumption that lock is dropped on VM_FAULT_RETRY.
5284  	 */
5285  	if (WARN_ON_ONCE((*flags &
5286  			(FAULT_FLAG_VMA_LOCK | FAULT_FLAG_RETRY_NOWAIT)) ==
5287  			(FAULT_FLAG_VMA_LOCK | FAULT_FLAG_RETRY_NOWAIT)))
5288  		return VM_FAULT_SIGSEGV;
5289  #endif
5290  
5291  	return 0;
5292  }
5293  
5294  /*
5295   * By the time we get here, we already hold the mm semaphore
5296   *
5297   * The mmap_lock may have been released depending on flags and our
5298   * return value.  See filemap_fault() and __folio_lock_or_retry().
5299   */
handle_mm_fault(struct vm_area_struct * vma,unsigned long address,unsigned int flags,struct pt_regs * regs)5300  vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
5301  			   unsigned int flags, struct pt_regs *regs)
5302  {
5303  	/* If the fault handler drops the mmap_lock, vma may be freed */
5304  	struct mm_struct *mm = vma->vm_mm;
5305  	vm_fault_t ret;
5306  
5307  	__set_current_state(TASK_RUNNING);
5308  
5309  	ret = sanitize_fault_flags(vma, &flags);
5310  	if (ret)
5311  		goto out;
5312  
5313  	if (!arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE,
5314  					    flags & FAULT_FLAG_INSTRUCTION,
5315  					    flags & FAULT_FLAG_REMOTE)) {
5316  		ret = VM_FAULT_SIGSEGV;
5317  		goto out;
5318  	}
5319  
5320  	/*
5321  	 * Enable the memcg OOM handling for faults triggered in user
5322  	 * space.  Kernel faults are handled more gracefully.
5323  	 */
5324  	if (flags & FAULT_FLAG_USER)
5325  		mem_cgroup_enter_user_fault();
5326  
5327  	lru_gen_enter_fault(vma);
5328  
5329  	if (unlikely(is_vm_hugetlb_page(vma)))
5330  		ret = hugetlb_fault(vma->vm_mm, vma, address, flags);
5331  	else
5332  		ret = __handle_mm_fault(vma, address, flags);
5333  
5334  	lru_gen_exit_fault();
5335  
5336  	if (flags & FAULT_FLAG_USER) {
5337  		mem_cgroup_exit_user_fault();
5338  		/*
5339  		 * The task may have entered a memcg OOM situation but
5340  		 * if the allocation error was handled gracefully (no
5341  		 * VM_FAULT_OOM), there is no need to kill anything.
5342  		 * Just clean up the OOM state peacefully.
5343  		 */
5344  		if (task_in_memcg_oom(current) && !(ret & VM_FAULT_OOM))
5345  			mem_cgroup_oom_synchronize(false);
5346  	}
5347  out:
5348  	mm_account_fault(mm, regs, address, flags, ret);
5349  
5350  	return ret;
5351  }
5352  EXPORT_SYMBOL_GPL(handle_mm_fault);
5353  
5354  #ifdef CONFIG_LOCK_MM_AND_FIND_VMA
5355  #include <linux/extable.h>
5356  
get_mmap_lock_carefully(struct mm_struct * mm,struct pt_regs * regs)5357  static inline bool get_mmap_lock_carefully(struct mm_struct *mm, struct pt_regs *regs)
5358  {
5359  	if (likely(mmap_read_trylock(mm)))
5360  		return true;
5361  
5362  	if (regs && !user_mode(regs)) {
5363  		unsigned long ip = exception_ip(regs);
5364  		if (!search_exception_tables(ip))
5365  			return false;
5366  	}
5367  
5368  	return !mmap_read_lock_killable(mm);
5369  }
5370  
mmap_upgrade_trylock(struct mm_struct * mm)5371  static inline bool mmap_upgrade_trylock(struct mm_struct *mm)
5372  {
5373  	/*
5374  	 * We don't have this operation yet.
5375  	 *
5376  	 * It should be easy enough to do: it's basically a
5377  	 *    atomic_long_try_cmpxchg_acquire()
5378  	 * from RWSEM_READER_BIAS -> RWSEM_WRITER_LOCKED, but
5379  	 * it also needs the proper lockdep magic etc.
5380  	 */
5381  	return false;
5382  }
5383  
upgrade_mmap_lock_carefully(struct mm_struct * mm,struct pt_regs * regs)5384  static inline bool upgrade_mmap_lock_carefully(struct mm_struct *mm, struct pt_regs *regs)
5385  {
5386  	mmap_read_unlock(mm);
5387  	if (regs && !user_mode(regs)) {
5388  		unsigned long ip = exception_ip(regs);
5389  		if (!search_exception_tables(ip))
5390  			return false;
5391  	}
5392  	return !mmap_write_lock_killable(mm);
5393  }
5394  
5395  /*
5396   * Helper for page fault handling.
5397   *
5398   * This is kind of equivalend to "mmap_read_lock()" followed
5399   * by "find_extend_vma()", except it's a lot more careful about
5400   * the locking (and will drop the lock on failure).
5401   *
5402   * For example, if we have a kernel bug that causes a page
5403   * fault, we don't want to just use mmap_read_lock() to get
5404   * the mm lock, because that would deadlock if the bug were
5405   * to happen while we're holding the mm lock for writing.
5406   *
5407   * So this checks the exception tables on kernel faults in
5408   * order to only do this all for instructions that are actually
5409   * expected to fault.
5410   *
5411   * We can also actually take the mm lock for writing if we
5412   * need to extend the vma, which helps the VM layer a lot.
5413   */
lock_mm_and_find_vma(struct mm_struct * mm,unsigned long addr,struct pt_regs * regs)5414  struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm,
5415  			unsigned long addr, struct pt_regs *regs)
5416  {
5417  	struct vm_area_struct *vma;
5418  
5419  	if (!get_mmap_lock_carefully(mm, regs))
5420  		return NULL;
5421  
5422  	vma = find_vma(mm, addr);
5423  	if (likely(vma && (vma->vm_start <= addr)))
5424  		return vma;
5425  
5426  	/*
5427  	 * Well, dang. We might still be successful, but only
5428  	 * if we can extend a vma to do so.
5429  	 */
5430  	if (!vma || !(vma->vm_flags & VM_GROWSDOWN)) {
5431  		mmap_read_unlock(mm);
5432  		return NULL;
5433  	}
5434  
5435  	/*
5436  	 * We can try to upgrade the mmap lock atomically,
5437  	 * in which case we can continue to use the vma
5438  	 * we already looked up.
5439  	 *
5440  	 * Otherwise we'll have to drop the mmap lock and
5441  	 * re-take it, and also look up the vma again,
5442  	 * re-checking it.
5443  	 */
5444  	if (!mmap_upgrade_trylock(mm)) {
5445  		if (!upgrade_mmap_lock_carefully(mm, regs))
5446  			return NULL;
5447  
5448  		vma = find_vma(mm, addr);
5449  		if (!vma)
5450  			goto fail;
5451  		if (vma->vm_start <= addr)
5452  			goto success;
5453  		if (!(vma->vm_flags & VM_GROWSDOWN))
5454  			goto fail;
5455  	}
5456  
5457  	if (expand_stack_locked(vma, addr))
5458  		goto fail;
5459  
5460  success:
5461  	mmap_write_downgrade(mm);
5462  	return vma;
5463  
5464  fail:
5465  	mmap_write_unlock(mm);
5466  	return NULL;
5467  }
5468  #endif
5469  
5470  #ifdef CONFIG_PER_VMA_LOCK
5471  /*
5472   * Lookup and lock a VMA under RCU protection. Returned VMA is guaranteed to be
5473   * stable and not isolated. If the VMA is not found or is being modified the
5474   * function returns NULL.
5475   */
lock_vma_under_rcu(struct mm_struct * mm,unsigned long address)5476  struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm,
5477  					  unsigned long address)
5478  {
5479  	MA_STATE(mas, &mm->mm_mt, address, address);
5480  	struct vm_area_struct *vma;
5481  
5482  	rcu_read_lock();
5483  retry:
5484  	vma = mas_walk(&mas);
5485  	if (!vma)
5486  		goto inval;
5487  
5488  	if (!vma_start_read(vma))
5489  		goto inval;
5490  
5491  	/*
5492  	 * find_mergeable_anon_vma uses adjacent vmas which are not locked.
5493  	 * This check must happen after vma_start_read(); otherwise, a
5494  	 * concurrent mremap() with MREMAP_DONTUNMAP could dissociate the VMA
5495  	 * from its anon_vma.
5496  	 */
5497  	if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma))
5498  		goto inval_end_read;
5499  
5500  	/* Check since vm_start/vm_end might change before we lock the VMA */
5501  	if (unlikely(address < vma->vm_start || address >= vma->vm_end))
5502  		goto inval_end_read;
5503  
5504  	/* Check if the VMA got isolated after we found it */
5505  	if (vma->detached) {
5506  		vma_end_read(vma);
5507  		count_vm_vma_lock_event(VMA_LOCK_MISS);
5508  		/* The area was replaced with another one */
5509  		goto retry;
5510  	}
5511  
5512  	rcu_read_unlock();
5513  	return vma;
5514  
5515  inval_end_read:
5516  	vma_end_read(vma);
5517  inval:
5518  	rcu_read_unlock();
5519  	count_vm_vma_lock_event(VMA_LOCK_ABORT);
5520  	return NULL;
5521  }
5522  #endif /* CONFIG_PER_VMA_LOCK */
5523  
5524  #ifndef __PAGETABLE_P4D_FOLDED
5525  /*
5526   * Allocate p4d page table.
5527   * We've already handled the fast-path in-line.
5528   */
__p4d_alloc(struct mm_struct * mm,pgd_t * pgd,unsigned long address)5529  int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
5530  {
5531  	p4d_t *new = p4d_alloc_one(mm, address);
5532  	if (!new)
5533  		return -ENOMEM;
5534  
5535  	spin_lock(&mm->page_table_lock);
5536  	if (pgd_present(*pgd)) {	/* Another has populated it */
5537  		p4d_free(mm, new);
5538  	} else {
5539  		smp_wmb(); /* See comment in pmd_install() */
5540  		pgd_populate(mm, pgd, new);
5541  	}
5542  	spin_unlock(&mm->page_table_lock);
5543  	return 0;
5544  }
5545  #endif /* __PAGETABLE_P4D_FOLDED */
5546  
5547  #ifndef __PAGETABLE_PUD_FOLDED
5548  /*
5549   * Allocate page upper directory.
5550   * We've already handled the fast-path in-line.
5551   */
__pud_alloc(struct mm_struct * mm,p4d_t * p4d,unsigned long address)5552  int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address)
5553  {
5554  	pud_t *new = pud_alloc_one(mm, address);
5555  	if (!new)
5556  		return -ENOMEM;
5557  
5558  	spin_lock(&mm->page_table_lock);
5559  	if (!p4d_present(*p4d)) {
5560  		mm_inc_nr_puds(mm);
5561  		smp_wmb(); /* See comment in pmd_install() */
5562  		p4d_populate(mm, p4d, new);
5563  	} else	/* Another has populated it */
5564  		pud_free(mm, new);
5565  	spin_unlock(&mm->page_table_lock);
5566  	return 0;
5567  }
5568  #endif /* __PAGETABLE_PUD_FOLDED */
5569  
5570  #ifndef __PAGETABLE_PMD_FOLDED
5571  /*
5572   * Allocate page middle directory.
5573   * We've already handled the fast-path in-line.
5574   */
__pmd_alloc(struct mm_struct * mm,pud_t * pud,unsigned long address)5575  int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
5576  {
5577  	spinlock_t *ptl;
5578  	pmd_t *new = pmd_alloc_one(mm, address);
5579  	if (!new)
5580  		return -ENOMEM;
5581  
5582  	ptl = pud_lock(mm, pud);
5583  	if (!pud_present(*pud)) {
5584  		mm_inc_nr_pmds(mm);
5585  		smp_wmb(); /* See comment in pmd_install() */
5586  		pud_populate(mm, pud, new);
5587  	} else {	/* Another has populated it */
5588  		pmd_free(mm, new);
5589  	}
5590  	spin_unlock(ptl);
5591  	return 0;
5592  }
5593  #endif /* __PAGETABLE_PMD_FOLDED */
5594  
5595  /**
5596   * follow_pte - look up PTE at a user virtual address
5597   * @mm: the mm_struct of the target address space
5598   * @address: user virtual address
5599   * @ptepp: location to store found PTE
5600   * @ptlp: location to store the lock for the PTE
5601   *
5602   * On a successful return, the pointer to the PTE is stored in @ptepp;
5603   * the corresponding lock is taken and its location is stored in @ptlp.
5604   * The contents of the PTE are only stable until @ptlp is released;
5605   * any further use, if any, must be protected against invalidation
5606   * with MMU notifiers.
5607   *
5608   * Only IO mappings and raw PFN mappings are allowed.  The mmap semaphore
5609   * should be taken for read.
5610   *
5611   * KVM uses this function.  While it is arguably less bad than ``follow_pfn``,
5612   * it is not a good general-purpose API.
5613   *
5614   * Return: zero on success, -ve otherwise.
5615   */
follow_pte(struct mm_struct * mm,unsigned long address,pte_t ** ptepp,spinlock_t ** ptlp)5616  int follow_pte(struct mm_struct *mm, unsigned long address,
5617  	       pte_t **ptepp, spinlock_t **ptlp)
5618  {
5619  	pgd_t *pgd;
5620  	p4d_t *p4d;
5621  	pud_t *pud;
5622  	pmd_t *pmd;
5623  	pte_t *ptep;
5624  
5625  	pgd = pgd_offset(mm, address);
5626  	if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
5627  		goto out;
5628  
5629  	p4d = p4d_offset(pgd, address);
5630  	if (p4d_none(*p4d) || unlikely(p4d_bad(*p4d)))
5631  		goto out;
5632  
5633  	pud = pud_offset(p4d, address);
5634  	if (pud_none(*pud) || unlikely(pud_bad(*pud)))
5635  		goto out;
5636  
5637  	pmd = pmd_offset(pud, address);
5638  	VM_BUG_ON(pmd_trans_huge(*pmd));
5639  
5640  	ptep = pte_offset_map_lock(mm, pmd, address, ptlp);
5641  	if (!ptep)
5642  		goto out;
5643  	if (!pte_present(ptep_get(ptep)))
5644  		goto unlock;
5645  	*ptepp = ptep;
5646  	return 0;
5647  unlock:
5648  	pte_unmap_unlock(ptep, *ptlp);
5649  out:
5650  	return -EINVAL;
5651  }
5652  EXPORT_SYMBOL_GPL(follow_pte);
5653  
5654  /**
5655   * follow_pfn - look up PFN at a user virtual address
5656   * @vma: memory mapping
5657   * @address: user virtual address
5658   * @pfn: location to store found PFN
5659   *
5660   * Only IO mappings and raw PFN mappings are allowed.
5661   *
5662   * This function does not allow the caller to read the permissions
5663   * of the PTE.  Do not use it.
5664   *
5665   * Return: zero and the pfn at @pfn on success, -ve otherwise.
5666   */
follow_pfn(struct vm_area_struct * vma,unsigned long address,unsigned long * pfn)5667  int follow_pfn(struct vm_area_struct *vma, unsigned long address,
5668  	unsigned long *pfn)
5669  {
5670  	int ret = -EINVAL;
5671  	spinlock_t *ptl;
5672  	pte_t *ptep;
5673  
5674  	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
5675  		return ret;
5676  
5677  	ret = follow_pte(vma->vm_mm, address, &ptep, &ptl);
5678  	if (ret)
5679  		return ret;
5680  	*pfn = pte_pfn(ptep_get(ptep));
5681  	pte_unmap_unlock(ptep, ptl);
5682  	return 0;
5683  }
5684  EXPORT_SYMBOL(follow_pfn);
5685  
5686  #ifdef CONFIG_HAVE_IOREMAP_PROT
follow_phys(struct vm_area_struct * vma,unsigned long address,unsigned int flags,unsigned long * prot,resource_size_t * phys)5687  int follow_phys(struct vm_area_struct *vma,
5688  		unsigned long address, unsigned int flags,
5689  		unsigned long *prot, resource_size_t *phys)
5690  {
5691  	int ret = -EINVAL;
5692  	pte_t *ptep, pte;
5693  	spinlock_t *ptl;
5694  
5695  	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
5696  		goto out;
5697  
5698  	if (follow_pte(vma->vm_mm, address, &ptep, &ptl))
5699  		goto out;
5700  	pte = ptep_get(ptep);
5701  
5702  	/* Never return PFNs of anon folios in COW mappings. */
5703  	if (vm_normal_folio(vma, address, pte))
5704  		goto unlock;
5705  
5706  	if ((flags & FOLL_WRITE) && !pte_write(pte))
5707  		goto unlock;
5708  
5709  	*prot = pgprot_val(pte_pgprot(pte));
5710  	*phys = (resource_size_t)pte_pfn(pte) << PAGE_SHIFT;
5711  
5712  	ret = 0;
5713  unlock:
5714  	pte_unmap_unlock(ptep, ptl);
5715  out:
5716  	return ret;
5717  }
5718  
5719  /**
5720   * generic_access_phys - generic implementation for iomem mmap access
5721   * @vma: the vma to access
5722   * @addr: userspace address, not relative offset within @vma
5723   * @buf: buffer to read/write
5724   * @len: length of transfer
5725   * @write: set to FOLL_WRITE when writing, otherwise reading
5726   *
5727   * This is a generic implementation for &vm_operations_struct.access for an
5728   * iomem mapping. This callback is used by access_process_vm() when the @vma is
5729   * not page based.
5730   */
generic_access_phys(struct vm_area_struct * vma,unsigned long addr,void * buf,int len,int write)5731  int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
5732  			void *buf, int len, int write)
5733  {
5734  	resource_size_t phys_addr;
5735  	unsigned long prot = 0;
5736  	void __iomem *maddr;
5737  	pte_t *ptep, pte;
5738  	spinlock_t *ptl;
5739  	int offset = offset_in_page(addr);
5740  	int ret = -EINVAL;
5741  
5742  	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
5743  		return -EINVAL;
5744  
5745  retry:
5746  	if (follow_pte(vma->vm_mm, addr, &ptep, &ptl))
5747  		return -EINVAL;
5748  	pte = ptep_get(ptep);
5749  	pte_unmap_unlock(ptep, ptl);
5750  
5751  	prot = pgprot_val(pte_pgprot(pte));
5752  	phys_addr = (resource_size_t)pte_pfn(pte) << PAGE_SHIFT;
5753  
5754  	if ((write & FOLL_WRITE) && !pte_write(pte))
5755  		return -EINVAL;
5756  
5757  	maddr = ioremap_prot(phys_addr, PAGE_ALIGN(len + offset), prot);
5758  	if (!maddr)
5759  		return -ENOMEM;
5760  
5761  	if (follow_pte(vma->vm_mm, addr, &ptep, &ptl))
5762  		goto out_unmap;
5763  
5764  	if (!pte_same(pte, ptep_get(ptep))) {
5765  		pte_unmap_unlock(ptep, ptl);
5766  		iounmap(maddr);
5767  
5768  		goto retry;
5769  	}
5770  
5771  	if (write)
5772  		memcpy_toio(maddr + offset, buf, len);
5773  	else
5774  		memcpy_fromio(buf, maddr + offset, len);
5775  	ret = len;
5776  	pte_unmap_unlock(ptep, ptl);
5777  out_unmap:
5778  	iounmap(maddr);
5779  
5780  	return ret;
5781  }
5782  EXPORT_SYMBOL_GPL(generic_access_phys);
5783  #endif
5784  
5785  /*
5786   * Access another process' address space as given in mm.
5787   */
__access_remote_vm(struct mm_struct * mm,unsigned long addr,void * buf,int len,unsigned int gup_flags)5788  int __access_remote_vm(struct mm_struct *mm, unsigned long addr, void *buf,
5789  		       int len, unsigned int gup_flags)
5790  {
5791  	void *old_buf = buf;
5792  	int write = gup_flags & FOLL_WRITE;
5793  
5794  	if (mmap_read_lock_killable(mm))
5795  		return 0;
5796  
5797  	/* Untag the address before looking up the VMA */
5798  	addr = untagged_addr_remote(mm, addr);
5799  
5800  	/* Avoid triggering the temporary warning in __get_user_pages */
5801  	if (!vma_lookup(mm, addr) && !expand_stack(mm, addr))
5802  		return 0;
5803  
5804  	/* ignore errors, just check how much was successfully transferred */
5805  	while (len) {
5806  		int bytes, offset;
5807  		void *maddr;
5808  		struct vm_area_struct *vma = NULL;
5809  		struct page *page = get_user_page_vma_remote(mm, addr,
5810  							     gup_flags, &vma);
5811  
5812  		if (IS_ERR_OR_NULL(page)) {
5813  			/* We might need to expand the stack to access it */
5814  			vma = vma_lookup(mm, addr);
5815  			if (!vma) {
5816  				vma = expand_stack(mm, addr);
5817  
5818  				/* mmap_lock was dropped on failure */
5819  				if (!vma)
5820  					return buf - old_buf;
5821  
5822  				/* Try again if stack expansion worked */
5823  				continue;
5824  			}
5825  
5826  
5827  			/*
5828  			 * Check if this is a VM_IO | VM_PFNMAP VMA, which
5829  			 * we can access using slightly different code.
5830  			 */
5831  			bytes = 0;
5832  #ifdef CONFIG_HAVE_IOREMAP_PROT
5833  			if (vma->vm_ops && vma->vm_ops->access)
5834  				bytes = vma->vm_ops->access(vma, addr, buf,
5835  							    len, write);
5836  #endif
5837  			if (bytes <= 0)
5838  				break;
5839  		} else {
5840  			bytes = len;
5841  			offset = addr & (PAGE_SIZE-1);
5842  			if (bytes > PAGE_SIZE-offset)
5843  				bytes = PAGE_SIZE-offset;
5844  
5845  			maddr = kmap(page);
5846  			if (write) {
5847  				copy_to_user_page(vma, page, addr,
5848  						  maddr + offset, buf, bytes);
5849  				set_page_dirty_lock(page);
5850  			} else {
5851  				copy_from_user_page(vma, page, addr,
5852  						    buf, maddr + offset, bytes);
5853  			}
5854  			kunmap(page);
5855  			put_page(page);
5856  		}
5857  		len -= bytes;
5858  		buf += bytes;
5859  		addr += bytes;
5860  	}
5861  	mmap_read_unlock(mm);
5862  
5863  	return buf - old_buf;
5864  }
5865  
5866  /**
5867   * access_remote_vm - access another process' address space
5868   * @mm:		the mm_struct of the target address space
5869   * @addr:	start address to access
5870   * @buf:	source or destination buffer
5871   * @len:	number of bytes to transfer
5872   * @gup_flags:	flags modifying lookup behaviour
5873   *
5874   * The caller must hold a reference on @mm.
5875   *
5876   * Return: number of bytes copied from source to destination.
5877   */
access_remote_vm(struct mm_struct * mm,unsigned long addr,void * buf,int len,unsigned int gup_flags)5878  int access_remote_vm(struct mm_struct *mm, unsigned long addr,
5879  		void *buf, int len, unsigned int gup_flags)
5880  {
5881  	return __access_remote_vm(mm, addr, buf, len, gup_flags);
5882  }
5883  
5884  /*
5885   * Access another process' address space.
5886   * Source/target buffer must be kernel space,
5887   * Do not walk the page table directly, use get_user_pages
5888   */
access_process_vm(struct task_struct * tsk,unsigned long addr,void * buf,int len,unsigned int gup_flags)5889  int access_process_vm(struct task_struct *tsk, unsigned long addr,
5890  		void *buf, int len, unsigned int gup_flags)
5891  {
5892  	struct mm_struct *mm;
5893  	int ret;
5894  
5895  	mm = get_task_mm(tsk);
5896  	if (!mm)
5897  		return 0;
5898  
5899  	ret = __access_remote_vm(mm, addr, buf, len, gup_flags);
5900  
5901  	mmput(mm);
5902  
5903  	return ret;
5904  }
5905  EXPORT_SYMBOL_GPL(access_process_vm);
5906  
5907  /*
5908   * Print the name of a VMA.
5909   */
print_vma_addr(char * prefix,unsigned long ip)5910  void print_vma_addr(char *prefix, unsigned long ip)
5911  {
5912  	struct mm_struct *mm = current->mm;
5913  	struct vm_area_struct *vma;
5914  
5915  	/*
5916  	 * we might be running from an atomic context so we cannot sleep
5917  	 */
5918  	if (!mmap_read_trylock(mm))
5919  		return;
5920  
5921  	vma = find_vma(mm, ip);
5922  	if (vma && vma->vm_file) {
5923  		struct file *f = vma->vm_file;
5924  		char *buf = (char *)__get_free_page(GFP_NOWAIT);
5925  		if (buf) {
5926  			char *p;
5927  
5928  			p = file_path(f, buf, PAGE_SIZE);
5929  			if (IS_ERR(p))
5930  				p = "?";
5931  			printk("%s%s[%lx+%lx]", prefix, kbasename(p),
5932  					vma->vm_start,
5933  					vma->vm_end - vma->vm_start);
5934  			free_page((unsigned long)buf);
5935  		}
5936  	}
5937  	mmap_read_unlock(mm);
5938  }
5939  
5940  #if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP)
__might_fault(const char * file,int line)5941  void __might_fault(const char *file, int line)
5942  {
5943  	if (pagefault_disabled())
5944  		return;
5945  	__might_sleep(file, line);
5946  #if defined(CONFIG_DEBUG_ATOMIC_SLEEP)
5947  	if (current->mm)
5948  		might_lock_read(&current->mm->mmap_lock);
5949  #endif
5950  }
5951  EXPORT_SYMBOL(__might_fault);
5952  #endif
5953  
5954  #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
5955  /*
5956   * Process all subpages of the specified huge page with the specified
5957   * operation.  The target subpage will be processed last to keep its
5958   * cache lines hot.
5959   */
process_huge_page(unsigned long addr_hint,unsigned int pages_per_huge_page,int (* process_subpage)(unsigned long addr,int idx,void * arg),void * arg)5960  static inline int process_huge_page(
5961  	unsigned long addr_hint, unsigned int pages_per_huge_page,
5962  	int (*process_subpage)(unsigned long addr, int idx, void *arg),
5963  	void *arg)
5964  {
5965  	int i, n, base, l, ret;
5966  	unsigned long addr = addr_hint &
5967  		~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
5968  
5969  	/* Process target subpage last to keep its cache lines hot */
5970  	might_sleep();
5971  	n = (addr_hint - addr) / PAGE_SIZE;
5972  	if (2 * n <= pages_per_huge_page) {
5973  		/* If target subpage in first half of huge page */
5974  		base = 0;
5975  		l = n;
5976  		/* Process subpages at the end of huge page */
5977  		for (i = pages_per_huge_page - 1; i >= 2 * n; i--) {
5978  			cond_resched();
5979  			ret = process_subpage(addr + i * PAGE_SIZE, i, arg);
5980  			if (ret)
5981  				return ret;
5982  		}
5983  	} else {
5984  		/* If target subpage in second half of huge page */
5985  		base = pages_per_huge_page - 2 * (pages_per_huge_page - n);
5986  		l = pages_per_huge_page - n;
5987  		/* Process subpages at the begin of huge page */
5988  		for (i = 0; i < base; i++) {
5989  			cond_resched();
5990  			ret = process_subpage(addr + i * PAGE_SIZE, i, arg);
5991  			if (ret)
5992  				return ret;
5993  		}
5994  	}
5995  	/*
5996  	 * Process remaining subpages in left-right-left-right pattern
5997  	 * towards the target subpage
5998  	 */
5999  	for (i = 0; i < l; i++) {
6000  		int left_idx = base + i;
6001  		int right_idx = base + 2 * l - 1 - i;
6002  
6003  		cond_resched();
6004  		ret = process_subpage(addr + left_idx * PAGE_SIZE, left_idx, arg);
6005  		if (ret)
6006  			return ret;
6007  		cond_resched();
6008  		ret = process_subpage(addr + right_idx * PAGE_SIZE, right_idx, arg);
6009  		if (ret)
6010  			return ret;
6011  	}
6012  	return 0;
6013  }
6014  
clear_gigantic_page(struct page * page,unsigned long addr,unsigned int pages_per_huge_page)6015  static void clear_gigantic_page(struct page *page,
6016  				unsigned long addr,
6017  				unsigned int pages_per_huge_page)
6018  {
6019  	int i;
6020  	struct page *p;
6021  
6022  	might_sleep();
6023  	for (i = 0; i < pages_per_huge_page; i++) {
6024  		p = nth_page(page, i);
6025  		cond_resched();
6026  		clear_user_highpage(p, addr + i * PAGE_SIZE);
6027  	}
6028  }
6029  
clear_subpage(unsigned long addr,int idx,void * arg)6030  static int clear_subpage(unsigned long addr, int idx, void *arg)
6031  {
6032  	struct page *page = arg;
6033  
6034  	clear_user_highpage(page + idx, addr);
6035  	return 0;
6036  }
6037  
clear_huge_page(struct page * page,unsigned long addr_hint,unsigned int pages_per_huge_page)6038  void clear_huge_page(struct page *page,
6039  		     unsigned long addr_hint, unsigned int pages_per_huge_page)
6040  {
6041  	unsigned long addr = addr_hint &
6042  		~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
6043  
6044  	if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) {
6045  		clear_gigantic_page(page, addr, pages_per_huge_page);
6046  		return;
6047  	}
6048  
6049  	process_huge_page(addr_hint, pages_per_huge_page, clear_subpage, page);
6050  }
6051  
copy_user_gigantic_page(struct folio * dst,struct folio * src,unsigned long addr,struct vm_area_struct * vma,unsigned int pages_per_huge_page)6052  static int copy_user_gigantic_page(struct folio *dst, struct folio *src,
6053  				     unsigned long addr,
6054  				     struct vm_area_struct *vma,
6055  				     unsigned int pages_per_huge_page)
6056  {
6057  	int i;
6058  	struct page *dst_page;
6059  	struct page *src_page;
6060  
6061  	for (i = 0; i < pages_per_huge_page; i++) {
6062  		dst_page = folio_page(dst, i);
6063  		src_page = folio_page(src, i);
6064  
6065  		cond_resched();
6066  		if (copy_mc_user_highpage(dst_page, src_page,
6067  					  addr + i*PAGE_SIZE, vma)) {
6068  			memory_failure_queue(page_to_pfn(src_page), 0);
6069  			return -EHWPOISON;
6070  		}
6071  	}
6072  	return 0;
6073  }
6074  
6075  struct copy_subpage_arg {
6076  	struct page *dst;
6077  	struct page *src;
6078  	struct vm_area_struct *vma;
6079  };
6080  
copy_subpage(unsigned long addr,int idx,void * arg)6081  static int copy_subpage(unsigned long addr, int idx, void *arg)
6082  {
6083  	struct copy_subpage_arg *copy_arg = arg;
6084  
6085  	if (copy_mc_user_highpage(copy_arg->dst + idx, copy_arg->src + idx,
6086  				  addr, copy_arg->vma)) {
6087  		memory_failure_queue(page_to_pfn(copy_arg->src + idx), 0);
6088  		return -EHWPOISON;
6089  	}
6090  	return 0;
6091  }
6092  
copy_user_large_folio(struct folio * dst,struct folio * src,unsigned long addr_hint,struct vm_area_struct * vma)6093  int copy_user_large_folio(struct folio *dst, struct folio *src,
6094  			  unsigned long addr_hint, struct vm_area_struct *vma)
6095  {
6096  	unsigned int pages_per_huge_page = folio_nr_pages(dst);
6097  	unsigned long addr = addr_hint &
6098  		~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
6099  	struct copy_subpage_arg arg = {
6100  		.dst = &dst->page,
6101  		.src = &src->page,
6102  		.vma = vma,
6103  	};
6104  
6105  	if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES))
6106  		return copy_user_gigantic_page(dst, src, addr, vma,
6107  					       pages_per_huge_page);
6108  
6109  	return process_huge_page(addr_hint, pages_per_huge_page, copy_subpage, &arg);
6110  }
6111  
copy_folio_from_user(struct folio * dst_folio,const void __user * usr_src,bool allow_pagefault)6112  long copy_folio_from_user(struct folio *dst_folio,
6113  			   const void __user *usr_src,
6114  			   bool allow_pagefault)
6115  {
6116  	void *kaddr;
6117  	unsigned long i, rc = 0;
6118  	unsigned int nr_pages = folio_nr_pages(dst_folio);
6119  	unsigned long ret_val = nr_pages * PAGE_SIZE;
6120  	struct page *subpage;
6121  
6122  	for (i = 0; i < nr_pages; i++) {
6123  		subpage = folio_page(dst_folio, i);
6124  		kaddr = kmap_local_page(subpage);
6125  		if (!allow_pagefault)
6126  			pagefault_disable();
6127  		rc = copy_from_user(kaddr, usr_src + i * PAGE_SIZE, PAGE_SIZE);
6128  		if (!allow_pagefault)
6129  			pagefault_enable();
6130  		kunmap_local(kaddr);
6131  
6132  		ret_val -= (PAGE_SIZE - rc);
6133  		if (rc)
6134  			break;
6135  
6136  		flush_dcache_page(subpage);
6137  
6138  		cond_resched();
6139  	}
6140  	return ret_val;
6141  }
6142  #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
6143  
6144  #if USE_SPLIT_PTE_PTLOCKS && ALLOC_SPLIT_PTLOCKS
6145  
6146  static struct kmem_cache *page_ptl_cachep;
6147  
ptlock_cache_init(void)6148  void __init ptlock_cache_init(void)
6149  {
6150  	page_ptl_cachep = kmem_cache_create("page->ptl", sizeof(spinlock_t), 0,
6151  			SLAB_PANIC, NULL);
6152  }
6153  
ptlock_alloc(struct ptdesc * ptdesc)6154  bool ptlock_alloc(struct ptdesc *ptdesc)
6155  {
6156  	spinlock_t *ptl;
6157  
6158  	ptl = kmem_cache_alloc(page_ptl_cachep, GFP_KERNEL);
6159  	if (!ptl)
6160  		return false;
6161  	ptdesc->ptl = ptl;
6162  	return true;
6163  }
6164  
ptlock_free(struct ptdesc * ptdesc)6165  void ptlock_free(struct ptdesc *ptdesc)
6166  {
6167  	kmem_cache_free(page_ptl_cachep, ptdesc->ptl);
6168  }
6169  #endif
6170