xref: /openbmc/linux/mm/khugepaged.c (revision e961cc56)
1 // SPDX-License-Identifier: GPL-2.0
2 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
3 
4 #include <linux/mm.h>
5 #include <linux/sched.h>
6 #include <linux/sched/mm.h>
7 #include <linux/sched/coredump.h>
8 #include <linux/mmu_notifier.h>
9 #include <linux/rmap.h>
10 #include <linux/swap.h>
11 #include <linux/mm_inline.h>
12 #include <linux/kthread.h>
13 #include <linux/khugepaged.h>
14 #include <linux/freezer.h>
15 #include <linux/mman.h>
16 #include <linux/hashtable.h>
17 #include <linux/userfaultfd_k.h>
18 #include <linux/page_idle.h>
19 #include <linux/page_table_check.h>
20 #include <linux/swapops.h>
21 #include <linux/shmem_fs.h>
22 
23 #include <asm/tlb.h>
24 #include <asm/pgalloc.h>
25 #include "internal.h"
26 #include "mm_slot.h"
27 
28 enum scan_result {
29 	SCAN_FAIL,
30 	SCAN_SUCCEED,
31 	SCAN_PMD_NULL,
32 	SCAN_PMD_NONE,
33 	SCAN_PMD_MAPPED,
34 	SCAN_EXCEED_NONE_PTE,
35 	SCAN_EXCEED_SWAP_PTE,
36 	SCAN_EXCEED_SHARED_PTE,
37 	SCAN_PTE_NON_PRESENT,
38 	SCAN_PTE_UFFD_WP,
39 	SCAN_PTE_MAPPED_HUGEPAGE,
40 	SCAN_PAGE_RO,
41 	SCAN_LACK_REFERENCED_PAGE,
42 	SCAN_PAGE_NULL,
43 	SCAN_SCAN_ABORT,
44 	SCAN_PAGE_COUNT,
45 	SCAN_PAGE_LRU,
46 	SCAN_PAGE_LOCK,
47 	SCAN_PAGE_ANON,
48 	SCAN_PAGE_COMPOUND,
49 	SCAN_ANY_PROCESS,
50 	SCAN_VMA_NULL,
51 	SCAN_VMA_CHECK,
52 	SCAN_ADDRESS_RANGE,
53 	SCAN_DEL_PAGE_LRU,
54 	SCAN_ALLOC_HUGE_PAGE_FAIL,
55 	SCAN_CGROUP_CHARGE_FAIL,
56 	SCAN_TRUNCATED,
57 	SCAN_PAGE_HAS_PRIVATE,
58 	SCAN_STORE_FAILED,
59 };
60 
61 #define CREATE_TRACE_POINTS
62 #include <trace/events/huge_memory.h>
63 
64 static struct task_struct *khugepaged_thread __read_mostly;
65 static DEFINE_MUTEX(khugepaged_mutex);
66 
67 /* default scan 8*512 pte (or vmas) every 30 second */
68 static unsigned int khugepaged_pages_to_scan __read_mostly;
69 static unsigned int khugepaged_pages_collapsed;
70 static unsigned int khugepaged_full_scans;
71 static unsigned int khugepaged_scan_sleep_millisecs __read_mostly = 10000;
72 /* during fragmentation poll the hugepage allocator once every minute */
73 static unsigned int khugepaged_alloc_sleep_millisecs __read_mostly = 60000;
74 static unsigned long khugepaged_sleep_expire;
75 static DEFINE_SPINLOCK(khugepaged_mm_lock);
76 static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait);
77 /*
78  * default collapse hugepages if there is at least one pte mapped like
79  * it would have happened if the vma was large enough during page
80  * fault.
81  *
82  * Note that these are only respected if collapse was initiated by khugepaged.
83  */
84 static unsigned int khugepaged_max_ptes_none __read_mostly;
85 static unsigned int khugepaged_max_ptes_swap __read_mostly;
86 static unsigned int khugepaged_max_ptes_shared __read_mostly;
87 
88 #define MM_SLOTS_HASH_BITS 10
89 static __read_mostly DEFINE_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS);
90 
91 static struct kmem_cache *mm_slot_cache __read_mostly;
92 
93 #define MAX_PTE_MAPPED_THP 8
94 
95 struct collapse_control {
96 	bool is_khugepaged;
97 
98 	/* Num pages scanned per node */
99 	u32 node_load[MAX_NUMNODES];
100 
101 	/* nodemask for allocation fallback */
102 	nodemask_t alloc_nmask;
103 };
104 
105 /**
106  * struct khugepaged_mm_slot - khugepaged information per mm that is being scanned
107  * @slot: hash lookup from mm to mm_slot
108  * @nr_pte_mapped_thp: number of pte mapped THP
109  * @pte_mapped_thp: address array corresponding pte mapped THP
110  */
111 struct khugepaged_mm_slot {
112 	struct mm_slot slot;
113 
114 	/* pte-mapped THP in this mm */
115 	int nr_pte_mapped_thp;
116 	unsigned long pte_mapped_thp[MAX_PTE_MAPPED_THP];
117 };
118 
119 /**
120  * struct khugepaged_scan - cursor for scanning
121  * @mm_head: the head of the mm list to scan
122  * @mm_slot: the current mm_slot we are scanning
123  * @address: the next address inside that to be scanned
124  *
125  * There is only the one khugepaged_scan instance of this cursor structure.
126  */
127 struct khugepaged_scan {
128 	struct list_head mm_head;
129 	struct khugepaged_mm_slot *mm_slot;
130 	unsigned long address;
131 };
132 
133 static struct khugepaged_scan khugepaged_scan = {
134 	.mm_head = LIST_HEAD_INIT(khugepaged_scan.mm_head),
135 };
136 
137 #ifdef CONFIG_SYSFS
138 static ssize_t scan_sleep_millisecs_show(struct kobject *kobj,
139 					 struct kobj_attribute *attr,
140 					 char *buf)
141 {
142 	return sysfs_emit(buf, "%u\n", khugepaged_scan_sleep_millisecs);
143 }
144 
145 static ssize_t scan_sleep_millisecs_store(struct kobject *kobj,
146 					  struct kobj_attribute *attr,
147 					  const char *buf, size_t count)
148 {
149 	unsigned int msecs;
150 	int err;
151 
152 	err = kstrtouint(buf, 10, &msecs);
153 	if (err)
154 		return -EINVAL;
155 
156 	khugepaged_scan_sleep_millisecs = msecs;
157 	khugepaged_sleep_expire = 0;
158 	wake_up_interruptible(&khugepaged_wait);
159 
160 	return count;
161 }
162 static struct kobj_attribute scan_sleep_millisecs_attr =
163 	__ATTR_RW(scan_sleep_millisecs);
164 
165 static ssize_t alloc_sleep_millisecs_show(struct kobject *kobj,
166 					  struct kobj_attribute *attr,
167 					  char *buf)
168 {
169 	return sysfs_emit(buf, "%u\n", khugepaged_alloc_sleep_millisecs);
170 }
171 
172 static ssize_t alloc_sleep_millisecs_store(struct kobject *kobj,
173 					   struct kobj_attribute *attr,
174 					   const char *buf, size_t count)
175 {
176 	unsigned int msecs;
177 	int err;
178 
179 	err = kstrtouint(buf, 10, &msecs);
180 	if (err)
181 		return -EINVAL;
182 
183 	khugepaged_alloc_sleep_millisecs = msecs;
184 	khugepaged_sleep_expire = 0;
185 	wake_up_interruptible(&khugepaged_wait);
186 
187 	return count;
188 }
189 static struct kobj_attribute alloc_sleep_millisecs_attr =
190 	__ATTR_RW(alloc_sleep_millisecs);
191 
192 static ssize_t pages_to_scan_show(struct kobject *kobj,
193 				  struct kobj_attribute *attr,
194 				  char *buf)
195 {
196 	return sysfs_emit(buf, "%u\n", khugepaged_pages_to_scan);
197 }
198 static ssize_t pages_to_scan_store(struct kobject *kobj,
199 				   struct kobj_attribute *attr,
200 				   const char *buf, size_t count)
201 {
202 	unsigned int pages;
203 	int err;
204 
205 	err = kstrtouint(buf, 10, &pages);
206 	if (err || !pages)
207 		return -EINVAL;
208 
209 	khugepaged_pages_to_scan = pages;
210 
211 	return count;
212 }
213 static struct kobj_attribute pages_to_scan_attr =
214 	__ATTR_RW(pages_to_scan);
215 
216 static ssize_t pages_collapsed_show(struct kobject *kobj,
217 				    struct kobj_attribute *attr,
218 				    char *buf)
219 {
220 	return sysfs_emit(buf, "%u\n", khugepaged_pages_collapsed);
221 }
222 static struct kobj_attribute pages_collapsed_attr =
223 	__ATTR_RO(pages_collapsed);
224 
225 static ssize_t full_scans_show(struct kobject *kobj,
226 			       struct kobj_attribute *attr,
227 			       char *buf)
228 {
229 	return sysfs_emit(buf, "%u\n", khugepaged_full_scans);
230 }
231 static struct kobj_attribute full_scans_attr =
232 	__ATTR_RO(full_scans);
233 
234 static ssize_t defrag_show(struct kobject *kobj,
235 			   struct kobj_attribute *attr, char *buf)
236 {
237 	return single_hugepage_flag_show(kobj, attr, buf,
238 					 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG);
239 }
240 static ssize_t defrag_store(struct kobject *kobj,
241 			    struct kobj_attribute *attr,
242 			    const char *buf, size_t count)
243 {
244 	return single_hugepage_flag_store(kobj, attr, buf, count,
245 				 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG);
246 }
247 static struct kobj_attribute khugepaged_defrag_attr =
248 	__ATTR_RW(defrag);
249 
250 /*
251  * max_ptes_none controls if khugepaged should collapse hugepages over
252  * any unmapped ptes in turn potentially increasing the memory
253  * footprint of the vmas. When max_ptes_none is 0 khugepaged will not
254  * reduce the available free memory in the system as it
255  * runs. Increasing max_ptes_none will instead potentially reduce the
256  * free memory in the system during the khugepaged scan.
257  */
258 static ssize_t max_ptes_none_show(struct kobject *kobj,
259 				  struct kobj_attribute *attr,
260 				  char *buf)
261 {
262 	return sysfs_emit(buf, "%u\n", khugepaged_max_ptes_none);
263 }
264 static ssize_t max_ptes_none_store(struct kobject *kobj,
265 				   struct kobj_attribute *attr,
266 				   const char *buf, size_t count)
267 {
268 	int err;
269 	unsigned long max_ptes_none;
270 
271 	err = kstrtoul(buf, 10, &max_ptes_none);
272 	if (err || max_ptes_none > HPAGE_PMD_NR - 1)
273 		return -EINVAL;
274 
275 	khugepaged_max_ptes_none = max_ptes_none;
276 
277 	return count;
278 }
279 static struct kobj_attribute khugepaged_max_ptes_none_attr =
280 	__ATTR_RW(max_ptes_none);
281 
282 static ssize_t max_ptes_swap_show(struct kobject *kobj,
283 				  struct kobj_attribute *attr,
284 				  char *buf)
285 {
286 	return sysfs_emit(buf, "%u\n", khugepaged_max_ptes_swap);
287 }
288 
289 static ssize_t max_ptes_swap_store(struct kobject *kobj,
290 				   struct kobj_attribute *attr,
291 				   const char *buf, size_t count)
292 {
293 	int err;
294 	unsigned long max_ptes_swap;
295 
296 	err  = kstrtoul(buf, 10, &max_ptes_swap);
297 	if (err || max_ptes_swap > HPAGE_PMD_NR - 1)
298 		return -EINVAL;
299 
300 	khugepaged_max_ptes_swap = max_ptes_swap;
301 
302 	return count;
303 }
304 
305 static struct kobj_attribute khugepaged_max_ptes_swap_attr =
306 	__ATTR_RW(max_ptes_swap);
307 
308 static ssize_t max_ptes_shared_show(struct kobject *kobj,
309 				    struct kobj_attribute *attr,
310 				    char *buf)
311 {
312 	return sysfs_emit(buf, "%u\n", khugepaged_max_ptes_shared);
313 }
314 
315 static ssize_t max_ptes_shared_store(struct kobject *kobj,
316 				     struct kobj_attribute *attr,
317 				     const char *buf, size_t count)
318 {
319 	int err;
320 	unsigned long max_ptes_shared;
321 
322 	err  = kstrtoul(buf, 10, &max_ptes_shared);
323 	if (err || max_ptes_shared > HPAGE_PMD_NR - 1)
324 		return -EINVAL;
325 
326 	khugepaged_max_ptes_shared = max_ptes_shared;
327 
328 	return count;
329 }
330 
331 static struct kobj_attribute khugepaged_max_ptes_shared_attr =
332 	__ATTR_RW(max_ptes_shared);
333 
334 static struct attribute *khugepaged_attr[] = {
335 	&khugepaged_defrag_attr.attr,
336 	&khugepaged_max_ptes_none_attr.attr,
337 	&khugepaged_max_ptes_swap_attr.attr,
338 	&khugepaged_max_ptes_shared_attr.attr,
339 	&pages_to_scan_attr.attr,
340 	&pages_collapsed_attr.attr,
341 	&full_scans_attr.attr,
342 	&scan_sleep_millisecs_attr.attr,
343 	&alloc_sleep_millisecs_attr.attr,
344 	NULL,
345 };
346 
347 struct attribute_group khugepaged_attr_group = {
348 	.attrs = khugepaged_attr,
349 	.name = "khugepaged",
350 };
351 #endif /* CONFIG_SYSFS */
352 
353 int hugepage_madvise(struct vm_area_struct *vma,
354 		     unsigned long *vm_flags, int advice)
355 {
356 	switch (advice) {
357 	case MADV_HUGEPAGE:
358 #ifdef CONFIG_S390
359 		/*
360 		 * qemu blindly sets MADV_HUGEPAGE on all allocations, but s390
361 		 * can't handle this properly after s390_enable_sie, so we simply
362 		 * ignore the madvise to prevent qemu from causing a SIGSEGV.
363 		 */
364 		if (mm_has_pgste(vma->vm_mm))
365 			return 0;
366 #endif
367 		*vm_flags &= ~VM_NOHUGEPAGE;
368 		*vm_flags |= VM_HUGEPAGE;
369 		/*
370 		 * If the vma become good for khugepaged to scan,
371 		 * register it here without waiting a page fault that
372 		 * may not happen any time soon.
373 		 */
374 		khugepaged_enter_vma(vma, *vm_flags);
375 		break;
376 	case MADV_NOHUGEPAGE:
377 		*vm_flags &= ~VM_HUGEPAGE;
378 		*vm_flags |= VM_NOHUGEPAGE;
379 		/*
380 		 * Setting VM_NOHUGEPAGE will prevent khugepaged from scanning
381 		 * this vma even if we leave the mm registered in khugepaged if
382 		 * it got registered before VM_NOHUGEPAGE was set.
383 		 */
384 		break;
385 	}
386 
387 	return 0;
388 }
389 
390 int __init khugepaged_init(void)
391 {
392 	mm_slot_cache = kmem_cache_create("khugepaged_mm_slot",
393 					  sizeof(struct khugepaged_mm_slot),
394 					  __alignof__(struct khugepaged_mm_slot),
395 					  0, NULL);
396 	if (!mm_slot_cache)
397 		return -ENOMEM;
398 
399 	khugepaged_pages_to_scan = HPAGE_PMD_NR * 8;
400 	khugepaged_max_ptes_none = HPAGE_PMD_NR - 1;
401 	khugepaged_max_ptes_swap = HPAGE_PMD_NR / 8;
402 	khugepaged_max_ptes_shared = HPAGE_PMD_NR / 2;
403 
404 	return 0;
405 }
406 
407 void __init khugepaged_destroy(void)
408 {
409 	kmem_cache_destroy(mm_slot_cache);
410 }
411 
412 static inline int hpage_collapse_test_exit(struct mm_struct *mm)
413 {
414 	return atomic_read(&mm->mm_users) == 0;
415 }
416 
417 void __khugepaged_enter(struct mm_struct *mm)
418 {
419 	struct khugepaged_mm_slot *mm_slot;
420 	struct mm_slot *slot;
421 	int wakeup;
422 
423 	mm_slot = mm_slot_alloc(mm_slot_cache);
424 	if (!mm_slot)
425 		return;
426 
427 	slot = &mm_slot->slot;
428 
429 	/* __khugepaged_exit() must not run from under us */
430 	VM_BUG_ON_MM(hpage_collapse_test_exit(mm), mm);
431 	if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) {
432 		mm_slot_free(mm_slot_cache, mm_slot);
433 		return;
434 	}
435 
436 	spin_lock(&khugepaged_mm_lock);
437 	mm_slot_insert(mm_slots_hash, mm, slot);
438 	/*
439 	 * Insert just behind the scanning cursor, to let the area settle
440 	 * down a little.
441 	 */
442 	wakeup = list_empty(&khugepaged_scan.mm_head);
443 	list_add_tail(&slot->mm_node, &khugepaged_scan.mm_head);
444 	spin_unlock(&khugepaged_mm_lock);
445 
446 	mmgrab(mm);
447 	if (wakeup)
448 		wake_up_interruptible(&khugepaged_wait);
449 }
450 
451 void khugepaged_enter_vma(struct vm_area_struct *vma,
452 			  unsigned long vm_flags)
453 {
454 	if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags) &&
455 	    hugepage_flags_enabled()) {
456 		if (hugepage_vma_check(vma, vm_flags, false, false, true))
457 			__khugepaged_enter(vma->vm_mm);
458 	}
459 }
460 
461 void __khugepaged_exit(struct mm_struct *mm)
462 {
463 	struct khugepaged_mm_slot *mm_slot;
464 	struct mm_slot *slot;
465 	int free = 0;
466 
467 	spin_lock(&khugepaged_mm_lock);
468 	slot = mm_slot_lookup(mm_slots_hash, mm);
469 	mm_slot = mm_slot_entry(slot, struct khugepaged_mm_slot, slot);
470 	if (mm_slot && khugepaged_scan.mm_slot != mm_slot) {
471 		hash_del(&slot->hash);
472 		list_del(&slot->mm_node);
473 		free = 1;
474 	}
475 	spin_unlock(&khugepaged_mm_lock);
476 
477 	if (free) {
478 		clear_bit(MMF_VM_HUGEPAGE, &mm->flags);
479 		mm_slot_free(mm_slot_cache, mm_slot);
480 		mmdrop(mm);
481 	} else if (mm_slot) {
482 		/*
483 		 * This is required to serialize against
484 		 * hpage_collapse_test_exit() (which is guaranteed to run
485 		 * under mmap sem read mode). Stop here (after we return all
486 		 * pagetables will be destroyed) until khugepaged has finished
487 		 * working on the pagetables under the mmap_lock.
488 		 */
489 		mmap_write_lock(mm);
490 		mmap_write_unlock(mm);
491 	}
492 }
493 
494 static void release_pte_folio(struct folio *folio)
495 {
496 	node_stat_mod_folio(folio,
497 			NR_ISOLATED_ANON + folio_is_file_lru(folio),
498 			-folio_nr_pages(folio));
499 	folio_unlock(folio);
500 	folio_putback_lru(folio);
501 }
502 
503 static void release_pte_page(struct page *page)
504 {
505 	release_pte_folio(page_folio(page));
506 }
507 
508 static void release_pte_pages(pte_t *pte, pte_t *_pte,
509 		struct list_head *compound_pagelist)
510 {
511 	struct folio *folio, *tmp;
512 
513 	while (--_pte >= pte) {
514 		pte_t pteval = *_pte;
515 		unsigned long pfn;
516 
517 		if (pte_none(pteval))
518 			continue;
519 		pfn = pte_pfn(pteval);
520 		if (is_zero_pfn(pfn))
521 			continue;
522 		folio = pfn_folio(pfn);
523 		if (folio_test_large(folio))
524 			continue;
525 		release_pte_folio(folio);
526 	}
527 
528 	list_for_each_entry_safe(folio, tmp, compound_pagelist, lru) {
529 		list_del(&folio->lru);
530 		release_pte_folio(folio);
531 	}
532 }
533 
534 static bool is_refcount_suitable(struct page *page)
535 {
536 	int expected_refcount;
537 
538 	expected_refcount = total_mapcount(page);
539 	if (PageSwapCache(page))
540 		expected_refcount += compound_nr(page);
541 
542 	return page_count(page) == expected_refcount;
543 }
544 
545 static int __collapse_huge_page_isolate(struct vm_area_struct *vma,
546 					unsigned long address,
547 					pte_t *pte,
548 					struct collapse_control *cc,
549 					struct list_head *compound_pagelist)
550 {
551 	struct page *page = NULL;
552 	pte_t *_pte;
553 	int none_or_zero = 0, shared = 0, result = SCAN_FAIL, referenced = 0;
554 	bool writable = false;
555 
556 	for (_pte = pte; _pte < pte + HPAGE_PMD_NR;
557 	     _pte++, address += PAGE_SIZE) {
558 		pte_t pteval = *_pte;
559 		if (pte_none(pteval) || (pte_present(pteval) &&
560 				is_zero_pfn(pte_pfn(pteval)))) {
561 			++none_or_zero;
562 			if (!userfaultfd_armed(vma) &&
563 			    (!cc->is_khugepaged ||
564 			     none_or_zero <= khugepaged_max_ptes_none)) {
565 				continue;
566 			} else {
567 				result = SCAN_EXCEED_NONE_PTE;
568 				count_vm_event(THP_SCAN_EXCEED_NONE_PTE);
569 				goto out;
570 			}
571 		}
572 		if (!pte_present(pteval)) {
573 			result = SCAN_PTE_NON_PRESENT;
574 			goto out;
575 		}
576 		if (pte_uffd_wp(pteval)) {
577 			result = SCAN_PTE_UFFD_WP;
578 			goto out;
579 		}
580 		page = vm_normal_page(vma, address, pteval);
581 		if (unlikely(!page) || unlikely(is_zone_device_page(page))) {
582 			result = SCAN_PAGE_NULL;
583 			goto out;
584 		}
585 
586 		VM_BUG_ON_PAGE(!PageAnon(page), page);
587 
588 		if (page_mapcount(page) > 1) {
589 			++shared;
590 			if (cc->is_khugepaged &&
591 			    shared > khugepaged_max_ptes_shared) {
592 				result = SCAN_EXCEED_SHARED_PTE;
593 				count_vm_event(THP_SCAN_EXCEED_SHARED_PTE);
594 				goto out;
595 			}
596 		}
597 
598 		if (PageCompound(page)) {
599 			struct page *p;
600 			page = compound_head(page);
601 
602 			/*
603 			 * Check if we have dealt with the compound page
604 			 * already
605 			 */
606 			list_for_each_entry(p, compound_pagelist, lru) {
607 				if (page == p)
608 					goto next;
609 			}
610 		}
611 
612 		/*
613 		 * We can do it before isolate_lru_page because the
614 		 * page can't be freed from under us. NOTE: PG_lock
615 		 * is needed to serialize against split_huge_page
616 		 * when invoked from the VM.
617 		 */
618 		if (!trylock_page(page)) {
619 			result = SCAN_PAGE_LOCK;
620 			goto out;
621 		}
622 
623 		/*
624 		 * Check if the page has any GUP (or other external) pins.
625 		 *
626 		 * The page table that maps the page has been already unlinked
627 		 * from the page table tree and this process cannot get
628 		 * an additional pin on the page.
629 		 *
630 		 * New pins can come later if the page is shared across fork,
631 		 * but not from this process. The other process cannot write to
632 		 * the page, only trigger CoW.
633 		 */
634 		if (!is_refcount_suitable(page)) {
635 			unlock_page(page);
636 			result = SCAN_PAGE_COUNT;
637 			goto out;
638 		}
639 
640 		/*
641 		 * Isolate the page to avoid collapsing an hugepage
642 		 * currently in use by the VM.
643 		 */
644 		if (!isolate_lru_page(page)) {
645 			unlock_page(page);
646 			result = SCAN_DEL_PAGE_LRU;
647 			goto out;
648 		}
649 		mod_node_page_state(page_pgdat(page),
650 				NR_ISOLATED_ANON + page_is_file_lru(page),
651 				compound_nr(page));
652 		VM_BUG_ON_PAGE(!PageLocked(page), page);
653 		VM_BUG_ON_PAGE(PageLRU(page), page);
654 
655 		if (PageCompound(page))
656 			list_add_tail(&page->lru, compound_pagelist);
657 next:
658 		/*
659 		 * If collapse was initiated by khugepaged, check that there is
660 		 * enough young pte to justify collapsing the page
661 		 */
662 		if (cc->is_khugepaged &&
663 		    (pte_young(pteval) || page_is_young(page) ||
664 		     PageReferenced(page) || mmu_notifier_test_young(vma->vm_mm,
665 								     address)))
666 			referenced++;
667 
668 		if (pte_write(pteval))
669 			writable = true;
670 	}
671 
672 	if (unlikely(!writable)) {
673 		result = SCAN_PAGE_RO;
674 	} else if (unlikely(cc->is_khugepaged && !referenced)) {
675 		result = SCAN_LACK_REFERENCED_PAGE;
676 	} else {
677 		result = SCAN_SUCCEED;
678 		trace_mm_collapse_huge_page_isolate(page, none_or_zero,
679 						    referenced, writable, result);
680 		return result;
681 	}
682 out:
683 	release_pte_pages(pte, _pte, compound_pagelist);
684 	trace_mm_collapse_huge_page_isolate(page, none_or_zero,
685 					    referenced, writable, result);
686 	return result;
687 }
688 
689 static void __collapse_huge_page_copy(pte_t *pte, struct page *page,
690 				      struct vm_area_struct *vma,
691 				      unsigned long address,
692 				      spinlock_t *ptl,
693 				      struct list_head *compound_pagelist)
694 {
695 	struct page *src_page, *tmp;
696 	pte_t *_pte;
697 	for (_pte = pte; _pte < pte + HPAGE_PMD_NR;
698 				_pte++, page++, address += PAGE_SIZE) {
699 		pte_t pteval = *_pte;
700 
701 		if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) {
702 			clear_user_highpage(page, address);
703 			add_mm_counter(vma->vm_mm, MM_ANONPAGES, 1);
704 			if (is_zero_pfn(pte_pfn(pteval))) {
705 				/*
706 				 * ptl mostly unnecessary.
707 				 */
708 				spin_lock(ptl);
709 				ptep_clear(vma->vm_mm, address, _pte);
710 				spin_unlock(ptl);
711 			}
712 		} else {
713 			src_page = pte_page(pteval);
714 			copy_user_highpage(page, src_page, address, vma);
715 			if (!PageCompound(src_page))
716 				release_pte_page(src_page);
717 			/*
718 			 * ptl mostly unnecessary, but preempt has to
719 			 * be disabled to update the per-cpu stats
720 			 * inside page_remove_rmap().
721 			 */
722 			spin_lock(ptl);
723 			ptep_clear(vma->vm_mm, address, _pte);
724 			page_remove_rmap(src_page, vma, false);
725 			spin_unlock(ptl);
726 			free_page_and_swap_cache(src_page);
727 		}
728 	}
729 
730 	list_for_each_entry_safe(src_page, tmp, compound_pagelist, lru) {
731 		list_del(&src_page->lru);
732 		mod_node_page_state(page_pgdat(src_page),
733 				    NR_ISOLATED_ANON + page_is_file_lru(src_page),
734 				    -compound_nr(src_page));
735 		unlock_page(src_page);
736 		free_swap_cache(src_page);
737 		putback_lru_page(src_page);
738 	}
739 }
740 
741 static void khugepaged_alloc_sleep(void)
742 {
743 	DEFINE_WAIT(wait);
744 
745 	add_wait_queue(&khugepaged_wait, &wait);
746 	__set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE);
747 	schedule_timeout(msecs_to_jiffies(khugepaged_alloc_sleep_millisecs));
748 	remove_wait_queue(&khugepaged_wait, &wait);
749 }
750 
751 struct collapse_control khugepaged_collapse_control = {
752 	.is_khugepaged = true,
753 };
754 
755 static bool hpage_collapse_scan_abort(int nid, struct collapse_control *cc)
756 {
757 	int i;
758 
759 	/*
760 	 * If node_reclaim_mode is disabled, then no extra effort is made to
761 	 * allocate memory locally.
762 	 */
763 	if (!node_reclaim_enabled())
764 		return false;
765 
766 	/* If there is a count for this node already, it must be acceptable */
767 	if (cc->node_load[nid])
768 		return false;
769 
770 	for (i = 0; i < MAX_NUMNODES; i++) {
771 		if (!cc->node_load[i])
772 			continue;
773 		if (node_distance(nid, i) > node_reclaim_distance)
774 			return true;
775 	}
776 	return false;
777 }
778 
779 #define khugepaged_defrag()					\
780 	(transparent_hugepage_flags &				\
781 	 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG))
782 
783 /* Defrag for khugepaged will enter direct reclaim/compaction if necessary */
784 static inline gfp_t alloc_hugepage_khugepaged_gfpmask(void)
785 {
786 	return khugepaged_defrag() ? GFP_TRANSHUGE : GFP_TRANSHUGE_LIGHT;
787 }
788 
789 #ifdef CONFIG_NUMA
790 static int hpage_collapse_find_target_node(struct collapse_control *cc)
791 {
792 	int nid, target_node = 0, max_value = 0;
793 
794 	/* find first node with max normal pages hit */
795 	for (nid = 0; nid < MAX_NUMNODES; nid++)
796 		if (cc->node_load[nid] > max_value) {
797 			max_value = cc->node_load[nid];
798 			target_node = nid;
799 		}
800 
801 	for_each_online_node(nid) {
802 		if (max_value == cc->node_load[nid])
803 			node_set(nid, cc->alloc_nmask);
804 	}
805 
806 	return target_node;
807 }
808 #else
809 static int hpage_collapse_find_target_node(struct collapse_control *cc)
810 {
811 	return 0;
812 }
813 #endif
814 
815 static bool hpage_collapse_alloc_page(struct page **hpage, gfp_t gfp, int node,
816 				      nodemask_t *nmask)
817 {
818 	*hpage = __alloc_pages(gfp, HPAGE_PMD_ORDER, node, nmask);
819 	if (unlikely(!*hpage)) {
820 		count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
821 		return false;
822 	}
823 
824 	prep_transhuge_page(*hpage);
825 	count_vm_event(THP_COLLAPSE_ALLOC);
826 	return true;
827 }
828 
829 /*
830  * If mmap_lock temporarily dropped, revalidate vma
831  * before taking mmap_lock.
832  * Returns enum scan_result value.
833  */
834 
835 static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address,
836 				   bool expect_anon,
837 				   struct vm_area_struct **vmap,
838 				   struct collapse_control *cc)
839 {
840 	struct vm_area_struct *vma;
841 
842 	if (unlikely(hpage_collapse_test_exit(mm)))
843 		return SCAN_ANY_PROCESS;
844 
845 	*vmap = vma = find_vma(mm, address);
846 	if (!vma)
847 		return SCAN_VMA_NULL;
848 
849 	if (!transhuge_vma_suitable(vma, address))
850 		return SCAN_ADDRESS_RANGE;
851 	if (!hugepage_vma_check(vma, vma->vm_flags, false, false,
852 				cc->is_khugepaged))
853 		return SCAN_VMA_CHECK;
854 	/*
855 	 * Anon VMA expected, the address may be unmapped then
856 	 * remapped to file after khugepaged reaquired the mmap_lock.
857 	 *
858 	 * hugepage_vma_check may return true for qualified file
859 	 * vmas.
860 	 */
861 	if (expect_anon && (!(*vmap)->anon_vma || !vma_is_anonymous(*vmap)))
862 		return SCAN_PAGE_ANON;
863 	return SCAN_SUCCEED;
864 }
865 
866 /*
867  * See pmd_trans_unstable() for how the result may change out from
868  * underneath us, even if we hold mmap_lock in read.
869  */
870 static int find_pmd_or_thp_or_none(struct mm_struct *mm,
871 				   unsigned long address,
872 				   pmd_t **pmd)
873 {
874 	pmd_t pmde;
875 
876 	*pmd = mm_find_pmd(mm, address);
877 	if (!*pmd)
878 		return SCAN_PMD_NULL;
879 
880 	pmde = pmdp_get_lockless(*pmd);
881 
882 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
883 	/* See comments in pmd_none_or_trans_huge_or_clear_bad() */
884 	barrier();
885 #endif
886 	if (pmd_none(pmde))
887 		return SCAN_PMD_NONE;
888 	if (!pmd_present(pmde))
889 		return SCAN_PMD_NULL;
890 	if (pmd_trans_huge(pmde))
891 		return SCAN_PMD_MAPPED;
892 	if (pmd_devmap(pmde))
893 		return SCAN_PMD_NULL;
894 	if (pmd_bad(pmde))
895 		return SCAN_PMD_NULL;
896 	return SCAN_SUCCEED;
897 }
898 
899 static int check_pmd_still_valid(struct mm_struct *mm,
900 				 unsigned long address,
901 				 pmd_t *pmd)
902 {
903 	pmd_t *new_pmd;
904 	int result = find_pmd_or_thp_or_none(mm, address, &new_pmd);
905 
906 	if (result != SCAN_SUCCEED)
907 		return result;
908 	if (new_pmd != pmd)
909 		return SCAN_FAIL;
910 	return SCAN_SUCCEED;
911 }
912 
913 /*
914  * Bring missing pages in from swap, to complete THP collapse.
915  * Only done if hpage_collapse_scan_pmd believes it is worthwhile.
916  *
917  * Called and returns without pte mapped or spinlocks held.
918  * Note that if false is returned, mmap_lock will be released.
919  */
920 
921 static int __collapse_huge_page_swapin(struct mm_struct *mm,
922 				       struct vm_area_struct *vma,
923 				       unsigned long haddr, pmd_t *pmd,
924 				       int referenced)
925 {
926 	int swapped_in = 0;
927 	vm_fault_t ret = 0;
928 	unsigned long address, end = haddr + (HPAGE_PMD_NR * PAGE_SIZE);
929 
930 	for (address = haddr; address < end; address += PAGE_SIZE) {
931 		struct vm_fault vmf = {
932 			.vma = vma,
933 			.address = address,
934 			.pgoff = linear_page_index(vma, haddr),
935 			.flags = FAULT_FLAG_ALLOW_RETRY,
936 			.pmd = pmd,
937 		};
938 
939 		vmf.pte = pte_offset_map(pmd, address);
940 		vmf.orig_pte = *vmf.pte;
941 		if (!is_swap_pte(vmf.orig_pte)) {
942 			pte_unmap(vmf.pte);
943 			continue;
944 		}
945 		ret = do_swap_page(&vmf);
946 
947 		/*
948 		 * do_swap_page returns VM_FAULT_RETRY with released mmap_lock.
949 		 * Note we treat VM_FAULT_RETRY as VM_FAULT_ERROR here because
950 		 * we do not retry here and swap entry will remain in pagetable
951 		 * resulting in later failure.
952 		 */
953 		if (ret & VM_FAULT_RETRY) {
954 			trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0);
955 			/* Likely, but not guaranteed, that page lock failed */
956 			return SCAN_PAGE_LOCK;
957 		}
958 		if (ret & VM_FAULT_ERROR) {
959 			mmap_read_unlock(mm);
960 			trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0);
961 			return SCAN_FAIL;
962 		}
963 		swapped_in++;
964 	}
965 
966 	/* Drain LRU add pagevec to remove extra pin on the swapped in pages */
967 	if (swapped_in)
968 		lru_add_drain();
969 
970 	trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 1);
971 	return SCAN_SUCCEED;
972 }
973 
974 static int alloc_charge_hpage(struct page **hpage, struct mm_struct *mm,
975 			      struct collapse_control *cc)
976 {
977 	gfp_t gfp = (cc->is_khugepaged ? alloc_hugepage_khugepaged_gfpmask() :
978 		     GFP_TRANSHUGE);
979 	int node = hpage_collapse_find_target_node(cc);
980 	struct folio *folio;
981 
982 	if (!hpage_collapse_alloc_page(hpage, gfp, node, &cc->alloc_nmask))
983 		return SCAN_ALLOC_HUGE_PAGE_FAIL;
984 
985 	folio = page_folio(*hpage);
986 	if (unlikely(mem_cgroup_charge(folio, mm, gfp))) {
987 		folio_put(folio);
988 		*hpage = NULL;
989 		return SCAN_CGROUP_CHARGE_FAIL;
990 	}
991 	count_memcg_page_event(*hpage, THP_COLLAPSE_ALLOC);
992 
993 	return SCAN_SUCCEED;
994 }
995 
996 static int collapse_huge_page(struct mm_struct *mm, unsigned long address,
997 			      int referenced, int unmapped,
998 			      struct collapse_control *cc)
999 {
1000 	LIST_HEAD(compound_pagelist);
1001 	pmd_t *pmd, _pmd;
1002 	pte_t *pte;
1003 	pgtable_t pgtable;
1004 	struct page *hpage;
1005 	spinlock_t *pmd_ptl, *pte_ptl;
1006 	int result = SCAN_FAIL;
1007 	struct vm_area_struct *vma;
1008 	struct mmu_notifier_range range;
1009 
1010 	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
1011 
1012 	/*
1013 	 * Before allocating the hugepage, release the mmap_lock read lock.
1014 	 * The allocation can take potentially a long time if it involves
1015 	 * sync compaction, and we do not need to hold the mmap_lock during
1016 	 * that. We will recheck the vma after taking it again in write mode.
1017 	 */
1018 	mmap_read_unlock(mm);
1019 
1020 	result = alloc_charge_hpage(&hpage, mm, cc);
1021 	if (result != SCAN_SUCCEED)
1022 		goto out_nolock;
1023 
1024 	mmap_read_lock(mm);
1025 	result = hugepage_vma_revalidate(mm, address, true, &vma, cc);
1026 	if (result != SCAN_SUCCEED) {
1027 		mmap_read_unlock(mm);
1028 		goto out_nolock;
1029 	}
1030 
1031 	result = find_pmd_or_thp_or_none(mm, address, &pmd);
1032 	if (result != SCAN_SUCCEED) {
1033 		mmap_read_unlock(mm);
1034 		goto out_nolock;
1035 	}
1036 
1037 	if (unmapped) {
1038 		/*
1039 		 * __collapse_huge_page_swapin will return with mmap_lock
1040 		 * released when it fails. So we jump out_nolock directly in
1041 		 * that case.  Continuing to collapse causes inconsistency.
1042 		 */
1043 		result = __collapse_huge_page_swapin(mm, vma, address, pmd,
1044 						     referenced);
1045 		if (result != SCAN_SUCCEED)
1046 			goto out_nolock;
1047 	}
1048 
1049 	mmap_read_unlock(mm);
1050 	/*
1051 	 * Prevent all access to pagetables with the exception of
1052 	 * gup_fast later handled by the ptep_clear_flush and the VM
1053 	 * handled by the anon_vma lock + PG_lock.
1054 	 */
1055 	mmap_write_lock(mm);
1056 	result = hugepage_vma_revalidate(mm, address, true, &vma, cc);
1057 	if (result != SCAN_SUCCEED)
1058 		goto out_up_write;
1059 	/* check if the pmd is still valid */
1060 	result = check_pmd_still_valid(mm, address, pmd);
1061 	if (result != SCAN_SUCCEED)
1062 		goto out_up_write;
1063 
1064 	vma_start_write(vma);
1065 	anon_vma_lock_write(vma->anon_vma);
1066 
1067 	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, address,
1068 				address + HPAGE_PMD_SIZE);
1069 	mmu_notifier_invalidate_range_start(&range);
1070 
1071 	pte = pte_offset_map(pmd, address);
1072 	pte_ptl = pte_lockptr(mm, pmd);
1073 
1074 	pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */
1075 	/*
1076 	 * This removes any huge TLB entry from the CPU so we won't allow
1077 	 * huge and small TLB entries for the same virtual address to
1078 	 * avoid the risk of CPU bugs in that area.
1079 	 *
1080 	 * Parallel fast GUP is fine since fast GUP will back off when
1081 	 * it detects PMD is changed.
1082 	 */
1083 	_pmd = pmdp_collapse_flush(vma, address, pmd);
1084 	spin_unlock(pmd_ptl);
1085 	mmu_notifier_invalidate_range_end(&range);
1086 	tlb_remove_table_sync_one();
1087 
1088 	spin_lock(pte_ptl);
1089 	result =  __collapse_huge_page_isolate(vma, address, pte, cc,
1090 					       &compound_pagelist);
1091 	spin_unlock(pte_ptl);
1092 
1093 	if (unlikely(result != SCAN_SUCCEED)) {
1094 		pte_unmap(pte);
1095 		spin_lock(pmd_ptl);
1096 		BUG_ON(!pmd_none(*pmd));
1097 		/*
1098 		 * We can only use set_pmd_at when establishing
1099 		 * hugepmds and never for establishing regular pmds that
1100 		 * points to regular pagetables. Use pmd_populate for that
1101 		 */
1102 		pmd_populate(mm, pmd, pmd_pgtable(_pmd));
1103 		spin_unlock(pmd_ptl);
1104 		anon_vma_unlock_write(vma->anon_vma);
1105 		goto out_up_write;
1106 	}
1107 
1108 	/*
1109 	 * All pages are isolated and locked so anon_vma rmap
1110 	 * can't run anymore.
1111 	 */
1112 	anon_vma_unlock_write(vma->anon_vma);
1113 
1114 	__collapse_huge_page_copy(pte, hpage, vma, address, pte_ptl,
1115 				  &compound_pagelist);
1116 	pte_unmap(pte);
1117 	/*
1118 	 * spin_lock() below is not the equivalent of smp_wmb(), but
1119 	 * the smp_wmb() inside __SetPageUptodate() can be reused to
1120 	 * avoid the copy_huge_page writes to become visible after
1121 	 * the set_pmd_at() write.
1122 	 */
1123 	__SetPageUptodate(hpage);
1124 	pgtable = pmd_pgtable(_pmd);
1125 
1126 	_pmd = mk_huge_pmd(hpage, vma->vm_page_prot);
1127 	_pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma);
1128 
1129 	spin_lock(pmd_ptl);
1130 	BUG_ON(!pmd_none(*pmd));
1131 	page_add_new_anon_rmap(hpage, vma, address);
1132 	lru_cache_add_inactive_or_unevictable(hpage, vma);
1133 	pgtable_trans_huge_deposit(mm, pmd, pgtable);
1134 	set_pmd_at(mm, address, pmd, _pmd);
1135 	update_mmu_cache_pmd(vma, address, pmd);
1136 	spin_unlock(pmd_ptl);
1137 
1138 	hpage = NULL;
1139 
1140 	result = SCAN_SUCCEED;
1141 out_up_write:
1142 	mmap_write_unlock(mm);
1143 out_nolock:
1144 	if (hpage)
1145 		put_page(hpage);
1146 	trace_mm_collapse_huge_page(mm, result == SCAN_SUCCEED, result);
1147 	return result;
1148 }
1149 
1150 static int hpage_collapse_scan_pmd(struct mm_struct *mm,
1151 				   struct vm_area_struct *vma,
1152 				   unsigned long address, bool *mmap_locked,
1153 				   struct collapse_control *cc)
1154 {
1155 	pmd_t *pmd;
1156 	pte_t *pte, *_pte;
1157 	int result = SCAN_FAIL, referenced = 0;
1158 	int none_or_zero = 0, shared = 0;
1159 	struct page *page = NULL;
1160 	unsigned long _address;
1161 	spinlock_t *ptl;
1162 	int node = NUMA_NO_NODE, unmapped = 0;
1163 	bool writable = false;
1164 
1165 	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
1166 
1167 	result = find_pmd_or_thp_or_none(mm, address, &pmd);
1168 	if (result != SCAN_SUCCEED)
1169 		goto out;
1170 
1171 	memset(cc->node_load, 0, sizeof(cc->node_load));
1172 	nodes_clear(cc->alloc_nmask);
1173 	pte = pte_offset_map_lock(mm, pmd, address, &ptl);
1174 	for (_address = address, _pte = pte; _pte < pte + HPAGE_PMD_NR;
1175 	     _pte++, _address += PAGE_SIZE) {
1176 		pte_t pteval = *_pte;
1177 		if (is_swap_pte(pteval)) {
1178 			++unmapped;
1179 			if (!cc->is_khugepaged ||
1180 			    unmapped <= khugepaged_max_ptes_swap) {
1181 				/*
1182 				 * Always be strict with uffd-wp
1183 				 * enabled swap entries.  Please see
1184 				 * comment below for pte_uffd_wp().
1185 				 */
1186 				if (pte_swp_uffd_wp_any(pteval)) {
1187 					result = SCAN_PTE_UFFD_WP;
1188 					goto out_unmap;
1189 				}
1190 				continue;
1191 			} else {
1192 				result = SCAN_EXCEED_SWAP_PTE;
1193 				count_vm_event(THP_SCAN_EXCEED_SWAP_PTE);
1194 				goto out_unmap;
1195 			}
1196 		}
1197 		if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) {
1198 			++none_or_zero;
1199 			if (!userfaultfd_armed(vma) &&
1200 			    (!cc->is_khugepaged ||
1201 			     none_or_zero <= khugepaged_max_ptes_none)) {
1202 				continue;
1203 			} else {
1204 				result = SCAN_EXCEED_NONE_PTE;
1205 				count_vm_event(THP_SCAN_EXCEED_NONE_PTE);
1206 				goto out_unmap;
1207 			}
1208 		}
1209 		if (pte_uffd_wp(pteval)) {
1210 			/*
1211 			 * Don't collapse the page if any of the small
1212 			 * PTEs are armed with uffd write protection.
1213 			 * Here we can also mark the new huge pmd as
1214 			 * write protected if any of the small ones is
1215 			 * marked but that could bring unknown
1216 			 * userfault messages that falls outside of
1217 			 * the registered range.  So, just be simple.
1218 			 */
1219 			result = SCAN_PTE_UFFD_WP;
1220 			goto out_unmap;
1221 		}
1222 		if (pte_write(pteval))
1223 			writable = true;
1224 
1225 		page = vm_normal_page(vma, _address, pteval);
1226 		if (unlikely(!page) || unlikely(is_zone_device_page(page))) {
1227 			result = SCAN_PAGE_NULL;
1228 			goto out_unmap;
1229 		}
1230 
1231 		if (page_mapcount(page) > 1) {
1232 			++shared;
1233 			if (cc->is_khugepaged &&
1234 			    shared > khugepaged_max_ptes_shared) {
1235 				result = SCAN_EXCEED_SHARED_PTE;
1236 				count_vm_event(THP_SCAN_EXCEED_SHARED_PTE);
1237 				goto out_unmap;
1238 			}
1239 		}
1240 
1241 		page = compound_head(page);
1242 
1243 		/*
1244 		 * Record which node the original page is from and save this
1245 		 * information to cc->node_load[].
1246 		 * Khugepaged will allocate hugepage from the node has the max
1247 		 * hit record.
1248 		 */
1249 		node = page_to_nid(page);
1250 		if (hpage_collapse_scan_abort(node, cc)) {
1251 			result = SCAN_SCAN_ABORT;
1252 			goto out_unmap;
1253 		}
1254 		cc->node_load[node]++;
1255 		if (!PageLRU(page)) {
1256 			result = SCAN_PAGE_LRU;
1257 			goto out_unmap;
1258 		}
1259 		if (PageLocked(page)) {
1260 			result = SCAN_PAGE_LOCK;
1261 			goto out_unmap;
1262 		}
1263 		if (!PageAnon(page)) {
1264 			result = SCAN_PAGE_ANON;
1265 			goto out_unmap;
1266 		}
1267 
1268 		/*
1269 		 * Check if the page has any GUP (or other external) pins.
1270 		 *
1271 		 * Here the check may be racy:
1272 		 * it may see total_mapcount > refcount in some cases?
1273 		 * But such case is ephemeral we could always retry collapse
1274 		 * later.  However it may report false positive if the page
1275 		 * has excessive GUP pins (i.e. 512).  Anyway the same check
1276 		 * will be done again later the risk seems low.
1277 		 */
1278 		if (!is_refcount_suitable(page)) {
1279 			result = SCAN_PAGE_COUNT;
1280 			goto out_unmap;
1281 		}
1282 
1283 		/*
1284 		 * If collapse was initiated by khugepaged, check that there is
1285 		 * enough young pte to justify collapsing the page
1286 		 */
1287 		if (cc->is_khugepaged &&
1288 		    (pte_young(pteval) || page_is_young(page) ||
1289 		     PageReferenced(page) || mmu_notifier_test_young(vma->vm_mm,
1290 								     address)))
1291 			referenced++;
1292 	}
1293 	if (!writable) {
1294 		result = SCAN_PAGE_RO;
1295 	} else if (cc->is_khugepaged &&
1296 		   (!referenced ||
1297 		    (unmapped && referenced < HPAGE_PMD_NR / 2))) {
1298 		result = SCAN_LACK_REFERENCED_PAGE;
1299 	} else {
1300 		result = SCAN_SUCCEED;
1301 	}
1302 out_unmap:
1303 	pte_unmap_unlock(pte, ptl);
1304 	if (result == SCAN_SUCCEED) {
1305 		result = collapse_huge_page(mm, address, referenced,
1306 					    unmapped, cc);
1307 		/* collapse_huge_page will return with the mmap_lock released */
1308 		*mmap_locked = false;
1309 	}
1310 out:
1311 	trace_mm_khugepaged_scan_pmd(mm, page, writable, referenced,
1312 				     none_or_zero, result, unmapped);
1313 	return result;
1314 }
1315 
1316 static void collect_mm_slot(struct khugepaged_mm_slot *mm_slot)
1317 {
1318 	struct mm_slot *slot = &mm_slot->slot;
1319 	struct mm_struct *mm = slot->mm;
1320 
1321 	lockdep_assert_held(&khugepaged_mm_lock);
1322 
1323 	if (hpage_collapse_test_exit(mm)) {
1324 		/* free mm_slot */
1325 		hash_del(&slot->hash);
1326 		list_del(&slot->mm_node);
1327 
1328 		/*
1329 		 * Not strictly needed because the mm exited already.
1330 		 *
1331 		 * clear_bit(MMF_VM_HUGEPAGE, &mm->flags);
1332 		 */
1333 
1334 		/* khugepaged_mm_lock actually not necessary for the below */
1335 		mm_slot_free(mm_slot_cache, mm_slot);
1336 		mmdrop(mm);
1337 	}
1338 }
1339 
1340 #ifdef CONFIG_SHMEM
1341 /*
1342  * Notify khugepaged that given addr of the mm is pte-mapped THP. Then
1343  * khugepaged should try to collapse the page table.
1344  *
1345  * Note that following race exists:
1346  * (1) khugepaged calls khugepaged_collapse_pte_mapped_thps() for mm_struct A,
1347  *     emptying the A's ->pte_mapped_thp[] array.
1348  * (2) MADV_COLLAPSE collapses some file extent with target mm_struct B, and
1349  *     retract_page_tables() finds a VMA in mm_struct A mapping the same extent
1350  *     (at virtual address X) and adds an entry (for X) into mm_struct A's
1351  *     ->pte-mapped_thp[] array.
1352  * (3) khugepaged calls khugepaged_collapse_scan_file() for mm_struct A at X,
1353  *     sees a pte-mapped THP (SCAN_PTE_MAPPED_HUGEPAGE) and adds an entry
1354  *     (for X) into mm_struct A's ->pte-mapped_thp[] array.
1355  * Thus, it's possible the same address is added multiple times for the same
1356  * mm_struct.  Should this happen, we'll simply attempt
1357  * collapse_pte_mapped_thp() multiple times for the same address, under the same
1358  * exclusive mmap_lock, and assuming the first call is successful, subsequent
1359  * attempts will return quickly (without grabbing any additional locks) when
1360  * a huge pmd is found in find_pmd_or_thp_or_none().  Since this is a cheap
1361  * check, and since this is a rare occurrence, the cost of preventing this
1362  * "multiple-add" is thought to be more expensive than just handling it, should
1363  * it occur.
1364  */
1365 static bool khugepaged_add_pte_mapped_thp(struct mm_struct *mm,
1366 					  unsigned long addr)
1367 {
1368 	struct khugepaged_mm_slot *mm_slot;
1369 	struct mm_slot *slot;
1370 	bool ret = false;
1371 
1372 	VM_BUG_ON(addr & ~HPAGE_PMD_MASK);
1373 
1374 	spin_lock(&khugepaged_mm_lock);
1375 	slot = mm_slot_lookup(mm_slots_hash, mm);
1376 	mm_slot = mm_slot_entry(slot, struct khugepaged_mm_slot, slot);
1377 	if (likely(mm_slot && mm_slot->nr_pte_mapped_thp < MAX_PTE_MAPPED_THP)) {
1378 		mm_slot->pte_mapped_thp[mm_slot->nr_pte_mapped_thp++] = addr;
1379 		ret = true;
1380 	}
1381 	spin_unlock(&khugepaged_mm_lock);
1382 	return ret;
1383 }
1384 
1385 /* hpage must be locked, and mmap_lock must be held in write */
1386 static int set_huge_pmd(struct vm_area_struct *vma, unsigned long addr,
1387 			pmd_t *pmdp, struct page *hpage)
1388 {
1389 	struct vm_fault vmf = {
1390 		.vma = vma,
1391 		.address = addr,
1392 		.flags = 0,
1393 		.pmd = pmdp,
1394 	};
1395 
1396 	VM_BUG_ON(!PageTransHuge(hpage));
1397 	mmap_assert_write_locked(vma->vm_mm);
1398 
1399 	if (do_set_pmd(&vmf, hpage))
1400 		return SCAN_FAIL;
1401 
1402 	get_page(hpage);
1403 	return SCAN_SUCCEED;
1404 }
1405 
1406 /*
1407  * A note about locking:
1408  * Trying to take the page table spinlocks would be useless here because those
1409  * are only used to synchronize:
1410  *
1411  *  - modifying terminal entries (ones that point to a data page, not to another
1412  *    page table)
1413  *  - installing *new* non-terminal entries
1414  *
1415  * Instead, we need roughly the same kind of protection as free_pgtables() or
1416  * mm_take_all_locks() (but only for a single VMA):
1417  * The mmap lock together with this VMA's rmap locks covers all paths towards
1418  * the page table entries we're messing with here, except for hardware page
1419  * table walks and lockless_pages_from_mm().
1420  */
1421 static void collapse_and_free_pmd(struct mm_struct *mm, struct vm_area_struct *vma,
1422 				  unsigned long addr, pmd_t *pmdp)
1423 {
1424 	pmd_t pmd;
1425 	struct mmu_notifier_range range;
1426 
1427 	mmap_assert_write_locked(mm);
1428 	if (vma->vm_file)
1429 		lockdep_assert_held_write(&vma->vm_file->f_mapping->i_mmap_rwsem);
1430 	/*
1431 	 * All anon_vmas attached to the VMA have the same root and are
1432 	 * therefore locked by the same lock.
1433 	 */
1434 	if (vma->anon_vma)
1435 		lockdep_assert_held_write(&vma->anon_vma->root->rwsem);
1436 
1437 	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, addr,
1438 				addr + HPAGE_PMD_SIZE);
1439 	mmu_notifier_invalidate_range_start(&range);
1440 	pmd = pmdp_collapse_flush(vma, addr, pmdp);
1441 	tlb_remove_table_sync_one();
1442 	mmu_notifier_invalidate_range_end(&range);
1443 	mm_dec_nr_ptes(mm);
1444 	page_table_check_pte_clear_range(mm, addr, pmd);
1445 	pte_free(mm, pmd_pgtable(pmd));
1446 }
1447 
1448 /**
1449  * collapse_pte_mapped_thp - Try to collapse a pte-mapped THP for mm at
1450  * address haddr.
1451  *
1452  * @mm: process address space where collapse happens
1453  * @addr: THP collapse address
1454  * @install_pmd: If a huge PMD should be installed
1455  *
1456  * This function checks whether all the PTEs in the PMD are pointing to the
1457  * right THP. If so, retract the page table so the THP can refault in with
1458  * as pmd-mapped. Possibly install a huge PMD mapping the THP.
1459  */
1460 int collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr,
1461 			    bool install_pmd)
1462 {
1463 	unsigned long haddr = addr & HPAGE_PMD_MASK;
1464 	struct vm_area_struct *vma = vma_lookup(mm, haddr);
1465 	struct page *hpage;
1466 	pte_t *start_pte, *pte;
1467 	pmd_t *pmd;
1468 	spinlock_t *ptl;
1469 	int count = 0, result = SCAN_FAIL;
1470 	int i;
1471 
1472 	mmap_assert_write_locked(mm);
1473 
1474 	/* Fast check before locking page if already PMD-mapped */
1475 	result = find_pmd_or_thp_or_none(mm, haddr, &pmd);
1476 	if (result == SCAN_PMD_MAPPED)
1477 		return result;
1478 
1479 	if (!vma || !vma->vm_file ||
1480 	    !range_in_vma(vma, haddr, haddr + HPAGE_PMD_SIZE))
1481 		return SCAN_VMA_CHECK;
1482 
1483 	/*
1484 	 * If we are here, we've succeeded in replacing all the native pages
1485 	 * in the page cache with a single hugepage. If a mm were to fault-in
1486 	 * this memory (mapped by a suitably aligned VMA), we'd get the hugepage
1487 	 * and map it by a PMD, regardless of sysfs THP settings. As such, let's
1488 	 * analogously elide sysfs THP settings here.
1489 	 */
1490 	if (!hugepage_vma_check(vma, vma->vm_flags, false, false, false))
1491 		return SCAN_VMA_CHECK;
1492 
1493 	/* Keep pmd pgtable for uffd-wp; see comment in retract_page_tables() */
1494 	if (userfaultfd_wp(vma))
1495 		return SCAN_PTE_UFFD_WP;
1496 
1497 	hpage = find_lock_page(vma->vm_file->f_mapping,
1498 			       linear_page_index(vma, haddr));
1499 	if (!hpage)
1500 		return SCAN_PAGE_NULL;
1501 
1502 	if (!PageHead(hpage)) {
1503 		result = SCAN_FAIL;
1504 		goto drop_hpage;
1505 	}
1506 
1507 	if (compound_order(hpage) != HPAGE_PMD_ORDER) {
1508 		result = SCAN_PAGE_COMPOUND;
1509 		goto drop_hpage;
1510 	}
1511 
1512 	switch (result) {
1513 	case SCAN_SUCCEED:
1514 		break;
1515 	case SCAN_PMD_NONE:
1516 		/*
1517 		 * In MADV_COLLAPSE path, possible race with khugepaged where
1518 		 * all pte entries have been removed and pmd cleared.  If so,
1519 		 * skip all the pte checks and just update the pmd mapping.
1520 		 */
1521 		goto maybe_install_pmd;
1522 	default:
1523 		goto drop_hpage;
1524 	}
1525 
1526 	/* Lock the vma before taking i_mmap and page table locks */
1527 	vma_start_write(vma);
1528 
1529 	/*
1530 	 * We need to lock the mapping so that from here on, only GUP-fast and
1531 	 * hardware page walks can access the parts of the page tables that
1532 	 * we're operating on.
1533 	 * See collapse_and_free_pmd().
1534 	 */
1535 	i_mmap_lock_write(vma->vm_file->f_mapping);
1536 
1537 	/*
1538 	 * This spinlock should be unnecessary: Nobody else should be accessing
1539 	 * the page tables under spinlock protection here, only
1540 	 * lockless_pages_from_mm() and the hardware page walker can access page
1541 	 * tables while all the high-level locks are held in write mode.
1542 	 */
1543 	start_pte = pte_offset_map_lock(mm, pmd, haddr, &ptl);
1544 	result = SCAN_FAIL;
1545 
1546 	/* step 1: check all mapped PTEs are to the right huge page */
1547 	for (i = 0, addr = haddr, pte = start_pte;
1548 	     i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE, pte++) {
1549 		struct page *page;
1550 
1551 		/* empty pte, skip */
1552 		if (pte_none(*pte))
1553 			continue;
1554 
1555 		/* page swapped out, abort */
1556 		if (!pte_present(*pte)) {
1557 			result = SCAN_PTE_NON_PRESENT;
1558 			goto abort;
1559 		}
1560 
1561 		page = vm_normal_page(vma, addr, *pte);
1562 		if (WARN_ON_ONCE(page && is_zone_device_page(page)))
1563 			page = NULL;
1564 		/*
1565 		 * Note that uprobe, debugger, or MAP_PRIVATE may change the
1566 		 * page table, but the new page will not be a subpage of hpage.
1567 		 */
1568 		if (hpage + i != page)
1569 			goto abort;
1570 		count++;
1571 	}
1572 
1573 	/* step 2: adjust rmap */
1574 	for (i = 0, addr = haddr, pte = start_pte;
1575 	     i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE, pte++) {
1576 		struct page *page;
1577 
1578 		if (pte_none(*pte))
1579 			continue;
1580 		page = vm_normal_page(vma, addr, *pte);
1581 		if (WARN_ON_ONCE(page && is_zone_device_page(page)))
1582 			goto abort;
1583 		page_remove_rmap(page, vma, false);
1584 	}
1585 
1586 	pte_unmap_unlock(start_pte, ptl);
1587 
1588 	/* step 3: set proper refcount and mm_counters. */
1589 	if (count) {
1590 		page_ref_sub(hpage, count);
1591 		add_mm_counter(vma->vm_mm, mm_counter_file(hpage), -count);
1592 	}
1593 
1594 	/* step 4: remove pte entries */
1595 	/* we make no change to anon, but protect concurrent anon page lookup */
1596 	if (vma->anon_vma)
1597 		anon_vma_lock_write(vma->anon_vma);
1598 
1599 	collapse_and_free_pmd(mm, vma, haddr, pmd);
1600 
1601 	if (vma->anon_vma)
1602 		anon_vma_unlock_write(vma->anon_vma);
1603 	i_mmap_unlock_write(vma->vm_file->f_mapping);
1604 
1605 maybe_install_pmd:
1606 	/* step 5: install pmd entry */
1607 	result = install_pmd
1608 			? set_huge_pmd(vma, haddr, pmd, hpage)
1609 			: SCAN_SUCCEED;
1610 
1611 drop_hpage:
1612 	unlock_page(hpage);
1613 	put_page(hpage);
1614 	return result;
1615 
1616 abort:
1617 	pte_unmap_unlock(start_pte, ptl);
1618 	i_mmap_unlock_write(vma->vm_file->f_mapping);
1619 	goto drop_hpage;
1620 }
1621 
1622 static void khugepaged_collapse_pte_mapped_thps(struct khugepaged_mm_slot *mm_slot)
1623 {
1624 	struct mm_slot *slot = &mm_slot->slot;
1625 	struct mm_struct *mm = slot->mm;
1626 	int i;
1627 
1628 	if (likely(mm_slot->nr_pte_mapped_thp == 0))
1629 		return;
1630 
1631 	if (!mmap_write_trylock(mm))
1632 		return;
1633 
1634 	if (unlikely(hpage_collapse_test_exit(mm)))
1635 		goto out;
1636 
1637 	for (i = 0; i < mm_slot->nr_pte_mapped_thp; i++)
1638 		collapse_pte_mapped_thp(mm, mm_slot->pte_mapped_thp[i], false);
1639 
1640 out:
1641 	mm_slot->nr_pte_mapped_thp = 0;
1642 	mmap_write_unlock(mm);
1643 }
1644 
1645 static int retract_page_tables(struct address_space *mapping, pgoff_t pgoff,
1646 			       struct mm_struct *target_mm,
1647 			       unsigned long target_addr, struct page *hpage,
1648 			       struct collapse_control *cc)
1649 {
1650 	struct vm_area_struct *vma;
1651 	int target_result = SCAN_FAIL;
1652 
1653 	i_mmap_lock_write(mapping);
1654 	vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
1655 		int result = SCAN_FAIL;
1656 		struct mm_struct *mm = NULL;
1657 		unsigned long addr = 0;
1658 		pmd_t *pmd;
1659 		bool is_target = false;
1660 
1661 		/*
1662 		 * Check vma->anon_vma to exclude MAP_PRIVATE mappings that
1663 		 * got written to. These VMAs are likely not worth investing
1664 		 * mmap_write_lock(mm) as PMD-mapping is likely to be split
1665 		 * later.
1666 		 *
1667 		 * Note that vma->anon_vma check is racy: it can be set up after
1668 		 * the check but before we took mmap_lock by the fault path.
1669 		 * But page lock would prevent establishing any new ptes of the
1670 		 * page, so we are safe.
1671 		 *
1672 		 * An alternative would be drop the check, but check that page
1673 		 * table is clear before calling pmdp_collapse_flush() under
1674 		 * ptl. It has higher chance to recover THP for the VMA, but
1675 		 * has higher cost too. It would also probably require locking
1676 		 * the anon_vma.
1677 		 */
1678 		if (READ_ONCE(vma->anon_vma)) {
1679 			result = SCAN_PAGE_ANON;
1680 			goto next;
1681 		}
1682 		addr = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
1683 		if (addr & ~HPAGE_PMD_MASK ||
1684 		    vma->vm_end < addr + HPAGE_PMD_SIZE) {
1685 			result = SCAN_VMA_CHECK;
1686 			goto next;
1687 		}
1688 		mm = vma->vm_mm;
1689 		is_target = mm == target_mm && addr == target_addr;
1690 		result = find_pmd_or_thp_or_none(mm, addr, &pmd);
1691 		if (result != SCAN_SUCCEED)
1692 			goto next;
1693 		/*
1694 		 * We need exclusive mmap_lock to retract page table.
1695 		 *
1696 		 * We use trylock due to lock inversion: we need to acquire
1697 		 * mmap_lock while holding page lock. Fault path does it in
1698 		 * reverse order. Trylock is a way to avoid deadlock.
1699 		 *
1700 		 * Also, it's not MADV_COLLAPSE's job to collapse other
1701 		 * mappings - let khugepaged take care of them later.
1702 		 */
1703 		result = SCAN_PTE_MAPPED_HUGEPAGE;
1704 		if ((cc->is_khugepaged || is_target) &&
1705 		    mmap_write_trylock(mm)) {
1706 			/* trylock for the same lock inversion as above */
1707 			if (!vma_try_start_write(vma))
1708 				goto unlock_next;
1709 
1710 			/*
1711 			 * Re-check whether we have an ->anon_vma, because
1712 			 * collapse_and_free_pmd() requires that either no
1713 			 * ->anon_vma exists or the anon_vma is locked.
1714 			 * We already checked ->anon_vma above, but that check
1715 			 * is racy because ->anon_vma can be populated under the
1716 			 * mmap lock in read mode.
1717 			 */
1718 			if (vma->anon_vma) {
1719 				result = SCAN_PAGE_ANON;
1720 				goto unlock_next;
1721 			}
1722 			/*
1723 			 * When a vma is registered with uffd-wp, we can't
1724 			 * recycle the pmd pgtable because there can be pte
1725 			 * markers installed.  Skip it only, so the rest mm/vma
1726 			 * can still have the same file mapped hugely, however
1727 			 * it'll always mapped in small page size for uffd-wp
1728 			 * registered ranges.
1729 			 */
1730 			if (hpage_collapse_test_exit(mm)) {
1731 				result = SCAN_ANY_PROCESS;
1732 				goto unlock_next;
1733 			}
1734 			if (userfaultfd_wp(vma)) {
1735 				result = SCAN_PTE_UFFD_WP;
1736 				goto unlock_next;
1737 			}
1738 			collapse_and_free_pmd(mm, vma, addr, pmd);
1739 			if (!cc->is_khugepaged && is_target)
1740 				result = set_huge_pmd(vma, addr, pmd, hpage);
1741 			else
1742 				result = SCAN_SUCCEED;
1743 
1744 unlock_next:
1745 			mmap_write_unlock(mm);
1746 			goto next;
1747 		}
1748 		/*
1749 		 * Calling context will handle target mm/addr. Otherwise, let
1750 		 * khugepaged try again later.
1751 		 */
1752 		if (!is_target) {
1753 			khugepaged_add_pte_mapped_thp(mm, addr);
1754 			continue;
1755 		}
1756 next:
1757 		if (is_target)
1758 			target_result = result;
1759 	}
1760 	i_mmap_unlock_write(mapping);
1761 	return target_result;
1762 }
1763 
1764 /**
1765  * collapse_file - collapse filemap/tmpfs/shmem pages into huge one.
1766  *
1767  * @mm: process address space where collapse happens
1768  * @addr: virtual collapse start address
1769  * @file: file that collapse on
1770  * @start: collapse start address
1771  * @cc: collapse context and scratchpad
1772  *
1773  * Basic scheme is simple, details are more complex:
1774  *  - allocate and lock a new huge page;
1775  *  - scan page cache replacing old pages with the new one
1776  *    + swap/gup in pages if necessary;
1777  *    + fill in gaps;
1778  *    + keep old pages around in case rollback is required;
1779  *  - if replacing succeeds:
1780  *    + copy data over;
1781  *    + free old pages;
1782  *    + unlock huge page;
1783  *  - if replacing failed;
1784  *    + put all pages back and unfreeze them;
1785  *    + restore gaps in the page cache;
1786  *    + unlock and free huge page;
1787  */
1788 static int collapse_file(struct mm_struct *mm, unsigned long addr,
1789 			 struct file *file, pgoff_t start,
1790 			 struct collapse_control *cc)
1791 {
1792 	struct address_space *mapping = file->f_mapping;
1793 	struct page *hpage;
1794 	pgoff_t index = 0, end = start + HPAGE_PMD_NR;
1795 	LIST_HEAD(pagelist);
1796 	XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER);
1797 	int nr_none = 0, result = SCAN_SUCCEED;
1798 	bool is_shmem = shmem_file(file);
1799 	int nr = 0;
1800 
1801 	VM_BUG_ON(!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !is_shmem);
1802 	VM_BUG_ON(start & (HPAGE_PMD_NR - 1));
1803 
1804 	result = alloc_charge_hpage(&hpage, mm, cc);
1805 	if (result != SCAN_SUCCEED)
1806 		goto out;
1807 
1808 	/*
1809 	 * Ensure we have slots for all the pages in the range.  This is
1810 	 * almost certainly a no-op because most of the pages must be present
1811 	 */
1812 	do {
1813 		xas_lock_irq(&xas);
1814 		xas_create_range(&xas);
1815 		if (!xas_error(&xas))
1816 			break;
1817 		xas_unlock_irq(&xas);
1818 		if (!xas_nomem(&xas, GFP_KERNEL)) {
1819 			result = SCAN_FAIL;
1820 			goto out;
1821 		}
1822 	} while (1);
1823 
1824 	__SetPageLocked(hpage);
1825 	if (is_shmem)
1826 		__SetPageSwapBacked(hpage);
1827 	hpage->index = start;
1828 	hpage->mapping = mapping;
1829 
1830 	/*
1831 	 * At this point the hpage is locked and not up-to-date.
1832 	 * It's safe to insert it into the page cache, because nobody would
1833 	 * be able to map it or use it in another way until we unlock it.
1834 	 */
1835 
1836 	xas_set(&xas, start);
1837 	for (index = start; index < end; index++) {
1838 		struct page *page = xas_next(&xas);
1839 		struct folio *folio;
1840 
1841 		VM_BUG_ON(index != xas.xa_index);
1842 		if (is_shmem) {
1843 			if (!page) {
1844 				/*
1845 				 * Stop if extent has been truncated or
1846 				 * hole-punched, and is now completely
1847 				 * empty.
1848 				 */
1849 				if (index == start) {
1850 					if (!xas_next_entry(&xas, end - 1)) {
1851 						result = SCAN_TRUNCATED;
1852 						goto xa_locked;
1853 					}
1854 					xas_set(&xas, index);
1855 				}
1856 				if (!shmem_charge(mapping->host, 1)) {
1857 					result = SCAN_FAIL;
1858 					goto xa_locked;
1859 				}
1860 				xas_store(&xas, hpage);
1861 				if (xas_error(&xas)) {
1862 					/* revert shmem_charge performed
1863 					 * in the previous condition
1864 					 */
1865 					mapping->nrpages--;
1866 					shmem_uncharge(mapping->host, 1);
1867 					result = SCAN_STORE_FAILED;
1868 					goto xa_locked;
1869 				}
1870 				nr_none++;
1871 				continue;
1872 			}
1873 
1874 			if (xa_is_value(page) || !PageUptodate(page)) {
1875 				xas_unlock_irq(&xas);
1876 				/* swap in or instantiate fallocated page */
1877 				if (shmem_get_folio(mapping->host, index,
1878 						&folio, SGP_NOALLOC)) {
1879 					result = SCAN_FAIL;
1880 					goto xa_unlocked;
1881 				}
1882 				page = folio_file_page(folio, index);
1883 			} else if (trylock_page(page)) {
1884 				get_page(page);
1885 				xas_unlock_irq(&xas);
1886 			} else {
1887 				result = SCAN_PAGE_LOCK;
1888 				goto xa_locked;
1889 			}
1890 		} else {	/* !is_shmem */
1891 			if (!page || xa_is_value(page)) {
1892 				xas_unlock_irq(&xas);
1893 				page_cache_sync_readahead(mapping, &file->f_ra,
1894 							  file, index,
1895 							  end - index);
1896 				/* drain pagevecs to help isolate_lru_page() */
1897 				lru_add_drain();
1898 				page = find_lock_page(mapping, index);
1899 				if (unlikely(page == NULL)) {
1900 					result = SCAN_FAIL;
1901 					goto xa_unlocked;
1902 				}
1903 			} else if (PageDirty(page)) {
1904 				/*
1905 				 * khugepaged only works on read-only fd,
1906 				 * so this page is dirty because it hasn't
1907 				 * been flushed since first write. There
1908 				 * won't be new dirty pages.
1909 				 *
1910 				 * Trigger async flush here and hope the
1911 				 * writeback is done when khugepaged
1912 				 * revisits this page.
1913 				 *
1914 				 * This is a one-off situation. We are not
1915 				 * forcing writeback in loop.
1916 				 */
1917 				xas_unlock_irq(&xas);
1918 				filemap_flush(mapping);
1919 				result = SCAN_FAIL;
1920 				goto xa_unlocked;
1921 			} else if (PageWriteback(page)) {
1922 				xas_unlock_irq(&xas);
1923 				result = SCAN_FAIL;
1924 				goto xa_unlocked;
1925 			} else if (trylock_page(page)) {
1926 				get_page(page);
1927 				xas_unlock_irq(&xas);
1928 			} else {
1929 				result = SCAN_PAGE_LOCK;
1930 				goto xa_locked;
1931 			}
1932 		}
1933 
1934 		/*
1935 		 * The page must be locked, so we can drop the i_pages lock
1936 		 * without racing with truncate.
1937 		 */
1938 		VM_BUG_ON_PAGE(!PageLocked(page), page);
1939 
1940 		/* make sure the page is up to date */
1941 		if (unlikely(!PageUptodate(page))) {
1942 			result = SCAN_FAIL;
1943 			goto out_unlock;
1944 		}
1945 
1946 		/*
1947 		 * If file was truncated then extended, or hole-punched, before
1948 		 * we locked the first page, then a THP might be there already.
1949 		 * This will be discovered on the first iteration.
1950 		 */
1951 		if (PageTransCompound(page)) {
1952 			struct page *head = compound_head(page);
1953 
1954 			result = compound_order(head) == HPAGE_PMD_ORDER &&
1955 					head->index == start
1956 					/* Maybe PMD-mapped */
1957 					? SCAN_PTE_MAPPED_HUGEPAGE
1958 					: SCAN_PAGE_COMPOUND;
1959 			goto out_unlock;
1960 		}
1961 
1962 		folio = page_folio(page);
1963 
1964 		if (folio_mapping(folio) != mapping) {
1965 			result = SCAN_TRUNCATED;
1966 			goto out_unlock;
1967 		}
1968 
1969 		if (!is_shmem && (folio_test_dirty(folio) ||
1970 				  folio_test_writeback(folio))) {
1971 			/*
1972 			 * khugepaged only works on read-only fd, so this
1973 			 * page is dirty because it hasn't been flushed
1974 			 * since first write.
1975 			 */
1976 			result = SCAN_FAIL;
1977 			goto out_unlock;
1978 		}
1979 
1980 		if (!folio_isolate_lru(folio)) {
1981 			result = SCAN_DEL_PAGE_LRU;
1982 			goto out_unlock;
1983 		}
1984 
1985 		if (folio_has_private(folio) &&
1986 		    !filemap_release_folio(folio, GFP_KERNEL)) {
1987 			result = SCAN_PAGE_HAS_PRIVATE;
1988 			folio_putback_lru(folio);
1989 			goto out_unlock;
1990 		}
1991 
1992 		if (folio_mapped(folio))
1993 			try_to_unmap(folio,
1994 					TTU_IGNORE_MLOCK | TTU_BATCH_FLUSH);
1995 
1996 		xas_lock_irq(&xas);
1997 		xas_set(&xas, index);
1998 
1999 		VM_BUG_ON_PAGE(page != xas_load(&xas), page);
2000 
2001 		/*
2002 		 * The page is expected to have page_count() == 3:
2003 		 *  - we hold a pin on it;
2004 		 *  - one reference from page cache;
2005 		 *  - one from isolate_lru_page;
2006 		 */
2007 		if (!page_ref_freeze(page, 3)) {
2008 			result = SCAN_PAGE_COUNT;
2009 			xas_unlock_irq(&xas);
2010 			putback_lru_page(page);
2011 			goto out_unlock;
2012 		}
2013 
2014 		/*
2015 		 * Add the page to the list to be able to undo the collapse if
2016 		 * something go wrong.
2017 		 */
2018 		list_add_tail(&page->lru, &pagelist);
2019 
2020 		/* Finally, replace with the new page. */
2021 		xas_store(&xas, hpage);
2022 		/* We can't get an ENOMEM here (because the allocation happened before)
2023 		 * but let's check for errors (XArray implementation can be
2024 		 * changed in the future)
2025 		 */
2026 		WARN_ON_ONCE(xas_error(&xas));
2027 		continue;
2028 out_unlock:
2029 		unlock_page(page);
2030 		put_page(page);
2031 		goto xa_unlocked;
2032 	}
2033 	nr = thp_nr_pages(hpage);
2034 
2035 	if (is_shmem)
2036 		__mod_lruvec_page_state(hpage, NR_SHMEM_THPS, nr);
2037 	else {
2038 		__mod_lruvec_page_state(hpage, NR_FILE_THPS, nr);
2039 		filemap_nr_thps_inc(mapping);
2040 		/*
2041 		 * Paired with smp_mb() in do_dentry_open() to ensure
2042 		 * i_writecount is up to date and the update to nr_thps is
2043 		 * visible. Ensures the page cache will be truncated if the
2044 		 * file is opened writable.
2045 		 */
2046 		smp_mb();
2047 		if (inode_is_open_for_write(mapping->host)) {
2048 			result = SCAN_FAIL;
2049 			__mod_lruvec_page_state(hpage, NR_FILE_THPS, -nr);
2050 			filemap_nr_thps_dec(mapping);
2051 			goto xa_locked;
2052 		}
2053 	}
2054 
2055 	if (nr_none) {
2056 		__mod_lruvec_page_state(hpage, NR_FILE_PAGES, nr_none);
2057 		/* nr_none is always 0 for non-shmem. */
2058 		__mod_lruvec_page_state(hpage, NR_SHMEM, nr_none);
2059 	}
2060 
2061 	/* Join all the small entries into a single multi-index entry */
2062 	xas_set_order(&xas, start, HPAGE_PMD_ORDER);
2063 	xas_store(&xas, hpage);
2064 	/* Here we can't get an ENOMEM (because entries were
2065 	 * previously allocated) But let's check for errors
2066 	 * (XArray implementation can be changed in the future)
2067 	 */
2068 	WARN_ON_ONCE(xas_error(&xas));
2069 xa_locked:
2070 	xas_unlock_irq(&xas);
2071 xa_unlocked:
2072 
2073 	/*
2074 	 * If collapse is successful, flush must be done now before copying.
2075 	 * If collapse is unsuccessful, does flush actually need to be done?
2076 	 * Do it anyway, to clear the state.
2077 	 */
2078 	try_to_unmap_flush();
2079 
2080 	if (result == SCAN_SUCCEED) {
2081 		struct page *page, *tmp;
2082 		struct folio *folio;
2083 
2084 		/*
2085 		 * Replacing old pages with new one has succeeded, now we
2086 		 * need to copy the content and free the old pages.
2087 		 */
2088 		index = start;
2089 		list_for_each_entry_safe(page, tmp, &pagelist, lru) {
2090 			while (index < page->index) {
2091 				clear_highpage(hpage + (index % HPAGE_PMD_NR));
2092 				index++;
2093 			}
2094 			copy_highpage(hpage + (page->index % HPAGE_PMD_NR),
2095 				      page);
2096 			list_del(&page->lru);
2097 			page->mapping = NULL;
2098 			page_ref_unfreeze(page, 1);
2099 			ClearPageActive(page);
2100 			ClearPageUnevictable(page);
2101 			unlock_page(page);
2102 			put_page(page);
2103 			index++;
2104 		}
2105 		while (index < end) {
2106 			clear_highpage(hpage + (index % HPAGE_PMD_NR));
2107 			index++;
2108 		}
2109 
2110 		folio = page_folio(hpage);
2111 		folio_mark_uptodate(folio);
2112 		folio_ref_add(folio, HPAGE_PMD_NR - 1);
2113 
2114 		if (is_shmem)
2115 			folio_mark_dirty(folio);
2116 		folio_add_lru(folio);
2117 
2118 		/*
2119 		 * Remove pte page tables, so we can re-fault the page as huge.
2120 		 */
2121 		result = retract_page_tables(mapping, start, mm, addr, hpage,
2122 					     cc);
2123 		unlock_page(hpage);
2124 		hpage = NULL;
2125 	} else {
2126 		struct page *page;
2127 
2128 		/* Something went wrong: roll back page cache changes */
2129 		xas_lock_irq(&xas);
2130 		if (nr_none) {
2131 			mapping->nrpages -= nr_none;
2132 			shmem_uncharge(mapping->host, nr_none);
2133 		}
2134 
2135 		xas_set(&xas, start);
2136 		xas_for_each(&xas, page, end - 1) {
2137 			page = list_first_entry_or_null(&pagelist,
2138 					struct page, lru);
2139 			if (!page || xas.xa_index < page->index) {
2140 				if (!nr_none)
2141 					break;
2142 				nr_none--;
2143 				/* Put holes back where they were */
2144 				xas_store(&xas, NULL);
2145 				continue;
2146 			}
2147 
2148 			VM_BUG_ON_PAGE(page->index != xas.xa_index, page);
2149 
2150 			/* Unfreeze the page. */
2151 			list_del(&page->lru);
2152 			page_ref_unfreeze(page, 2);
2153 			xas_store(&xas, page);
2154 			xas_pause(&xas);
2155 			xas_unlock_irq(&xas);
2156 			unlock_page(page);
2157 			putback_lru_page(page);
2158 			xas_lock_irq(&xas);
2159 		}
2160 		VM_BUG_ON(nr_none);
2161 		xas_unlock_irq(&xas);
2162 
2163 		hpage->mapping = NULL;
2164 	}
2165 
2166 	if (hpage)
2167 		unlock_page(hpage);
2168 out:
2169 	VM_BUG_ON(!list_empty(&pagelist));
2170 	if (hpage)
2171 		put_page(hpage);
2172 
2173 	trace_mm_khugepaged_collapse_file(mm, hpage, index, is_shmem, addr, file, nr, result);
2174 	return result;
2175 }
2176 
2177 static int hpage_collapse_scan_file(struct mm_struct *mm, unsigned long addr,
2178 				    struct file *file, pgoff_t start,
2179 				    struct collapse_control *cc)
2180 {
2181 	struct page *page = NULL;
2182 	struct address_space *mapping = file->f_mapping;
2183 	XA_STATE(xas, &mapping->i_pages, start);
2184 	int present, swap;
2185 	int node = NUMA_NO_NODE;
2186 	int result = SCAN_SUCCEED;
2187 
2188 	present = 0;
2189 	swap = 0;
2190 	memset(cc->node_load, 0, sizeof(cc->node_load));
2191 	nodes_clear(cc->alloc_nmask);
2192 	rcu_read_lock();
2193 	xas_for_each(&xas, page, start + HPAGE_PMD_NR - 1) {
2194 		if (xas_retry(&xas, page))
2195 			continue;
2196 
2197 		if (xa_is_value(page)) {
2198 			++swap;
2199 			if (cc->is_khugepaged &&
2200 			    swap > khugepaged_max_ptes_swap) {
2201 				result = SCAN_EXCEED_SWAP_PTE;
2202 				count_vm_event(THP_SCAN_EXCEED_SWAP_PTE);
2203 				break;
2204 			}
2205 			continue;
2206 		}
2207 
2208 		/*
2209 		 * TODO: khugepaged should compact smaller compound pages
2210 		 * into a PMD sized page
2211 		 */
2212 		if (PageTransCompound(page)) {
2213 			struct page *head = compound_head(page);
2214 
2215 			result = compound_order(head) == HPAGE_PMD_ORDER &&
2216 					head->index == start
2217 					/* Maybe PMD-mapped */
2218 					? SCAN_PTE_MAPPED_HUGEPAGE
2219 					: SCAN_PAGE_COMPOUND;
2220 			/*
2221 			 * For SCAN_PTE_MAPPED_HUGEPAGE, further processing
2222 			 * by the caller won't touch the page cache, and so
2223 			 * it's safe to skip LRU and refcount checks before
2224 			 * returning.
2225 			 */
2226 			break;
2227 		}
2228 
2229 		node = page_to_nid(page);
2230 		if (hpage_collapse_scan_abort(node, cc)) {
2231 			result = SCAN_SCAN_ABORT;
2232 			break;
2233 		}
2234 		cc->node_load[node]++;
2235 
2236 		if (!PageLRU(page)) {
2237 			result = SCAN_PAGE_LRU;
2238 			break;
2239 		}
2240 
2241 		if (page_count(page) !=
2242 		    1 + page_mapcount(page) + page_has_private(page)) {
2243 			result = SCAN_PAGE_COUNT;
2244 			break;
2245 		}
2246 
2247 		/*
2248 		 * We probably should check if the page is referenced here, but
2249 		 * nobody would transfer pte_young() to PageReferenced() for us.
2250 		 * And rmap walk here is just too costly...
2251 		 */
2252 
2253 		present++;
2254 
2255 		if (need_resched()) {
2256 			xas_pause(&xas);
2257 			cond_resched_rcu();
2258 		}
2259 	}
2260 	rcu_read_unlock();
2261 
2262 	if (result == SCAN_SUCCEED) {
2263 		if (cc->is_khugepaged &&
2264 		    present < HPAGE_PMD_NR - khugepaged_max_ptes_none) {
2265 			result = SCAN_EXCEED_NONE_PTE;
2266 			count_vm_event(THP_SCAN_EXCEED_NONE_PTE);
2267 		} else {
2268 			result = collapse_file(mm, addr, file, start, cc);
2269 		}
2270 	}
2271 
2272 	trace_mm_khugepaged_scan_file(mm, page, file, present, swap, result);
2273 	return result;
2274 }
2275 #else
2276 static int hpage_collapse_scan_file(struct mm_struct *mm, unsigned long addr,
2277 				    struct file *file, pgoff_t start,
2278 				    struct collapse_control *cc)
2279 {
2280 	BUILD_BUG();
2281 }
2282 
2283 static void khugepaged_collapse_pte_mapped_thps(struct khugepaged_mm_slot *mm_slot)
2284 {
2285 }
2286 
2287 static bool khugepaged_add_pte_mapped_thp(struct mm_struct *mm,
2288 					  unsigned long addr)
2289 {
2290 	return false;
2291 }
2292 #endif
2293 
2294 static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result,
2295 					    struct collapse_control *cc)
2296 	__releases(&khugepaged_mm_lock)
2297 	__acquires(&khugepaged_mm_lock)
2298 {
2299 	struct vma_iterator vmi;
2300 	struct khugepaged_mm_slot *mm_slot;
2301 	struct mm_slot *slot;
2302 	struct mm_struct *mm;
2303 	struct vm_area_struct *vma;
2304 	int progress = 0;
2305 
2306 	VM_BUG_ON(!pages);
2307 	lockdep_assert_held(&khugepaged_mm_lock);
2308 	*result = SCAN_FAIL;
2309 
2310 	if (khugepaged_scan.mm_slot) {
2311 		mm_slot = khugepaged_scan.mm_slot;
2312 		slot = &mm_slot->slot;
2313 	} else {
2314 		slot = list_entry(khugepaged_scan.mm_head.next,
2315 				     struct mm_slot, mm_node);
2316 		mm_slot = mm_slot_entry(slot, struct khugepaged_mm_slot, slot);
2317 		khugepaged_scan.address = 0;
2318 		khugepaged_scan.mm_slot = mm_slot;
2319 	}
2320 	spin_unlock(&khugepaged_mm_lock);
2321 	khugepaged_collapse_pte_mapped_thps(mm_slot);
2322 
2323 	mm = slot->mm;
2324 	/*
2325 	 * Don't wait for semaphore (to avoid long wait times).  Just move to
2326 	 * the next mm on the list.
2327 	 */
2328 	vma = NULL;
2329 	if (unlikely(!mmap_read_trylock(mm)))
2330 		goto breakouterloop_mmap_lock;
2331 
2332 	progress++;
2333 	if (unlikely(hpage_collapse_test_exit(mm)))
2334 		goto breakouterloop;
2335 
2336 	vma_iter_init(&vmi, mm, khugepaged_scan.address);
2337 	for_each_vma(vmi, vma) {
2338 		unsigned long hstart, hend;
2339 
2340 		cond_resched();
2341 		if (unlikely(hpage_collapse_test_exit(mm))) {
2342 			progress++;
2343 			break;
2344 		}
2345 		if (!hugepage_vma_check(vma, vma->vm_flags, false, false, true)) {
2346 skip:
2347 			progress++;
2348 			continue;
2349 		}
2350 		hstart = round_up(vma->vm_start, HPAGE_PMD_SIZE);
2351 		hend = round_down(vma->vm_end, HPAGE_PMD_SIZE);
2352 		if (khugepaged_scan.address > hend)
2353 			goto skip;
2354 		if (khugepaged_scan.address < hstart)
2355 			khugepaged_scan.address = hstart;
2356 		VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK);
2357 
2358 		while (khugepaged_scan.address < hend) {
2359 			bool mmap_locked = true;
2360 
2361 			cond_resched();
2362 			if (unlikely(hpage_collapse_test_exit(mm)))
2363 				goto breakouterloop;
2364 
2365 			VM_BUG_ON(khugepaged_scan.address < hstart ||
2366 				  khugepaged_scan.address + HPAGE_PMD_SIZE >
2367 				  hend);
2368 			if (IS_ENABLED(CONFIG_SHMEM) && vma->vm_file) {
2369 				struct file *file = get_file(vma->vm_file);
2370 				pgoff_t pgoff = linear_page_index(vma,
2371 						khugepaged_scan.address);
2372 
2373 				mmap_read_unlock(mm);
2374 				*result = hpage_collapse_scan_file(mm,
2375 								   khugepaged_scan.address,
2376 								   file, pgoff, cc);
2377 				mmap_locked = false;
2378 				fput(file);
2379 			} else {
2380 				*result = hpage_collapse_scan_pmd(mm, vma,
2381 								  khugepaged_scan.address,
2382 								  &mmap_locked,
2383 								  cc);
2384 			}
2385 			switch (*result) {
2386 			case SCAN_PTE_MAPPED_HUGEPAGE: {
2387 				pmd_t *pmd;
2388 
2389 				*result = find_pmd_or_thp_or_none(mm,
2390 								  khugepaged_scan.address,
2391 								  &pmd);
2392 				if (*result != SCAN_SUCCEED)
2393 					break;
2394 				if (!khugepaged_add_pte_mapped_thp(mm,
2395 								   khugepaged_scan.address))
2396 					break;
2397 			} fallthrough;
2398 			case SCAN_SUCCEED:
2399 				++khugepaged_pages_collapsed;
2400 				break;
2401 			default:
2402 				break;
2403 			}
2404 
2405 			/* move to next address */
2406 			khugepaged_scan.address += HPAGE_PMD_SIZE;
2407 			progress += HPAGE_PMD_NR;
2408 			if (!mmap_locked)
2409 				/*
2410 				 * We released mmap_lock so break loop.  Note
2411 				 * that we drop mmap_lock before all hugepage
2412 				 * allocations, so if allocation fails, we are
2413 				 * guaranteed to break here and report the
2414 				 * correct result back to caller.
2415 				 */
2416 				goto breakouterloop_mmap_lock;
2417 			if (progress >= pages)
2418 				goto breakouterloop;
2419 		}
2420 	}
2421 breakouterloop:
2422 	mmap_read_unlock(mm); /* exit_mmap will destroy ptes after this */
2423 breakouterloop_mmap_lock:
2424 
2425 	spin_lock(&khugepaged_mm_lock);
2426 	VM_BUG_ON(khugepaged_scan.mm_slot != mm_slot);
2427 	/*
2428 	 * Release the current mm_slot if this mm is about to die, or
2429 	 * if we scanned all vmas of this mm.
2430 	 */
2431 	if (hpage_collapse_test_exit(mm) || !vma) {
2432 		/*
2433 		 * Make sure that if mm_users is reaching zero while
2434 		 * khugepaged runs here, khugepaged_exit will find
2435 		 * mm_slot not pointing to the exiting mm.
2436 		 */
2437 		if (slot->mm_node.next != &khugepaged_scan.mm_head) {
2438 			slot = list_entry(slot->mm_node.next,
2439 					  struct mm_slot, mm_node);
2440 			khugepaged_scan.mm_slot =
2441 				mm_slot_entry(slot, struct khugepaged_mm_slot, slot);
2442 			khugepaged_scan.address = 0;
2443 		} else {
2444 			khugepaged_scan.mm_slot = NULL;
2445 			khugepaged_full_scans++;
2446 		}
2447 
2448 		collect_mm_slot(mm_slot);
2449 	}
2450 
2451 	return progress;
2452 }
2453 
2454 static int khugepaged_has_work(void)
2455 {
2456 	return !list_empty(&khugepaged_scan.mm_head) &&
2457 		hugepage_flags_enabled();
2458 }
2459 
2460 static int khugepaged_wait_event(void)
2461 {
2462 	return !list_empty(&khugepaged_scan.mm_head) ||
2463 		kthread_should_stop();
2464 }
2465 
2466 static void khugepaged_do_scan(struct collapse_control *cc)
2467 {
2468 	unsigned int progress = 0, pass_through_head = 0;
2469 	unsigned int pages = READ_ONCE(khugepaged_pages_to_scan);
2470 	bool wait = true;
2471 	int result = SCAN_SUCCEED;
2472 
2473 	lru_add_drain_all();
2474 
2475 	while (true) {
2476 		cond_resched();
2477 
2478 		if (unlikely(kthread_should_stop() || try_to_freeze()))
2479 			break;
2480 
2481 		spin_lock(&khugepaged_mm_lock);
2482 		if (!khugepaged_scan.mm_slot)
2483 			pass_through_head++;
2484 		if (khugepaged_has_work() &&
2485 		    pass_through_head < 2)
2486 			progress += khugepaged_scan_mm_slot(pages - progress,
2487 							    &result, cc);
2488 		else
2489 			progress = pages;
2490 		spin_unlock(&khugepaged_mm_lock);
2491 
2492 		if (progress >= pages)
2493 			break;
2494 
2495 		if (result == SCAN_ALLOC_HUGE_PAGE_FAIL) {
2496 			/*
2497 			 * If fail to allocate the first time, try to sleep for
2498 			 * a while.  When hit again, cancel the scan.
2499 			 */
2500 			if (!wait)
2501 				break;
2502 			wait = false;
2503 			khugepaged_alloc_sleep();
2504 		}
2505 	}
2506 }
2507 
2508 static bool khugepaged_should_wakeup(void)
2509 {
2510 	return kthread_should_stop() ||
2511 	       time_after_eq(jiffies, khugepaged_sleep_expire);
2512 }
2513 
2514 static void khugepaged_wait_work(void)
2515 {
2516 	if (khugepaged_has_work()) {
2517 		const unsigned long scan_sleep_jiffies =
2518 			msecs_to_jiffies(khugepaged_scan_sleep_millisecs);
2519 
2520 		if (!scan_sleep_jiffies)
2521 			return;
2522 
2523 		khugepaged_sleep_expire = jiffies + scan_sleep_jiffies;
2524 		wait_event_freezable_timeout(khugepaged_wait,
2525 					     khugepaged_should_wakeup(),
2526 					     scan_sleep_jiffies);
2527 		return;
2528 	}
2529 
2530 	if (hugepage_flags_enabled())
2531 		wait_event_freezable(khugepaged_wait, khugepaged_wait_event());
2532 }
2533 
2534 static int khugepaged(void *none)
2535 {
2536 	struct khugepaged_mm_slot *mm_slot;
2537 
2538 	set_freezable();
2539 	set_user_nice(current, MAX_NICE);
2540 
2541 	while (!kthread_should_stop()) {
2542 		khugepaged_do_scan(&khugepaged_collapse_control);
2543 		khugepaged_wait_work();
2544 	}
2545 
2546 	spin_lock(&khugepaged_mm_lock);
2547 	mm_slot = khugepaged_scan.mm_slot;
2548 	khugepaged_scan.mm_slot = NULL;
2549 	if (mm_slot)
2550 		collect_mm_slot(mm_slot);
2551 	spin_unlock(&khugepaged_mm_lock);
2552 	return 0;
2553 }
2554 
2555 static void set_recommended_min_free_kbytes(void)
2556 {
2557 	struct zone *zone;
2558 	int nr_zones = 0;
2559 	unsigned long recommended_min;
2560 
2561 	if (!hugepage_flags_enabled()) {
2562 		calculate_min_free_kbytes();
2563 		goto update_wmarks;
2564 	}
2565 
2566 	for_each_populated_zone(zone) {
2567 		/*
2568 		 * We don't need to worry about fragmentation of
2569 		 * ZONE_MOVABLE since it only has movable pages.
2570 		 */
2571 		if (zone_idx(zone) > gfp_zone(GFP_USER))
2572 			continue;
2573 
2574 		nr_zones++;
2575 	}
2576 
2577 	/* Ensure 2 pageblocks are free to assist fragmentation avoidance */
2578 	recommended_min = pageblock_nr_pages * nr_zones * 2;
2579 
2580 	/*
2581 	 * Make sure that on average at least two pageblocks are almost free
2582 	 * of another type, one for a migratetype to fall back to and a
2583 	 * second to avoid subsequent fallbacks of other types There are 3
2584 	 * MIGRATE_TYPES we care about.
2585 	 */
2586 	recommended_min += pageblock_nr_pages * nr_zones *
2587 			   MIGRATE_PCPTYPES * MIGRATE_PCPTYPES;
2588 
2589 	/* don't ever allow to reserve more than 5% of the lowmem */
2590 	recommended_min = min(recommended_min,
2591 			      (unsigned long) nr_free_buffer_pages() / 20);
2592 	recommended_min <<= (PAGE_SHIFT-10);
2593 
2594 	if (recommended_min > min_free_kbytes) {
2595 		if (user_min_free_kbytes >= 0)
2596 			pr_info("raising min_free_kbytes from %d to %lu to help transparent hugepage allocations\n",
2597 				min_free_kbytes, recommended_min);
2598 
2599 		min_free_kbytes = recommended_min;
2600 	}
2601 
2602 update_wmarks:
2603 	setup_per_zone_wmarks();
2604 }
2605 
2606 int start_stop_khugepaged(void)
2607 {
2608 	int err = 0;
2609 
2610 	mutex_lock(&khugepaged_mutex);
2611 	if (hugepage_flags_enabled()) {
2612 		if (!khugepaged_thread)
2613 			khugepaged_thread = kthread_run(khugepaged, NULL,
2614 							"khugepaged");
2615 		if (IS_ERR(khugepaged_thread)) {
2616 			pr_err("khugepaged: kthread_run(khugepaged) failed\n");
2617 			err = PTR_ERR(khugepaged_thread);
2618 			khugepaged_thread = NULL;
2619 			goto fail;
2620 		}
2621 
2622 		if (!list_empty(&khugepaged_scan.mm_head))
2623 			wake_up_interruptible(&khugepaged_wait);
2624 	} else if (khugepaged_thread) {
2625 		kthread_stop(khugepaged_thread);
2626 		khugepaged_thread = NULL;
2627 	}
2628 	set_recommended_min_free_kbytes();
2629 fail:
2630 	mutex_unlock(&khugepaged_mutex);
2631 	return err;
2632 }
2633 
2634 void khugepaged_min_free_kbytes_update(void)
2635 {
2636 	mutex_lock(&khugepaged_mutex);
2637 	if (hugepage_flags_enabled() && khugepaged_thread)
2638 		set_recommended_min_free_kbytes();
2639 	mutex_unlock(&khugepaged_mutex);
2640 }
2641 
2642 bool current_is_khugepaged(void)
2643 {
2644 	return kthread_func(current) == khugepaged;
2645 }
2646 
2647 static int madvise_collapse_errno(enum scan_result r)
2648 {
2649 	/*
2650 	 * MADV_COLLAPSE breaks from existing madvise(2) conventions to provide
2651 	 * actionable feedback to caller, so they may take an appropriate
2652 	 * fallback measure depending on the nature of the failure.
2653 	 */
2654 	switch (r) {
2655 	case SCAN_ALLOC_HUGE_PAGE_FAIL:
2656 		return -ENOMEM;
2657 	case SCAN_CGROUP_CHARGE_FAIL:
2658 		return -EBUSY;
2659 	/* Resource temporary unavailable - trying again might succeed */
2660 	case SCAN_PAGE_COUNT:
2661 	case SCAN_PAGE_LOCK:
2662 	case SCAN_PAGE_LRU:
2663 	case SCAN_DEL_PAGE_LRU:
2664 		return -EAGAIN;
2665 	/*
2666 	 * Other: Trying again likely not to succeed / error intrinsic to
2667 	 * specified memory range. khugepaged likely won't be able to collapse
2668 	 * either.
2669 	 */
2670 	default:
2671 		return -EINVAL;
2672 	}
2673 }
2674 
2675 int madvise_collapse(struct vm_area_struct *vma, struct vm_area_struct **prev,
2676 		     unsigned long start, unsigned long end)
2677 {
2678 	struct collapse_control *cc;
2679 	struct mm_struct *mm = vma->vm_mm;
2680 	unsigned long hstart, hend, addr;
2681 	int thps = 0, last_fail = SCAN_FAIL;
2682 	bool mmap_locked = true;
2683 
2684 	BUG_ON(vma->vm_start > start);
2685 	BUG_ON(vma->vm_end < end);
2686 
2687 	*prev = vma;
2688 
2689 	if (!hugepage_vma_check(vma, vma->vm_flags, false, false, false))
2690 		return -EINVAL;
2691 
2692 	cc = kmalloc(sizeof(*cc), GFP_KERNEL);
2693 	if (!cc)
2694 		return -ENOMEM;
2695 	cc->is_khugepaged = false;
2696 
2697 	mmgrab(mm);
2698 	lru_add_drain_all();
2699 
2700 	hstart = (start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
2701 	hend = end & HPAGE_PMD_MASK;
2702 
2703 	for (addr = hstart; addr < hend; addr += HPAGE_PMD_SIZE) {
2704 		int result = SCAN_FAIL;
2705 
2706 		if (!mmap_locked) {
2707 			cond_resched();
2708 			mmap_read_lock(mm);
2709 			mmap_locked = true;
2710 			result = hugepage_vma_revalidate(mm, addr, false, &vma,
2711 							 cc);
2712 			if (result  != SCAN_SUCCEED) {
2713 				last_fail = result;
2714 				goto out_nolock;
2715 			}
2716 
2717 			hend = min(hend, vma->vm_end & HPAGE_PMD_MASK);
2718 		}
2719 		mmap_assert_locked(mm);
2720 		memset(cc->node_load, 0, sizeof(cc->node_load));
2721 		nodes_clear(cc->alloc_nmask);
2722 		if (IS_ENABLED(CONFIG_SHMEM) && vma->vm_file) {
2723 			struct file *file = get_file(vma->vm_file);
2724 			pgoff_t pgoff = linear_page_index(vma, addr);
2725 
2726 			mmap_read_unlock(mm);
2727 			mmap_locked = false;
2728 			result = hpage_collapse_scan_file(mm, addr, file, pgoff,
2729 							  cc);
2730 			fput(file);
2731 		} else {
2732 			result = hpage_collapse_scan_pmd(mm, vma, addr,
2733 							 &mmap_locked, cc);
2734 		}
2735 		if (!mmap_locked)
2736 			*prev = NULL;  /* Tell caller we dropped mmap_lock */
2737 
2738 handle_result:
2739 		switch (result) {
2740 		case SCAN_SUCCEED:
2741 		case SCAN_PMD_MAPPED:
2742 			++thps;
2743 			break;
2744 		case SCAN_PTE_MAPPED_HUGEPAGE:
2745 			BUG_ON(mmap_locked);
2746 			BUG_ON(*prev);
2747 			mmap_write_lock(mm);
2748 			result = collapse_pte_mapped_thp(mm, addr, true);
2749 			mmap_write_unlock(mm);
2750 			goto handle_result;
2751 		/* Whitelisted set of results where continuing OK */
2752 		case SCAN_PMD_NULL:
2753 		case SCAN_PTE_NON_PRESENT:
2754 		case SCAN_PTE_UFFD_WP:
2755 		case SCAN_PAGE_RO:
2756 		case SCAN_LACK_REFERENCED_PAGE:
2757 		case SCAN_PAGE_NULL:
2758 		case SCAN_PAGE_COUNT:
2759 		case SCAN_PAGE_LOCK:
2760 		case SCAN_PAGE_COMPOUND:
2761 		case SCAN_PAGE_LRU:
2762 		case SCAN_DEL_PAGE_LRU:
2763 			last_fail = result;
2764 			break;
2765 		default:
2766 			last_fail = result;
2767 			/* Other error, exit */
2768 			goto out_maybelock;
2769 		}
2770 	}
2771 
2772 out_maybelock:
2773 	/* Caller expects us to hold mmap_lock on return */
2774 	if (!mmap_locked)
2775 		mmap_read_lock(mm);
2776 out_nolock:
2777 	mmap_assert_locked(mm);
2778 	mmdrop(mm);
2779 	kfree(cc);
2780 
2781 	return thps == ((hend - hstart) >> HPAGE_PMD_SHIFT) ? 0
2782 			: madvise_collapse_errno(last_fail);
2783 }
2784