xref: /openbmc/linux/mm/migrate.c (revision 4d906098)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Memory Migration functionality - linux/mm/migrate.c
4  *
5  * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter
6  *
7  * Page migration was first developed in the context of the memory hotplug
8  * project. The main authors of the migration code are:
9  *
10  * IWAMOTO Toshihiro <iwamoto@valinux.co.jp>
11  * Hirokazu Takahashi <taka@valinux.co.jp>
12  * Dave Hansen <haveblue@us.ibm.com>
13  * Christoph Lameter
14  */
15 
16 #include <linux/migrate.h>
17 #include <linux/export.h>
18 #include <linux/swap.h>
19 #include <linux/swapops.h>
20 #include <linux/pagemap.h>
21 #include <linux/buffer_head.h>
22 #include <linux/mm_inline.h>
23 #include <linux/nsproxy.h>
24 #include <linux/pagevec.h>
25 #include <linux/ksm.h>
26 #include <linux/rmap.h>
27 #include <linux/topology.h>
28 #include <linux/cpu.h>
29 #include <linux/cpuset.h>
30 #include <linux/writeback.h>
31 #include <linux/mempolicy.h>
32 #include <linux/vmalloc.h>
33 #include <linux/security.h>
34 #include <linux/backing-dev.h>
35 #include <linux/compaction.h>
36 #include <linux/syscalls.h>
37 #include <linux/compat.h>
38 #include <linux/hugetlb.h>
39 #include <linux/hugetlb_cgroup.h>
40 #include <linux/gfp.h>
41 #include <linux/pfn_t.h>
42 #include <linux/memremap.h>
43 #include <linux/userfaultfd_k.h>
44 #include <linux/balloon_compaction.h>
45 #include <linux/page_idle.h>
46 #include <linux/page_owner.h>
47 #include <linux/sched/mm.h>
48 #include <linux/ptrace.h>
49 #include <linux/oom.h>
50 #include <linux/memory.h>
51 #include <linux/random.h>
52 #include <linux/sched/sysctl.h>
53 #include <linux/memory-tiers.h>
54 
55 #include <asm/tlbflush.h>
56 
57 #include <trace/events/migrate.h>
58 
59 #include "internal.h"
60 
61 bool isolate_movable_page(struct page *page, isolate_mode_t mode)
62 {
63 	struct folio *folio = folio_get_nontail_page(page);
64 	const struct movable_operations *mops;
65 
66 	/*
67 	 * Avoid burning cycles with pages that are yet under __free_pages(),
68 	 * or just got freed under us.
69 	 *
70 	 * In case we 'win' a race for a movable page being freed under us and
71 	 * raise its refcount preventing __free_pages() from doing its job
72 	 * the put_page() at the end of this block will take care of
73 	 * release this page, thus avoiding a nasty leakage.
74 	 */
75 	if (!folio)
76 		goto out;
77 
78 	if (unlikely(folio_test_slab(folio)))
79 		goto out_putfolio;
80 	/* Pairs with smp_wmb() in slab freeing, e.g. SLUB's __free_slab() */
81 	smp_rmb();
82 	/*
83 	 * Check movable flag before taking the page lock because
84 	 * we use non-atomic bitops on newly allocated page flags so
85 	 * unconditionally grabbing the lock ruins page's owner side.
86 	 */
87 	if (unlikely(!__folio_test_movable(folio)))
88 		goto out_putfolio;
89 	/* Pairs with smp_wmb() in slab allocation, e.g. SLUB's alloc_slab_page() */
90 	smp_rmb();
91 	if (unlikely(folio_test_slab(folio)))
92 		goto out_putfolio;
93 
94 	/*
95 	 * As movable pages are not isolated from LRU lists, concurrent
96 	 * compaction threads can race against page migration functions
97 	 * as well as race against the releasing a page.
98 	 *
99 	 * In order to avoid having an already isolated movable page
100 	 * being (wrongly) re-isolated while it is under migration,
101 	 * or to avoid attempting to isolate pages being released,
102 	 * lets be sure we have the page lock
103 	 * before proceeding with the movable page isolation steps.
104 	 */
105 	if (unlikely(!folio_trylock(folio)))
106 		goto out_putfolio;
107 
108 	if (!folio_test_movable(folio) || folio_test_isolated(folio))
109 		goto out_no_isolated;
110 
111 	mops = folio_movable_ops(folio);
112 	VM_BUG_ON_FOLIO(!mops, folio);
113 
114 	if (!mops->isolate_page(&folio->page, mode))
115 		goto out_no_isolated;
116 
117 	/* Driver shouldn't use PG_isolated bit of page->flags */
118 	WARN_ON_ONCE(folio_test_isolated(folio));
119 	folio_set_isolated(folio);
120 	folio_unlock(folio);
121 
122 	return true;
123 
124 out_no_isolated:
125 	folio_unlock(folio);
126 out_putfolio:
127 	folio_put(folio);
128 out:
129 	return false;
130 }
131 
132 static void putback_movable_folio(struct folio *folio)
133 {
134 	const struct movable_operations *mops = folio_movable_ops(folio);
135 
136 	mops->putback_page(&folio->page);
137 	folio_clear_isolated(folio);
138 }
139 
140 /*
141  * Put previously isolated pages back onto the appropriate lists
142  * from where they were once taken off for compaction/migration.
143  *
144  * This function shall be used whenever the isolated pageset has been
145  * built from lru, balloon, hugetlbfs page. See isolate_migratepages_range()
146  * and isolate_hugetlb().
147  */
148 void putback_movable_pages(struct list_head *l)
149 {
150 	struct folio *folio;
151 	struct folio *folio2;
152 
153 	list_for_each_entry_safe(folio, folio2, l, lru) {
154 		if (unlikely(folio_test_hugetlb(folio))) {
155 			folio_putback_active_hugetlb(folio);
156 			continue;
157 		}
158 		list_del(&folio->lru);
159 		/*
160 		 * We isolated non-lru movable folio so here we can use
161 		 * __PageMovable because LRU folio's mapping cannot have
162 		 * PAGE_MAPPING_MOVABLE.
163 		 */
164 		if (unlikely(__folio_test_movable(folio))) {
165 			VM_BUG_ON_FOLIO(!folio_test_isolated(folio), folio);
166 			folio_lock(folio);
167 			if (folio_test_movable(folio))
168 				putback_movable_folio(folio);
169 			else
170 				folio_clear_isolated(folio);
171 			folio_unlock(folio);
172 			folio_put(folio);
173 		} else {
174 			node_stat_mod_folio(folio, NR_ISOLATED_ANON +
175 					folio_is_file_lru(folio), -folio_nr_pages(folio));
176 			folio_putback_lru(folio);
177 		}
178 	}
179 }
180 
181 /*
182  * Restore a potential migration pte to a working pte entry
183  */
184 static bool remove_migration_pte(struct folio *folio,
185 		struct vm_area_struct *vma, unsigned long addr, void *old)
186 {
187 	DEFINE_FOLIO_VMA_WALK(pvmw, old, vma, addr, PVMW_SYNC | PVMW_MIGRATION);
188 
189 	while (page_vma_mapped_walk(&pvmw)) {
190 		rmap_t rmap_flags = RMAP_NONE;
191 		pte_t pte;
192 		swp_entry_t entry;
193 		struct page *new;
194 		unsigned long idx = 0;
195 
196 		/* pgoff is invalid for ksm pages, but they are never large */
197 		if (folio_test_large(folio) && !folio_test_hugetlb(folio))
198 			idx = linear_page_index(vma, pvmw.address) - pvmw.pgoff;
199 		new = folio_page(folio, idx);
200 
201 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
202 		/* PMD-mapped THP migration entry */
203 		if (!pvmw.pte) {
204 			VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) ||
205 					!folio_test_pmd_mappable(folio), folio);
206 			remove_migration_pmd(&pvmw, new);
207 			continue;
208 		}
209 #endif
210 
211 		folio_get(folio);
212 		pte = mk_pte(new, READ_ONCE(vma->vm_page_prot));
213 		if (pte_swp_soft_dirty(*pvmw.pte))
214 			pte = pte_mksoft_dirty(pte);
215 
216 		entry = pte_to_swp_entry(*pvmw.pte);
217 		if (!is_migration_entry_young(entry))
218 			pte = pte_mkold(pte);
219 		if (folio_test_dirty(folio) && is_migration_entry_dirty(entry))
220 			pte = pte_mkdirty(pte);
221 		if (is_writable_migration_entry(entry))
222 			pte = pte_mkwrite(pte);
223 		else if (pte_swp_uffd_wp(*pvmw.pte))
224 			pte = pte_mkuffd_wp(pte);
225 
226 		if (folio_test_anon(folio) && !is_readable_migration_entry(entry))
227 			rmap_flags |= RMAP_EXCLUSIVE;
228 
229 		if (unlikely(is_device_private_page(new))) {
230 			if (pte_write(pte))
231 				entry = make_writable_device_private_entry(
232 							page_to_pfn(new));
233 			else
234 				entry = make_readable_device_private_entry(
235 							page_to_pfn(new));
236 			pte = swp_entry_to_pte(entry);
237 			if (pte_swp_soft_dirty(*pvmw.pte))
238 				pte = pte_swp_mksoft_dirty(pte);
239 			if (pte_swp_uffd_wp(*pvmw.pte))
240 				pte = pte_swp_mkuffd_wp(pte);
241 		}
242 
243 #ifdef CONFIG_HUGETLB_PAGE
244 		if (folio_test_hugetlb(folio)) {
245 			unsigned int shift = huge_page_shift(hstate_vma(vma));
246 
247 			pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
248 			if (folio_test_anon(folio))
249 				hugepage_add_anon_rmap(new, vma, pvmw.address,
250 						       rmap_flags);
251 			else
252 				page_dup_file_rmap(new, true);
253 			set_huge_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
254 		} else
255 #endif
256 		{
257 			if (folio_test_anon(folio))
258 				page_add_anon_rmap(new, vma, pvmw.address,
259 						   rmap_flags);
260 			else
261 				page_add_file_rmap(new, vma, false);
262 			set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
263 		}
264 		if (vma->vm_flags & VM_LOCKED)
265 			mlock_drain_local();
266 
267 		trace_remove_migration_pte(pvmw.address, pte_val(pte),
268 					   compound_order(new));
269 
270 		/* No need to invalidate - it was non-present before */
271 		update_mmu_cache(vma, pvmw.address, pvmw.pte);
272 	}
273 
274 	return true;
275 }
276 
277 /*
278  * Get rid of all migration entries and replace them by
279  * references to the indicated page.
280  */
281 void remove_migration_ptes(struct folio *src, struct folio *dst, bool locked)
282 {
283 	struct rmap_walk_control rwc = {
284 		.rmap_one = remove_migration_pte,
285 		.arg = src,
286 	};
287 
288 	if (locked)
289 		rmap_walk_locked(dst, &rwc);
290 	else
291 		rmap_walk(dst, &rwc);
292 }
293 
294 /*
295  * Something used the pte of a page under migration. We need to
296  * get to the page and wait until migration is finished.
297  * When we return from this function the fault will be retried.
298  */
299 void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep,
300 				spinlock_t *ptl)
301 {
302 	pte_t pte;
303 	swp_entry_t entry;
304 
305 	spin_lock(ptl);
306 	pte = *ptep;
307 	if (!is_swap_pte(pte))
308 		goto out;
309 
310 	entry = pte_to_swp_entry(pte);
311 	if (!is_migration_entry(entry))
312 		goto out;
313 
314 	migration_entry_wait_on_locked(entry, ptep, ptl);
315 	return;
316 out:
317 	pte_unmap_unlock(ptep, ptl);
318 }
319 
320 void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
321 				unsigned long address)
322 {
323 	spinlock_t *ptl = pte_lockptr(mm, pmd);
324 	pte_t *ptep = pte_offset_map(pmd, address);
325 	__migration_entry_wait(mm, ptep, ptl);
326 }
327 
328 #ifdef CONFIG_HUGETLB_PAGE
329 /*
330  * The vma read lock must be held upon entry. Holding that lock prevents either
331  * the pte or the ptl from being freed.
332  *
333  * This function will release the vma lock before returning.
334  */
335 void __migration_entry_wait_huge(struct vm_area_struct *vma,
336 				 pte_t *ptep, spinlock_t *ptl)
337 {
338 	pte_t pte;
339 
340 	hugetlb_vma_assert_locked(vma);
341 	spin_lock(ptl);
342 	pte = huge_ptep_get(ptep);
343 
344 	if (unlikely(!is_hugetlb_entry_migration(pte))) {
345 		spin_unlock(ptl);
346 		hugetlb_vma_unlock_read(vma);
347 	} else {
348 		/*
349 		 * If migration entry existed, safe to release vma lock
350 		 * here because the pgtable page won't be freed without the
351 		 * pgtable lock released.  See comment right above pgtable
352 		 * lock release in migration_entry_wait_on_locked().
353 		 */
354 		hugetlb_vma_unlock_read(vma);
355 		migration_entry_wait_on_locked(pte_to_swp_entry(pte), NULL, ptl);
356 	}
357 }
358 
359 void migration_entry_wait_huge(struct vm_area_struct *vma, pte_t *pte)
360 {
361 	spinlock_t *ptl = huge_pte_lockptr(hstate_vma(vma), vma->vm_mm, pte);
362 
363 	__migration_entry_wait_huge(vma, pte, ptl);
364 }
365 #endif
366 
367 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
368 void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd)
369 {
370 	spinlock_t *ptl;
371 
372 	ptl = pmd_lock(mm, pmd);
373 	if (!is_pmd_migration_entry(*pmd))
374 		goto unlock;
375 	migration_entry_wait_on_locked(pmd_to_swp_entry(*pmd), NULL, ptl);
376 	return;
377 unlock:
378 	spin_unlock(ptl);
379 }
380 #endif
381 
382 static int folio_expected_refs(struct address_space *mapping,
383 		struct folio *folio)
384 {
385 	int refs = 1;
386 	if (!mapping)
387 		return refs;
388 
389 	refs += folio_nr_pages(folio);
390 	if (folio_test_private(folio))
391 		refs++;
392 
393 	return refs;
394 }
395 
396 /*
397  * Replace the page in the mapping.
398  *
399  * The number of remaining references must be:
400  * 1 for anonymous pages without a mapping
401  * 2 for pages with a mapping
402  * 3 for pages with a mapping and PagePrivate/PagePrivate2 set.
403  */
404 int folio_migrate_mapping(struct address_space *mapping,
405 		struct folio *newfolio, struct folio *folio, int extra_count)
406 {
407 	XA_STATE(xas, &mapping->i_pages, folio_index(folio));
408 	struct zone *oldzone, *newzone;
409 	int dirty;
410 	int expected_count = folio_expected_refs(mapping, folio) + extra_count;
411 	long nr = folio_nr_pages(folio);
412 
413 	if (!mapping) {
414 		/* Anonymous page without mapping */
415 		if (folio_ref_count(folio) != expected_count)
416 			return -EAGAIN;
417 
418 		/* No turning back from here */
419 		newfolio->index = folio->index;
420 		newfolio->mapping = folio->mapping;
421 		if (folio_test_swapbacked(folio))
422 			__folio_set_swapbacked(newfolio);
423 
424 		return MIGRATEPAGE_SUCCESS;
425 	}
426 
427 	oldzone = folio_zone(folio);
428 	newzone = folio_zone(newfolio);
429 
430 	xas_lock_irq(&xas);
431 	if (!folio_ref_freeze(folio, expected_count)) {
432 		xas_unlock_irq(&xas);
433 		return -EAGAIN;
434 	}
435 
436 	/*
437 	 * Now we know that no one else is looking at the folio:
438 	 * no turning back from here.
439 	 */
440 	newfolio->index = folio->index;
441 	newfolio->mapping = folio->mapping;
442 	folio_ref_add(newfolio, nr); /* add cache reference */
443 	if (folio_test_swapbacked(folio)) {
444 		__folio_set_swapbacked(newfolio);
445 		if (folio_test_swapcache(folio)) {
446 			folio_set_swapcache(newfolio);
447 			newfolio->private = folio_get_private(folio);
448 		}
449 	} else {
450 		VM_BUG_ON_FOLIO(folio_test_swapcache(folio), folio);
451 	}
452 
453 	/* Move dirty while page refs frozen and newpage not yet exposed */
454 	dirty = folio_test_dirty(folio);
455 	if (dirty) {
456 		folio_clear_dirty(folio);
457 		folio_set_dirty(newfolio);
458 	}
459 
460 	xas_store(&xas, newfolio);
461 
462 	/*
463 	 * Drop cache reference from old page by unfreezing
464 	 * to one less reference.
465 	 * We know this isn't the last reference.
466 	 */
467 	folio_ref_unfreeze(folio, expected_count - nr);
468 
469 	xas_unlock(&xas);
470 	/* Leave irq disabled to prevent preemption while updating stats */
471 
472 	/*
473 	 * If moved to a different zone then also account
474 	 * the page for that zone. Other VM counters will be
475 	 * taken care of when we establish references to the
476 	 * new page and drop references to the old page.
477 	 *
478 	 * Note that anonymous pages are accounted for
479 	 * via NR_FILE_PAGES and NR_ANON_MAPPED if they
480 	 * are mapped to swap space.
481 	 */
482 	if (newzone != oldzone) {
483 		struct lruvec *old_lruvec, *new_lruvec;
484 		struct mem_cgroup *memcg;
485 
486 		memcg = folio_memcg(folio);
487 		old_lruvec = mem_cgroup_lruvec(memcg, oldzone->zone_pgdat);
488 		new_lruvec = mem_cgroup_lruvec(memcg, newzone->zone_pgdat);
489 
490 		__mod_lruvec_state(old_lruvec, NR_FILE_PAGES, -nr);
491 		__mod_lruvec_state(new_lruvec, NR_FILE_PAGES, nr);
492 		if (folio_test_swapbacked(folio) && !folio_test_swapcache(folio)) {
493 			__mod_lruvec_state(old_lruvec, NR_SHMEM, -nr);
494 			__mod_lruvec_state(new_lruvec, NR_SHMEM, nr);
495 		}
496 #ifdef CONFIG_SWAP
497 		if (folio_test_swapcache(folio)) {
498 			__mod_lruvec_state(old_lruvec, NR_SWAPCACHE, -nr);
499 			__mod_lruvec_state(new_lruvec, NR_SWAPCACHE, nr);
500 		}
501 #endif
502 		if (dirty && mapping_can_writeback(mapping)) {
503 			__mod_lruvec_state(old_lruvec, NR_FILE_DIRTY, -nr);
504 			__mod_zone_page_state(oldzone, NR_ZONE_WRITE_PENDING, -nr);
505 			__mod_lruvec_state(new_lruvec, NR_FILE_DIRTY, nr);
506 			__mod_zone_page_state(newzone, NR_ZONE_WRITE_PENDING, nr);
507 		}
508 	}
509 	local_irq_enable();
510 
511 	return MIGRATEPAGE_SUCCESS;
512 }
513 EXPORT_SYMBOL(folio_migrate_mapping);
514 
515 /*
516  * The expected number of remaining references is the same as that
517  * of folio_migrate_mapping().
518  */
519 int migrate_huge_page_move_mapping(struct address_space *mapping,
520 				   struct folio *dst, struct folio *src)
521 {
522 	XA_STATE(xas, &mapping->i_pages, folio_index(src));
523 	int expected_count;
524 
525 	xas_lock_irq(&xas);
526 	expected_count = 2 + folio_has_private(src);
527 	if (!folio_ref_freeze(src, expected_count)) {
528 		xas_unlock_irq(&xas);
529 		return -EAGAIN;
530 	}
531 
532 	dst->index = src->index;
533 	dst->mapping = src->mapping;
534 
535 	folio_get(dst);
536 
537 	xas_store(&xas, dst);
538 
539 	folio_ref_unfreeze(src, expected_count - 1);
540 
541 	xas_unlock_irq(&xas);
542 
543 	return MIGRATEPAGE_SUCCESS;
544 }
545 
546 /*
547  * Copy the flags and some other ancillary information
548  */
549 void folio_migrate_flags(struct folio *newfolio, struct folio *folio)
550 {
551 	int cpupid;
552 
553 	if (folio_test_error(folio))
554 		folio_set_error(newfolio);
555 	if (folio_test_referenced(folio))
556 		folio_set_referenced(newfolio);
557 	if (folio_test_uptodate(folio))
558 		folio_mark_uptodate(newfolio);
559 	if (folio_test_clear_active(folio)) {
560 		VM_BUG_ON_FOLIO(folio_test_unevictable(folio), folio);
561 		folio_set_active(newfolio);
562 	} else if (folio_test_clear_unevictable(folio))
563 		folio_set_unevictable(newfolio);
564 	if (folio_test_workingset(folio))
565 		folio_set_workingset(newfolio);
566 	if (folio_test_checked(folio))
567 		folio_set_checked(newfolio);
568 	/*
569 	 * PG_anon_exclusive (-> PG_mappedtodisk) is always migrated via
570 	 * migration entries. We can still have PG_anon_exclusive set on an
571 	 * effectively unmapped and unreferenced first sub-pages of an
572 	 * anonymous THP: we can simply copy it here via PG_mappedtodisk.
573 	 */
574 	if (folio_test_mappedtodisk(folio))
575 		folio_set_mappedtodisk(newfolio);
576 
577 	/* Move dirty on pages not done by folio_migrate_mapping() */
578 	if (folio_test_dirty(folio))
579 		folio_set_dirty(newfolio);
580 
581 	if (folio_test_young(folio))
582 		folio_set_young(newfolio);
583 	if (folio_test_idle(folio))
584 		folio_set_idle(newfolio);
585 
586 	/*
587 	 * Copy NUMA information to the new page, to prevent over-eager
588 	 * future migrations of this same page.
589 	 */
590 	cpupid = page_cpupid_xchg_last(&folio->page, -1);
591 	/*
592 	 * For memory tiering mode, when migrate between slow and fast
593 	 * memory node, reset cpupid, because that is used to record
594 	 * page access time in slow memory node.
595 	 */
596 	if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) {
597 		bool f_toptier = node_is_toptier(page_to_nid(&folio->page));
598 		bool t_toptier = node_is_toptier(page_to_nid(&newfolio->page));
599 
600 		if (f_toptier != t_toptier)
601 			cpupid = -1;
602 	}
603 	page_cpupid_xchg_last(&newfolio->page, cpupid);
604 
605 	folio_migrate_ksm(newfolio, folio);
606 	/*
607 	 * Please do not reorder this without considering how mm/ksm.c's
608 	 * get_ksm_page() depends upon ksm_migrate_page() and PageSwapCache().
609 	 */
610 	if (folio_test_swapcache(folio))
611 		folio_clear_swapcache(folio);
612 	folio_clear_private(folio);
613 
614 	/* page->private contains hugetlb specific flags */
615 	if (!folio_test_hugetlb(folio))
616 		folio->private = NULL;
617 
618 	/*
619 	 * If any waiters have accumulated on the new page then
620 	 * wake them up.
621 	 */
622 	if (folio_test_writeback(newfolio))
623 		folio_end_writeback(newfolio);
624 
625 	/*
626 	 * PG_readahead shares the same bit with PG_reclaim.  The above
627 	 * end_page_writeback() may clear PG_readahead mistakenly, so set the
628 	 * bit after that.
629 	 */
630 	if (folio_test_readahead(folio))
631 		folio_set_readahead(newfolio);
632 
633 	folio_copy_owner(newfolio, folio);
634 
635 	if (!folio_test_hugetlb(folio))
636 		mem_cgroup_migrate(folio, newfolio);
637 }
638 EXPORT_SYMBOL(folio_migrate_flags);
639 
640 void folio_migrate_copy(struct folio *newfolio, struct folio *folio)
641 {
642 	folio_copy(newfolio, folio);
643 	folio_migrate_flags(newfolio, folio);
644 }
645 EXPORT_SYMBOL(folio_migrate_copy);
646 
647 /************************************************************
648  *                    Migration functions
649  ***********************************************************/
650 
651 int migrate_folio_extra(struct address_space *mapping, struct folio *dst,
652 		struct folio *src, enum migrate_mode mode, int extra_count)
653 {
654 	int rc;
655 
656 	BUG_ON(folio_test_writeback(src));	/* Writeback must be complete */
657 
658 	rc = folio_migrate_mapping(mapping, dst, src, extra_count);
659 
660 	if (rc != MIGRATEPAGE_SUCCESS)
661 		return rc;
662 
663 	if (mode != MIGRATE_SYNC_NO_COPY)
664 		folio_migrate_copy(dst, src);
665 	else
666 		folio_migrate_flags(dst, src);
667 	return MIGRATEPAGE_SUCCESS;
668 }
669 
670 /**
671  * migrate_folio() - Simple folio migration.
672  * @mapping: The address_space containing the folio.
673  * @dst: The folio to migrate the data to.
674  * @src: The folio containing the current data.
675  * @mode: How to migrate the page.
676  *
677  * Common logic to directly migrate a single LRU folio suitable for
678  * folios that do not use PagePrivate/PagePrivate2.
679  *
680  * Folios are locked upon entry and exit.
681  */
682 int migrate_folio(struct address_space *mapping, struct folio *dst,
683 		struct folio *src, enum migrate_mode mode)
684 {
685 	return migrate_folio_extra(mapping, dst, src, mode, 0);
686 }
687 EXPORT_SYMBOL(migrate_folio);
688 
689 #ifdef CONFIG_BLOCK
690 /* Returns true if all buffers are successfully locked */
691 static bool buffer_migrate_lock_buffers(struct buffer_head *head,
692 							enum migrate_mode mode)
693 {
694 	struct buffer_head *bh = head;
695 
696 	/* Simple case, sync compaction */
697 	if (mode != MIGRATE_ASYNC) {
698 		do {
699 			lock_buffer(bh);
700 			bh = bh->b_this_page;
701 
702 		} while (bh != head);
703 
704 		return true;
705 	}
706 
707 	/* async case, we cannot block on lock_buffer so use trylock_buffer */
708 	do {
709 		if (!trylock_buffer(bh)) {
710 			/*
711 			 * We failed to lock the buffer and cannot stall in
712 			 * async migration. Release the taken locks
713 			 */
714 			struct buffer_head *failed_bh = bh;
715 			bh = head;
716 			while (bh != failed_bh) {
717 				unlock_buffer(bh);
718 				bh = bh->b_this_page;
719 			}
720 			return false;
721 		}
722 
723 		bh = bh->b_this_page;
724 	} while (bh != head);
725 	return true;
726 }
727 
728 static int __buffer_migrate_folio(struct address_space *mapping,
729 		struct folio *dst, struct folio *src, enum migrate_mode mode,
730 		bool check_refs)
731 {
732 	struct buffer_head *bh, *head;
733 	int rc;
734 	int expected_count;
735 
736 	head = folio_buffers(src);
737 	if (!head)
738 		return migrate_folio(mapping, dst, src, mode);
739 
740 	/* Check whether page does not have extra refs before we do more work */
741 	expected_count = folio_expected_refs(mapping, src);
742 	if (folio_ref_count(src) != expected_count)
743 		return -EAGAIN;
744 
745 	if (!buffer_migrate_lock_buffers(head, mode))
746 		return -EAGAIN;
747 
748 	if (check_refs) {
749 		bool busy;
750 		bool invalidated = false;
751 
752 recheck_buffers:
753 		busy = false;
754 		spin_lock(&mapping->private_lock);
755 		bh = head;
756 		do {
757 			if (atomic_read(&bh->b_count)) {
758 				busy = true;
759 				break;
760 			}
761 			bh = bh->b_this_page;
762 		} while (bh != head);
763 		if (busy) {
764 			if (invalidated) {
765 				rc = -EAGAIN;
766 				goto unlock_buffers;
767 			}
768 			spin_unlock(&mapping->private_lock);
769 			invalidate_bh_lrus();
770 			invalidated = true;
771 			goto recheck_buffers;
772 		}
773 	}
774 
775 	rc = folio_migrate_mapping(mapping, dst, src, 0);
776 	if (rc != MIGRATEPAGE_SUCCESS)
777 		goto unlock_buffers;
778 
779 	folio_attach_private(dst, folio_detach_private(src));
780 
781 	bh = head;
782 	do {
783 		set_bh_page(bh, &dst->page, bh_offset(bh));
784 		bh = bh->b_this_page;
785 	} while (bh != head);
786 
787 	if (mode != MIGRATE_SYNC_NO_COPY)
788 		folio_migrate_copy(dst, src);
789 	else
790 		folio_migrate_flags(dst, src);
791 
792 	rc = MIGRATEPAGE_SUCCESS;
793 unlock_buffers:
794 	if (check_refs)
795 		spin_unlock(&mapping->private_lock);
796 	bh = head;
797 	do {
798 		unlock_buffer(bh);
799 		bh = bh->b_this_page;
800 	} while (bh != head);
801 
802 	return rc;
803 }
804 
805 /**
806  * buffer_migrate_folio() - Migration function for folios with buffers.
807  * @mapping: The address space containing @src.
808  * @dst: The folio to migrate to.
809  * @src: The folio to migrate from.
810  * @mode: How to migrate the folio.
811  *
812  * This function can only be used if the underlying filesystem guarantees
813  * that no other references to @src exist. For example attached buffer
814  * heads are accessed only under the folio lock.  If your filesystem cannot
815  * provide this guarantee, buffer_migrate_folio_norefs() may be more
816  * appropriate.
817  *
818  * Return: 0 on success or a negative errno on failure.
819  */
820 int buffer_migrate_folio(struct address_space *mapping,
821 		struct folio *dst, struct folio *src, enum migrate_mode mode)
822 {
823 	return __buffer_migrate_folio(mapping, dst, src, mode, false);
824 }
825 EXPORT_SYMBOL(buffer_migrate_folio);
826 
827 /**
828  * buffer_migrate_folio_norefs() - Migration function for folios with buffers.
829  * @mapping: The address space containing @src.
830  * @dst: The folio to migrate to.
831  * @src: The folio to migrate from.
832  * @mode: How to migrate the folio.
833  *
834  * Like buffer_migrate_folio() except that this variant is more careful
835  * and checks that there are also no buffer head references. This function
836  * is the right one for mappings where buffer heads are directly looked
837  * up and referenced (such as block device mappings).
838  *
839  * Return: 0 on success or a negative errno on failure.
840  */
841 int buffer_migrate_folio_norefs(struct address_space *mapping,
842 		struct folio *dst, struct folio *src, enum migrate_mode mode)
843 {
844 	return __buffer_migrate_folio(mapping, dst, src, mode, true);
845 }
846 EXPORT_SYMBOL_GPL(buffer_migrate_folio_norefs);
847 #endif
848 
849 int filemap_migrate_folio(struct address_space *mapping,
850 		struct folio *dst, struct folio *src, enum migrate_mode mode)
851 {
852 	int ret;
853 
854 	ret = folio_migrate_mapping(mapping, dst, src, 0);
855 	if (ret != MIGRATEPAGE_SUCCESS)
856 		return ret;
857 
858 	if (folio_get_private(src))
859 		folio_attach_private(dst, folio_detach_private(src));
860 
861 	if (mode != MIGRATE_SYNC_NO_COPY)
862 		folio_migrate_copy(dst, src);
863 	else
864 		folio_migrate_flags(dst, src);
865 	return MIGRATEPAGE_SUCCESS;
866 }
867 EXPORT_SYMBOL_GPL(filemap_migrate_folio);
868 
869 /*
870  * Writeback a folio to clean the dirty state
871  */
872 static int writeout(struct address_space *mapping, struct folio *folio)
873 {
874 	struct writeback_control wbc = {
875 		.sync_mode = WB_SYNC_NONE,
876 		.nr_to_write = 1,
877 		.range_start = 0,
878 		.range_end = LLONG_MAX,
879 		.for_reclaim = 1
880 	};
881 	int rc;
882 
883 	if (!mapping->a_ops->writepage)
884 		/* No write method for the address space */
885 		return -EINVAL;
886 
887 	if (!folio_clear_dirty_for_io(folio))
888 		/* Someone else already triggered a write */
889 		return -EAGAIN;
890 
891 	/*
892 	 * A dirty folio may imply that the underlying filesystem has
893 	 * the folio on some queue. So the folio must be clean for
894 	 * migration. Writeout may mean we lose the lock and the
895 	 * folio state is no longer what we checked for earlier.
896 	 * At this point we know that the migration attempt cannot
897 	 * be successful.
898 	 */
899 	remove_migration_ptes(folio, folio, false);
900 
901 	rc = mapping->a_ops->writepage(&folio->page, &wbc);
902 
903 	if (rc != AOP_WRITEPAGE_ACTIVATE)
904 		/* unlocked. Relock */
905 		folio_lock(folio);
906 
907 	return (rc < 0) ? -EIO : -EAGAIN;
908 }
909 
910 /*
911  * Default handling if a filesystem does not provide a migration function.
912  */
913 static int fallback_migrate_folio(struct address_space *mapping,
914 		struct folio *dst, struct folio *src, enum migrate_mode mode)
915 {
916 	if (folio_test_dirty(src)) {
917 		/* Only writeback folios in full synchronous migration */
918 		switch (mode) {
919 		case MIGRATE_SYNC:
920 		case MIGRATE_SYNC_NO_COPY:
921 			break;
922 		default:
923 			return -EBUSY;
924 		}
925 		return writeout(mapping, src);
926 	}
927 
928 	/*
929 	 * Buffers may be managed in a filesystem specific way.
930 	 * We must have no buffers or drop them.
931 	 */
932 	if (folio_test_private(src) &&
933 	    !filemap_release_folio(src, GFP_KERNEL))
934 		return mode == MIGRATE_SYNC ? -EAGAIN : -EBUSY;
935 
936 	return migrate_folio(mapping, dst, src, mode);
937 }
938 
939 /*
940  * Move a page to a newly allocated page
941  * The page is locked and all ptes have been successfully removed.
942  *
943  * The new page will have replaced the old page if this function
944  * is successful.
945  *
946  * Return value:
947  *   < 0 - error code
948  *  MIGRATEPAGE_SUCCESS - success
949  */
950 static int move_to_new_folio(struct folio *dst, struct folio *src,
951 				enum migrate_mode mode)
952 {
953 	int rc = -EAGAIN;
954 	bool is_lru = !__PageMovable(&src->page);
955 
956 	VM_BUG_ON_FOLIO(!folio_test_locked(src), src);
957 	VM_BUG_ON_FOLIO(!folio_test_locked(dst), dst);
958 
959 	if (likely(is_lru)) {
960 		struct address_space *mapping = folio_mapping(src);
961 
962 		if (!mapping)
963 			rc = migrate_folio(mapping, dst, src, mode);
964 		else if (mapping->a_ops->migrate_folio)
965 			/*
966 			 * Most folios have a mapping and most filesystems
967 			 * provide a migrate_folio callback. Anonymous folios
968 			 * are part of swap space which also has its own
969 			 * migrate_folio callback. This is the most common path
970 			 * for page migration.
971 			 */
972 			rc = mapping->a_ops->migrate_folio(mapping, dst, src,
973 								mode);
974 		else
975 			rc = fallback_migrate_folio(mapping, dst, src, mode);
976 	} else {
977 		const struct movable_operations *mops;
978 
979 		/*
980 		 * In case of non-lru page, it could be released after
981 		 * isolation step. In that case, we shouldn't try migration.
982 		 */
983 		VM_BUG_ON_FOLIO(!folio_test_isolated(src), src);
984 		if (!folio_test_movable(src)) {
985 			rc = MIGRATEPAGE_SUCCESS;
986 			folio_clear_isolated(src);
987 			goto out;
988 		}
989 
990 		mops = folio_movable_ops(src);
991 		rc = mops->migrate_page(&dst->page, &src->page, mode);
992 		WARN_ON_ONCE(rc == MIGRATEPAGE_SUCCESS &&
993 				!folio_test_isolated(src));
994 	}
995 
996 	/*
997 	 * When successful, old pagecache src->mapping must be cleared before
998 	 * src is freed; but stats require that PageAnon be left as PageAnon.
999 	 */
1000 	if (rc == MIGRATEPAGE_SUCCESS) {
1001 		if (__PageMovable(&src->page)) {
1002 			VM_BUG_ON_FOLIO(!folio_test_isolated(src), src);
1003 
1004 			/*
1005 			 * We clear PG_movable under page_lock so any compactor
1006 			 * cannot try to migrate this page.
1007 			 */
1008 			folio_clear_isolated(src);
1009 		}
1010 
1011 		/*
1012 		 * Anonymous and movable src->mapping will be cleared by
1013 		 * free_pages_prepare so don't reset it here for keeping
1014 		 * the type to work PageAnon, for example.
1015 		 */
1016 		if (!folio_mapping_flags(src))
1017 			src->mapping = NULL;
1018 
1019 		if (likely(!folio_is_zone_device(dst)))
1020 			flush_dcache_folio(dst);
1021 	}
1022 out:
1023 	return rc;
1024 }
1025 
1026 /*
1027  * To record some information during migration, we use some unused
1028  * fields (mapping and private) of struct folio of the newly allocated
1029  * destination folio.  This is safe because nobody is using them
1030  * except us.
1031  */
1032 union migration_ptr {
1033 	struct anon_vma *anon_vma;
1034 	struct address_space *mapping;
1035 };
1036 static void __migrate_folio_record(struct folio *dst,
1037 				   unsigned long page_was_mapped,
1038 				   struct anon_vma *anon_vma)
1039 {
1040 	union migration_ptr ptr = { .anon_vma = anon_vma };
1041 	dst->mapping = ptr.mapping;
1042 	dst->private = (void *)page_was_mapped;
1043 }
1044 
1045 static void __migrate_folio_extract(struct folio *dst,
1046 				   int *page_was_mappedp,
1047 				   struct anon_vma **anon_vmap)
1048 {
1049 	union migration_ptr ptr = { .mapping = dst->mapping };
1050 	*anon_vmap = ptr.anon_vma;
1051 	*page_was_mappedp = (unsigned long)dst->private;
1052 	dst->mapping = NULL;
1053 	dst->private = NULL;
1054 }
1055 
1056 /* Restore the source folio to the original state upon failure */
1057 static void migrate_folio_undo_src(struct folio *src,
1058 				   int page_was_mapped,
1059 				   struct anon_vma *anon_vma,
1060 				   bool locked,
1061 				   struct list_head *ret)
1062 {
1063 	if (page_was_mapped)
1064 		remove_migration_ptes(src, src, false);
1065 	/* Drop an anon_vma reference if we took one */
1066 	if (anon_vma)
1067 		put_anon_vma(anon_vma);
1068 	if (locked)
1069 		folio_unlock(src);
1070 	if (ret)
1071 		list_move_tail(&src->lru, ret);
1072 }
1073 
1074 /* Restore the destination folio to the original state upon failure */
1075 static void migrate_folio_undo_dst(struct folio *dst,
1076 				   bool locked,
1077 				   free_page_t put_new_page,
1078 				   unsigned long private)
1079 {
1080 	if (locked)
1081 		folio_unlock(dst);
1082 	if (put_new_page)
1083 		put_new_page(&dst->page, private);
1084 	else
1085 		folio_put(dst);
1086 }
1087 
1088 /* Cleanup src folio upon migration success */
1089 static void migrate_folio_done(struct folio *src,
1090 			       enum migrate_reason reason)
1091 {
1092 	/*
1093 	 * Compaction can migrate also non-LRU pages which are
1094 	 * not accounted to NR_ISOLATED_*. They can be recognized
1095 	 * as __PageMovable
1096 	 */
1097 	if (likely(!__folio_test_movable(src)))
1098 		mod_node_page_state(folio_pgdat(src), NR_ISOLATED_ANON +
1099 				    folio_is_file_lru(src), -folio_nr_pages(src));
1100 
1101 	if (reason != MR_MEMORY_FAILURE)
1102 		/* We release the page in page_handle_poison. */
1103 		folio_put(src);
1104 }
1105 
1106 /* Obtain the lock on page, remove all ptes. */
1107 static int migrate_folio_unmap(new_page_t get_new_page, free_page_t put_new_page,
1108 			       unsigned long private, struct folio *src,
1109 			       struct folio **dstp, enum migrate_mode mode,
1110 			       enum migrate_reason reason, struct list_head *ret)
1111 {
1112 	struct folio *dst;
1113 	int rc = -EAGAIN;
1114 	struct page *newpage = NULL;
1115 	int page_was_mapped = 0;
1116 	struct anon_vma *anon_vma = NULL;
1117 	bool is_lru = !__PageMovable(&src->page);
1118 	bool locked = false;
1119 	bool dst_locked = false;
1120 
1121 	if (folio_ref_count(src) == 1) {
1122 		/* Folio was freed from under us. So we are done. */
1123 		folio_clear_active(src);
1124 		folio_clear_unevictable(src);
1125 		/* free_pages_prepare() will clear PG_isolated. */
1126 		list_del(&src->lru);
1127 		migrate_folio_done(src, reason);
1128 		return MIGRATEPAGE_SUCCESS;
1129 	}
1130 
1131 	newpage = get_new_page(&src->page, private);
1132 	if (!newpage)
1133 		return -ENOMEM;
1134 	dst = page_folio(newpage);
1135 	*dstp = dst;
1136 
1137 	dst->private = NULL;
1138 
1139 	if (!folio_trylock(src)) {
1140 		if (mode == MIGRATE_ASYNC)
1141 			goto out;
1142 
1143 		/*
1144 		 * It's not safe for direct compaction to call lock_page.
1145 		 * For example, during page readahead pages are added locked
1146 		 * to the LRU. Later, when the IO completes the pages are
1147 		 * marked uptodate and unlocked. However, the queueing
1148 		 * could be merging multiple pages for one bio (e.g.
1149 		 * mpage_readahead). If an allocation happens for the
1150 		 * second or third page, the process can end up locking
1151 		 * the same page twice and deadlocking. Rather than
1152 		 * trying to be clever about what pages can be locked,
1153 		 * avoid the use of lock_page for direct compaction
1154 		 * altogether.
1155 		 */
1156 		if (current->flags & PF_MEMALLOC)
1157 			goto out;
1158 
1159 		folio_lock(src);
1160 	}
1161 	locked = true;
1162 
1163 	if (folio_test_writeback(src)) {
1164 		/*
1165 		 * Only in the case of a full synchronous migration is it
1166 		 * necessary to wait for PageWriteback. In the async case,
1167 		 * the retry loop is too short and in the sync-light case,
1168 		 * the overhead of stalling is too much
1169 		 */
1170 		switch (mode) {
1171 		case MIGRATE_SYNC:
1172 		case MIGRATE_SYNC_NO_COPY:
1173 			break;
1174 		default:
1175 			rc = -EBUSY;
1176 			goto out;
1177 		}
1178 		folio_wait_writeback(src);
1179 	}
1180 
1181 	/*
1182 	 * By try_to_migrate(), src->mapcount goes down to 0 here. In this case,
1183 	 * we cannot notice that anon_vma is freed while we migrate a page.
1184 	 * This get_anon_vma() delays freeing anon_vma pointer until the end
1185 	 * of migration. File cache pages are no problem because of page_lock()
1186 	 * File Caches may use write_page() or lock_page() in migration, then,
1187 	 * just care Anon page here.
1188 	 *
1189 	 * Only folio_get_anon_vma() understands the subtleties of
1190 	 * getting a hold on an anon_vma from outside one of its mms.
1191 	 * But if we cannot get anon_vma, then we won't need it anyway,
1192 	 * because that implies that the anon page is no longer mapped
1193 	 * (and cannot be remapped so long as we hold the page lock).
1194 	 */
1195 	if (folio_test_anon(src) && !folio_test_ksm(src))
1196 		anon_vma = folio_get_anon_vma(src);
1197 
1198 	/*
1199 	 * Block others from accessing the new page when we get around to
1200 	 * establishing additional references. We are usually the only one
1201 	 * holding a reference to dst at this point. We used to have a BUG
1202 	 * here if folio_trylock(dst) fails, but would like to allow for
1203 	 * cases where there might be a race with the previous use of dst.
1204 	 * This is much like races on refcount of oldpage: just don't BUG().
1205 	 */
1206 	if (unlikely(!folio_trylock(dst)))
1207 		goto out;
1208 	dst_locked = true;
1209 
1210 	if (unlikely(!is_lru)) {
1211 		__migrate_folio_record(dst, page_was_mapped, anon_vma);
1212 		return MIGRATEPAGE_UNMAP;
1213 	}
1214 
1215 	/*
1216 	 * Corner case handling:
1217 	 * 1. When a new swap-cache page is read into, it is added to the LRU
1218 	 * and treated as swapcache but it has no rmap yet.
1219 	 * Calling try_to_unmap() against a src->mapping==NULL page will
1220 	 * trigger a BUG.  So handle it here.
1221 	 * 2. An orphaned page (see truncate_cleanup_page) might have
1222 	 * fs-private metadata. The page can be picked up due to memory
1223 	 * offlining.  Everywhere else except page reclaim, the page is
1224 	 * invisible to the vm, so the page can not be migrated.  So try to
1225 	 * free the metadata, so the page can be freed.
1226 	 */
1227 	if (!src->mapping) {
1228 		if (folio_test_private(src)) {
1229 			try_to_free_buffers(src);
1230 			goto out;
1231 		}
1232 	} else if (folio_mapped(src)) {
1233 		/* Establish migration ptes */
1234 		VM_BUG_ON_FOLIO(folio_test_anon(src) &&
1235 			       !folio_test_ksm(src) && !anon_vma, src);
1236 		try_to_migrate(src, mode == MIGRATE_ASYNC ? TTU_BATCH_FLUSH : 0);
1237 		page_was_mapped = 1;
1238 	}
1239 
1240 	if (!folio_mapped(src)) {
1241 		__migrate_folio_record(dst, page_was_mapped, anon_vma);
1242 		return MIGRATEPAGE_UNMAP;
1243 	}
1244 
1245 out:
1246 	/*
1247 	 * A folio that has not been unmapped will be restored to
1248 	 * right list unless we want to retry.
1249 	 */
1250 	if (rc == -EAGAIN)
1251 		ret = NULL;
1252 
1253 	migrate_folio_undo_src(src, page_was_mapped, anon_vma, locked, ret);
1254 	migrate_folio_undo_dst(dst, dst_locked, put_new_page, private);
1255 
1256 	return rc;
1257 }
1258 
1259 /* Migrate the folio to the newly allocated folio in dst. */
1260 static int migrate_folio_move(free_page_t put_new_page, unsigned long private,
1261 			      struct folio *src, struct folio *dst,
1262 			      enum migrate_mode mode, enum migrate_reason reason,
1263 			      struct list_head *ret)
1264 {
1265 	int rc;
1266 	int page_was_mapped = 0;
1267 	struct anon_vma *anon_vma = NULL;
1268 	bool is_lru = !__PageMovable(&src->page);
1269 	struct list_head *prev;
1270 
1271 	__migrate_folio_extract(dst, &page_was_mapped, &anon_vma);
1272 	prev = dst->lru.prev;
1273 	list_del(&dst->lru);
1274 
1275 	rc = move_to_new_folio(dst, src, mode);
1276 	if (rc)
1277 		goto out;
1278 
1279 	if (unlikely(!is_lru))
1280 		goto out_unlock_both;
1281 
1282 	/*
1283 	 * When successful, push dst to LRU immediately: so that if it
1284 	 * turns out to be an mlocked page, remove_migration_ptes() will
1285 	 * automatically build up the correct dst->mlock_count for it.
1286 	 *
1287 	 * We would like to do something similar for the old page, when
1288 	 * unsuccessful, and other cases when a page has been temporarily
1289 	 * isolated from the unevictable LRU: but this case is the easiest.
1290 	 */
1291 	folio_add_lru(dst);
1292 	if (page_was_mapped)
1293 		lru_add_drain();
1294 
1295 	if (page_was_mapped)
1296 		remove_migration_ptes(src, dst, false);
1297 
1298 out_unlock_both:
1299 	folio_unlock(dst);
1300 	set_page_owner_migrate_reason(&dst->page, reason);
1301 	/*
1302 	 * If migration is successful, decrease refcount of dst,
1303 	 * which will not free the page because new page owner increased
1304 	 * refcounter.
1305 	 */
1306 	folio_put(dst);
1307 
1308 	/*
1309 	 * A folio that has been migrated has all references removed
1310 	 * and will be freed.
1311 	 */
1312 	list_del(&src->lru);
1313 	/* Drop an anon_vma reference if we took one */
1314 	if (anon_vma)
1315 		put_anon_vma(anon_vma);
1316 	folio_unlock(src);
1317 	migrate_folio_done(src, reason);
1318 
1319 	return rc;
1320 out:
1321 	/*
1322 	 * A folio that has not been migrated will be restored to
1323 	 * right list unless we want to retry.
1324 	 */
1325 	if (rc == -EAGAIN) {
1326 		list_add(&dst->lru, prev);
1327 		__migrate_folio_record(dst, page_was_mapped, anon_vma);
1328 		return rc;
1329 	}
1330 
1331 	migrate_folio_undo_src(src, page_was_mapped, anon_vma, true, ret);
1332 	migrate_folio_undo_dst(dst, true, put_new_page, private);
1333 
1334 	return rc;
1335 }
1336 
1337 /*
1338  * Counterpart of unmap_and_move_page() for hugepage migration.
1339  *
1340  * This function doesn't wait the completion of hugepage I/O
1341  * because there is no race between I/O and migration for hugepage.
1342  * Note that currently hugepage I/O occurs only in direct I/O
1343  * where no lock is held and PG_writeback is irrelevant,
1344  * and writeback status of all subpages are counted in the reference
1345  * count of the head page (i.e. if all subpages of a 2MB hugepage are
1346  * under direct I/O, the reference of the head page is 512 and a bit more.)
1347  * This means that when we try to migrate hugepage whose subpages are
1348  * doing direct I/O, some references remain after try_to_unmap() and
1349  * hugepage migration fails without data corruption.
1350  *
1351  * There is also no race when direct I/O is issued on the page under migration,
1352  * because then pte is replaced with migration swap entry and direct I/O code
1353  * will wait in the page fault for migration to complete.
1354  */
1355 static int unmap_and_move_huge_page(new_page_t get_new_page,
1356 				free_page_t put_new_page, unsigned long private,
1357 				struct page *hpage, int force,
1358 				enum migrate_mode mode, int reason,
1359 				struct list_head *ret)
1360 {
1361 	struct folio *dst, *src = page_folio(hpage);
1362 	int rc = -EAGAIN;
1363 	int page_was_mapped = 0;
1364 	struct page *new_hpage;
1365 	struct anon_vma *anon_vma = NULL;
1366 	struct address_space *mapping = NULL;
1367 
1368 	if (folio_ref_count(src) == 1) {
1369 		/* page was freed from under us. So we are done. */
1370 		folio_putback_active_hugetlb(src);
1371 		return MIGRATEPAGE_SUCCESS;
1372 	}
1373 
1374 	new_hpage = get_new_page(hpage, private);
1375 	if (!new_hpage)
1376 		return -ENOMEM;
1377 	dst = page_folio(new_hpage);
1378 
1379 	if (!folio_trylock(src)) {
1380 		if (!force)
1381 			goto out;
1382 		switch (mode) {
1383 		case MIGRATE_SYNC:
1384 		case MIGRATE_SYNC_NO_COPY:
1385 			break;
1386 		default:
1387 			goto out;
1388 		}
1389 		folio_lock(src);
1390 	}
1391 
1392 	/*
1393 	 * Check for pages which are in the process of being freed.  Without
1394 	 * folio_mapping() set, hugetlbfs specific move page routine will not
1395 	 * be called and we could leak usage counts for subpools.
1396 	 */
1397 	if (hugetlb_folio_subpool(src) && !folio_mapping(src)) {
1398 		rc = -EBUSY;
1399 		goto out_unlock;
1400 	}
1401 
1402 	if (folio_test_anon(src))
1403 		anon_vma = folio_get_anon_vma(src);
1404 
1405 	if (unlikely(!folio_trylock(dst)))
1406 		goto put_anon;
1407 
1408 	if (folio_mapped(src)) {
1409 		enum ttu_flags ttu = 0;
1410 
1411 		if (!folio_test_anon(src)) {
1412 			/*
1413 			 * In shared mappings, try_to_unmap could potentially
1414 			 * call huge_pmd_unshare.  Because of this, take
1415 			 * semaphore in write mode here and set TTU_RMAP_LOCKED
1416 			 * to let lower levels know we have taken the lock.
1417 			 */
1418 			mapping = hugetlb_page_mapping_lock_write(hpage);
1419 			if (unlikely(!mapping))
1420 				goto unlock_put_anon;
1421 
1422 			ttu = TTU_RMAP_LOCKED;
1423 		}
1424 
1425 		try_to_migrate(src, ttu);
1426 		page_was_mapped = 1;
1427 
1428 		if (ttu & TTU_RMAP_LOCKED)
1429 			i_mmap_unlock_write(mapping);
1430 	}
1431 
1432 	if (!folio_mapped(src))
1433 		rc = move_to_new_folio(dst, src, mode);
1434 
1435 	if (page_was_mapped)
1436 		remove_migration_ptes(src,
1437 			rc == MIGRATEPAGE_SUCCESS ? dst : src, false);
1438 
1439 unlock_put_anon:
1440 	folio_unlock(dst);
1441 
1442 put_anon:
1443 	if (anon_vma)
1444 		put_anon_vma(anon_vma);
1445 
1446 	if (rc == MIGRATEPAGE_SUCCESS) {
1447 		move_hugetlb_state(src, dst, reason);
1448 		put_new_page = NULL;
1449 	}
1450 
1451 out_unlock:
1452 	folio_unlock(src);
1453 out:
1454 	if (rc == MIGRATEPAGE_SUCCESS)
1455 		folio_putback_active_hugetlb(src);
1456 	else if (rc != -EAGAIN)
1457 		list_move_tail(&src->lru, ret);
1458 
1459 	/*
1460 	 * If migration was not successful and there's a freeing callback, use
1461 	 * it.  Otherwise, put_page() will drop the reference grabbed during
1462 	 * isolation.
1463 	 */
1464 	if (put_new_page)
1465 		put_new_page(new_hpage, private);
1466 	else
1467 		folio_putback_active_hugetlb(dst);
1468 
1469 	return rc;
1470 }
1471 
1472 static inline int try_split_folio(struct folio *folio, struct list_head *split_folios)
1473 {
1474 	int rc;
1475 
1476 	folio_lock(folio);
1477 	rc = split_folio_to_list(folio, split_folios);
1478 	folio_unlock(folio);
1479 	if (!rc)
1480 		list_move_tail(&folio->lru, split_folios);
1481 
1482 	return rc;
1483 }
1484 
1485 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1486 #define NR_MAX_BATCHED_MIGRATION	HPAGE_PMD_NR
1487 #else
1488 #define NR_MAX_BATCHED_MIGRATION	512
1489 #endif
1490 #define NR_MAX_MIGRATE_PAGES_RETRY	10
1491 #define NR_MAX_MIGRATE_ASYNC_RETRY	3
1492 #define NR_MAX_MIGRATE_SYNC_RETRY					\
1493 	(NR_MAX_MIGRATE_PAGES_RETRY - NR_MAX_MIGRATE_ASYNC_RETRY)
1494 
1495 struct migrate_pages_stats {
1496 	int nr_succeeded;	/* Normal and large folios migrated successfully, in
1497 				   units of base pages */
1498 	int nr_failed_pages;	/* Normal and large folios failed to be migrated, in
1499 				   units of base pages.  Untried folios aren't counted */
1500 	int nr_thp_succeeded;	/* THP migrated successfully */
1501 	int nr_thp_failed;	/* THP failed to be migrated */
1502 	int nr_thp_split;	/* THP split before migrating */
1503 };
1504 
1505 /*
1506  * Returns the number of hugetlb folios that were not migrated, or an error code
1507  * after NR_MAX_MIGRATE_PAGES_RETRY attempts or if no hugetlb folios are movable
1508  * any more because the list has become empty or no retryable hugetlb folios
1509  * exist any more. It is caller's responsibility to call putback_movable_pages()
1510  * only if ret != 0.
1511  */
1512 static int migrate_hugetlbs(struct list_head *from, new_page_t get_new_page,
1513 			    free_page_t put_new_page, unsigned long private,
1514 			    enum migrate_mode mode, int reason,
1515 			    struct migrate_pages_stats *stats,
1516 			    struct list_head *ret_folios)
1517 {
1518 	int retry = 1;
1519 	int nr_failed = 0;
1520 	int nr_retry_pages = 0;
1521 	int pass = 0;
1522 	struct folio *folio, *folio2;
1523 	int rc, nr_pages;
1524 
1525 	for (pass = 0; pass < NR_MAX_MIGRATE_PAGES_RETRY && retry; pass++) {
1526 		retry = 0;
1527 		nr_retry_pages = 0;
1528 
1529 		list_for_each_entry_safe(folio, folio2, from, lru) {
1530 			if (!folio_test_hugetlb(folio))
1531 				continue;
1532 
1533 			nr_pages = folio_nr_pages(folio);
1534 
1535 			cond_resched();
1536 
1537 			/*
1538 			 * Migratability of hugepages depends on architectures and
1539 			 * their size.  This check is necessary because some callers
1540 			 * of hugepage migration like soft offline and memory
1541 			 * hotremove don't walk through page tables or check whether
1542 			 * the hugepage is pmd-based or not before kicking migration.
1543 			 */
1544 			if (!hugepage_migration_supported(folio_hstate(folio))) {
1545 				nr_failed++;
1546 				stats->nr_failed_pages += nr_pages;
1547 				list_move_tail(&folio->lru, ret_folios);
1548 				continue;
1549 			}
1550 
1551 			rc = unmap_and_move_huge_page(get_new_page,
1552 						      put_new_page, private,
1553 						      &folio->page, pass > 2, mode,
1554 						      reason, ret_folios);
1555 			/*
1556 			 * The rules are:
1557 			 *	Success: hugetlb folio will be put back
1558 			 *	-EAGAIN: stay on the from list
1559 			 *	-ENOMEM: stay on the from list
1560 			 *	Other errno: put on ret_folios list
1561 			 */
1562 			switch(rc) {
1563 			case -ENOMEM:
1564 				/*
1565 				 * When memory is low, don't bother to try to migrate
1566 				 * other folios, just exit.
1567 				 */
1568 				stats->nr_failed_pages += nr_pages + nr_retry_pages;
1569 				return -ENOMEM;
1570 			case -EAGAIN:
1571 				retry++;
1572 				nr_retry_pages += nr_pages;
1573 				break;
1574 			case MIGRATEPAGE_SUCCESS:
1575 				stats->nr_succeeded += nr_pages;
1576 				break;
1577 			default:
1578 				/*
1579 				 * Permanent failure (-EBUSY, etc.):
1580 				 * unlike -EAGAIN case, the failed folio is
1581 				 * removed from migration folio list and not
1582 				 * retried in the next outer loop.
1583 				 */
1584 				nr_failed++;
1585 				stats->nr_failed_pages += nr_pages;
1586 				break;
1587 			}
1588 		}
1589 	}
1590 	/*
1591 	 * nr_failed is number of hugetlb folios failed to be migrated.  After
1592 	 * NR_MAX_MIGRATE_PAGES_RETRY attempts, give up and count retried hugetlb
1593 	 * folios as failed.
1594 	 */
1595 	nr_failed += retry;
1596 	stats->nr_failed_pages += nr_retry_pages;
1597 
1598 	return nr_failed;
1599 }
1600 
1601 /*
1602  * migrate_pages_batch() first unmaps folios in the from list as many as
1603  * possible, then move the unmapped folios.
1604  *
1605  * We only batch migration if mode == MIGRATE_ASYNC to avoid to wait a
1606  * lock or bit when we have locked more than one folio.  Which may cause
1607  * deadlock (e.g., for loop device).  So, if mode != MIGRATE_ASYNC, the
1608  * length of the from list must be <= 1.
1609  */
1610 static int migrate_pages_batch(struct list_head *from, new_page_t get_new_page,
1611 		free_page_t put_new_page, unsigned long private,
1612 		enum migrate_mode mode, int reason, struct list_head *ret_folios,
1613 		struct list_head *split_folios, struct migrate_pages_stats *stats,
1614 		int nr_pass)
1615 {
1616 	int retry = 1;
1617 	int large_retry = 1;
1618 	int thp_retry = 1;
1619 	int nr_failed = 0;
1620 	int nr_retry_pages = 0;
1621 	int nr_large_failed = 0;
1622 	int pass = 0;
1623 	bool is_large = false;
1624 	bool is_thp = false;
1625 	struct folio *folio, *folio2, *dst = NULL, *dst2;
1626 	int rc, rc_saved = 0, nr_pages;
1627 	LIST_HEAD(unmap_folios);
1628 	LIST_HEAD(dst_folios);
1629 	bool nosplit = (reason == MR_NUMA_MISPLACED);
1630 
1631 	VM_WARN_ON_ONCE(mode != MIGRATE_ASYNC &&
1632 			!list_empty(from) && !list_is_singular(from));
1633 
1634 	for (pass = 0; pass < nr_pass && (retry || large_retry); pass++) {
1635 		retry = 0;
1636 		large_retry = 0;
1637 		thp_retry = 0;
1638 		nr_retry_pages = 0;
1639 
1640 		list_for_each_entry_safe(folio, folio2, from, lru) {
1641 			/*
1642 			 * Large folio statistics is based on the source large
1643 			 * folio. Capture required information that might get
1644 			 * lost during migration.
1645 			 */
1646 			is_large = folio_test_large(folio);
1647 			is_thp = is_large && folio_test_pmd_mappable(folio);
1648 			nr_pages = folio_nr_pages(folio);
1649 
1650 			cond_resched();
1651 
1652 			/*
1653 			 * Large folio migration might be unsupported or
1654 			 * the allocation might be failed so we should retry
1655 			 * on the same folio with the large folio split
1656 			 * to normal folios.
1657 			 *
1658 			 * Split folios are put in split_folios, and
1659 			 * we will migrate them after the rest of the
1660 			 * list is processed.
1661 			 */
1662 			if (!thp_migration_supported() && is_thp) {
1663 				nr_large_failed++;
1664 				stats->nr_thp_failed++;
1665 				if (!try_split_folio(folio, split_folios)) {
1666 					stats->nr_thp_split++;
1667 					continue;
1668 				}
1669 				stats->nr_failed_pages += nr_pages;
1670 				list_move_tail(&folio->lru, ret_folios);
1671 				continue;
1672 			}
1673 
1674 			rc = migrate_folio_unmap(get_new_page, put_new_page, private,
1675 						 folio, &dst, mode, reason, ret_folios);
1676 			/*
1677 			 * The rules are:
1678 			 *	Success: folio will be freed
1679 			 *	Unmap: folio will be put on unmap_folios list,
1680 			 *	       dst folio put on dst_folios list
1681 			 *	-EAGAIN: stay on the from list
1682 			 *	-ENOMEM: stay on the from list
1683 			 *	Other errno: put on ret_folios list
1684 			 */
1685 			switch(rc) {
1686 			case -ENOMEM:
1687 				/*
1688 				 * When memory is low, don't bother to try to migrate
1689 				 * other folios, move unmapped folios, then exit.
1690 				 */
1691 				if (is_large) {
1692 					nr_large_failed++;
1693 					stats->nr_thp_failed += is_thp;
1694 					/* Large folio NUMA faulting doesn't split to retry. */
1695 					if (!nosplit) {
1696 						int ret = try_split_folio(folio, split_folios);
1697 
1698 						if (!ret) {
1699 							stats->nr_thp_split += is_thp;
1700 							break;
1701 						} else if (reason == MR_LONGTERM_PIN &&
1702 							   ret == -EAGAIN) {
1703 							/*
1704 							 * Try again to split large folio to
1705 							 * mitigate the failure of longterm pinning.
1706 							 */
1707 							large_retry++;
1708 							thp_retry += is_thp;
1709 							nr_retry_pages += nr_pages;
1710 							/* Undo duplicated failure counting. */
1711 							nr_large_failed--;
1712 							stats->nr_thp_failed -= is_thp;
1713 							break;
1714 						}
1715 					}
1716 				} else {
1717 					nr_failed++;
1718 				}
1719 
1720 				stats->nr_failed_pages += nr_pages + nr_retry_pages;
1721 				/* nr_failed isn't updated for not used */
1722 				nr_large_failed += large_retry;
1723 				stats->nr_thp_failed += thp_retry;
1724 				rc_saved = rc;
1725 				if (list_empty(&unmap_folios))
1726 					goto out;
1727 				else
1728 					goto move;
1729 			case -EAGAIN:
1730 				if (is_large) {
1731 					large_retry++;
1732 					thp_retry += is_thp;
1733 				} else {
1734 					retry++;
1735 				}
1736 				nr_retry_pages += nr_pages;
1737 				break;
1738 			case MIGRATEPAGE_SUCCESS:
1739 				stats->nr_succeeded += nr_pages;
1740 				stats->nr_thp_succeeded += is_thp;
1741 				break;
1742 			case MIGRATEPAGE_UNMAP:
1743 				list_move_tail(&folio->lru, &unmap_folios);
1744 				list_add_tail(&dst->lru, &dst_folios);
1745 				break;
1746 			default:
1747 				/*
1748 				 * Permanent failure (-EBUSY, etc.):
1749 				 * unlike -EAGAIN case, the failed folio is
1750 				 * removed from migration folio list and not
1751 				 * retried in the next outer loop.
1752 				 */
1753 				if (is_large) {
1754 					nr_large_failed++;
1755 					stats->nr_thp_failed += is_thp;
1756 				} else {
1757 					nr_failed++;
1758 				}
1759 
1760 				stats->nr_failed_pages += nr_pages;
1761 				break;
1762 			}
1763 		}
1764 	}
1765 	nr_failed += retry;
1766 	nr_large_failed += large_retry;
1767 	stats->nr_thp_failed += thp_retry;
1768 	stats->nr_failed_pages += nr_retry_pages;
1769 move:
1770 	/* Flush TLBs for all unmapped folios */
1771 	try_to_unmap_flush();
1772 
1773 	retry = 1;
1774 	for (pass = 0; pass < nr_pass && (retry || large_retry); pass++) {
1775 		retry = 0;
1776 		large_retry = 0;
1777 		thp_retry = 0;
1778 		nr_retry_pages = 0;
1779 
1780 		dst = list_first_entry(&dst_folios, struct folio, lru);
1781 		dst2 = list_next_entry(dst, lru);
1782 		list_for_each_entry_safe(folio, folio2, &unmap_folios, lru) {
1783 			is_large = folio_test_large(folio);
1784 			is_thp = is_large && folio_test_pmd_mappable(folio);
1785 			nr_pages = folio_nr_pages(folio);
1786 
1787 			cond_resched();
1788 
1789 			rc = migrate_folio_move(put_new_page, private,
1790 						folio, dst, mode,
1791 						reason, ret_folios);
1792 			/*
1793 			 * The rules are:
1794 			 *	Success: folio will be freed
1795 			 *	-EAGAIN: stay on the unmap_folios list
1796 			 *	Other errno: put on ret_folios list
1797 			 */
1798 			switch(rc) {
1799 			case -EAGAIN:
1800 				if (is_large) {
1801 					large_retry++;
1802 					thp_retry += is_thp;
1803 				} else {
1804 					retry++;
1805 				}
1806 				nr_retry_pages += nr_pages;
1807 				break;
1808 			case MIGRATEPAGE_SUCCESS:
1809 				stats->nr_succeeded += nr_pages;
1810 				stats->nr_thp_succeeded += is_thp;
1811 				break;
1812 			default:
1813 				if (is_large) {
1814 					nr_large_failed++;
1815 					stats->nr_thp_failed += is_thp;
1816 				} else {
1817 					nr_failed++;
1818 				}
1819 
1820 				stats->nr_failed_pages += nr_pages;
1821 				break;
1822 			}
1823 			dst = dst2;
1824 			dst2 = list_next_entry(dst, lru);
1825 		}
1826 	}
1827 	nr_failed += retry;
1828 	nr_large_failed += large_retry;
1829 	stats->nr_thp_failed += thp_retry;
1830 	stats->nr_failed_pages += nr_retry_pages;
1831 
1832 	if (rc_saved)
1833 		rc = rc_saved;
1834 	else
1835 		rc = nr_failed + nr_large_failed;
1836 out:
1837 	/* Cleanup remaining folios */
1838 	dst = list_first_entry(&dst_folios, struct folio, lru);
1839 	dst2 = list_next_entry(dst, lru);
1840 	list_for_each_entry_safe(folio, folio2, &unmap_folios, lru) {
1841 		int page_was_mapped = 0;
1842 		struct anon_vma *anon_vma = NULL;
1843 
1844 		__migrate_folio_extract(dst, &page_was_mapped, &anon_vma);
1845 		migrate_folio_undo_src(folio, page_was_mapped, anon_vma,
1846 				       true, ret_folios);
1847 		list_del(&dst->lru);
1848 		migrate_folio_undo_dst(dst, true, put_new_page, private);
1849 		dst = dst2;
1850 		dst2 = list_next_entry(dst, lru);
1851 	}
1852 
1853 	return rc;
1854 }
1855 
1856 static int migrate_pages_sync(struct list_head *from, new_page_t get_new_page,
1857 		free_page_t put_new_page, unsigned long private,
1858 		enum migrate_mode mode, int reason, struct list_head *ret_folios,
1859 		struct list_head *split_folios, struct migrate_pages_stats *stats)
1860 {
1861 	int rc, nr_failed = 0;
1862 	LIST_HEAD(folios);
1863 	struct migrate_pages_stats astats;
1864 
1865 	memset(&astats, 0, sizeof(astats));
1866 	/* Try to migrate in batch with MIGRATE_ASYNC mode firstly */
1867 	rc = migrate_pages_batch(from, get_new_page, put_new_page, private, MIGRATE_ASYNC,
1868 				 reason, &folios, split_folios, &astats,
1869 				 NR_MAX_MIGRATE_ASYNC_RETRY);
1870 	stats->nr_succeeded += astats.nr_succeeded;
1871 	stats->nr_thp_succeeded += astats.nr_thp_succeeded;
1872 	stats->nr_thp_split += astats.nr_thp_split;
1873 	if (rc < 0) {
1874 		stats->nr_failed_pages += astats.nr_failed_pages;
1875 		stats->nr_thp_failed += astats.nr_thp_failed;
1876 		list_splice_tail(&folios, ret_folios);
1877 		return rc;
1878 	}
1879 	stats->nr_thp_failed += astats.nr_thp_split;
1880 	nr_failed += astats.nr_thp_split;
1881 	/*
1882 	 * Fall back to migrate all failed folios one by one synchronously. All
1883 	 * failed folios except split THPs will be retried, so their failure
1884 	 * isn't counted
1885 	 */
1886 	list_splice_tail_init(&folios, from);
1887 	while (!list_empty(from)) {
1888 		list_move(from->next, &folios);
1889 		rc = migrate_pages_batch(&folios, get_new_page, put_new_page,
1890 					 private, mode, reason, ret_folios,
1891 					 split_folios, stats, NR_MAX_MIGRATE_SYNC_RETRY);
1892 		list_splice_tail_init(&folios, ret_folios);
1893 		if (rc < 0)
1894 			return rc;
1895 		nr_failed += rc;
1896 	}
1897 
1898 	return nr_failed;
1899 }
1900 
1901 /*
1902  * migrate_pages - migrate the folios specified in a list, to the free folios
1903  *		   supplied as the target for the page migration
1904  *
1905  * @from:		The list of folios to be migrated.
1906  * @get_new_page:	The function used to allocate free folios to be used
1907  *			as the target of the folio migration.
1908  * @put_new_page:	The function used to free target folios if migration
1909  *			fails, or NULL if no special handling is necessary.
1910  * @private:		Private data to be passed on to get_new_page()
1911  * @mode:		The migration mode that specifies the constraints for
1912  *			folio migration, if any.
1913  * @reason:		The reason for folio migration.
1914  * @ret_succeeded:	Set to the number of folios migrated successfully if
1915  *			the caller passes a non-NULL pointer.
1916  *
1917  * The function returns after NR_MAX_MIGRATE_PAGES_RETRY attempts or if no folios
1918  * are movable any more because the list has become empty or no retryable folios
1919  * exist any more. It is caller's responsibility to call putback_movable_pages()
1920  * only if ret != 0.
1921  *
1922  * Returns the number of {normal folio, large folio, hugetlb} that were not
1923  * migrated, or an error code. The number of large folio splits will be
1924  * considered as the number of non-migrated large folio, no matter how many
1925  * split folios of the large folio are migrated successfully.
1926  */
1927 int migrate_pages(struct list_head *from, new_page_t get_new_page,
1928 		free_page_t put_new_page, unsigned long private,
1929 		enum migrate_mode mode, int reason, unsigned int *ret_succeeded)
1930 {
1931 	int rc, rc_gather;
1932 	int nr_pages;
1933 	struct folio *folio, *folio2;
1934 	LIST_HEAD(folios);
1935 	LIST_HEAD(ret_folios);
1936 	LIST_HEAD(split_folios);
1937 	struct migrate_pages_stats stats;
1938 
1939 	trace_mm_migrate_pages_start(mode, reason);
1940 
1941 	memset(&stats, 0, sizeof(stats));
1942 
1943 	rc_gather = migrate_hugetlbs(from, get_new_page, put_new_page, private,
1944 				     mode, reason, &stats, &ret_folios);
1945 	if (rc_gather < 0)
1946 		goto out;
1947 
1948 again:
1949 	nr_pages = 0;
1950 	list_for_each_entry_safe(folio, folio2, from, lru) {
1951 		/* Retried hugetlb folios will be kept in list  */
1952 		if (folio_test_hugetlb(folio)) {
1953 			list_move_tail(&folio->lru, &ret_folios);
1954 			continue;
1955 		}
1956 
1957 		nr_pages += folio_nr_pages(folio);
1958 		if (nr_pages >= NR_MAX_BATCHED_MIGRATION)
1959 			break;
1960 	}
1961 	if (nr_pages >= NR_MAX_BATCHED_MIGRATION)
1962 		list_cut_before(&folios, from, &folio2->lru);
1963 	else
1964 		list_splice_init(from, &folios);
1965 	if (mode == MIGRATE_ASYNC)
1966 		rc = migrate_pages_batch(&folios, get_new_page, put_new_page, private,
1967 					 mode, reason, &ret_folios, &split_folios, &stats,
1968 					 NR_MAX_MIGRATE_PAGES_RETRY);
1969 	else
1970 		rc = migrate_pages_sync(&folios, get_new_page, put_new_page, private,
1971 					mode, reason, &ret_folios, &split_folios, &stats);
1972 	list_splice_tail_init(&folios, &ret_folios);
1973 	if (rc < 0) {
1974 		rc_gather = rc;
1975 		list_splice_tail(&split_folios, &ret_folios);
1976 		goto out;
1977 	}
1978 	if (!list_empty(&split_folios)) {
1979 		/*
1980 		 * Failure isn't counted since all split folios of a large folio
1981 		 * is counted as 1 failure already.  And, we only try to migrate
1982 		 * with minimal effort, force MIGRATE_ASYNC mode and retry once.
1983 		 */
1984 		migrate_pages_batch(&split_folios, get_new_page, put_new_page, private,
1985 				    MIGRATE_ASYNC, reason, &ret_folios, NULL, &stats, 1);
1986 		list_splice_tail_init(&split_folios, &ret_folios);
1987 	}
1988 	rc_gather += rc;
1989 	if (!list_empty(from))
1990 		goto again;
1991 out:
1992 	/*
1993 	 * Put the permanent failure folio back to migration list, they
1994 	 * will be put back to the right list by the caller.
1995 	 */
1996 	list_splice(&ret_folios, from);
1997 
1998 	/*
1999 	 * Return 0 in case all split folios of fail-to-migrate large folios
2000 	 * are migrated successfully.
2001 	 */
2002 	if (list_empty(from))
2003 		rc_gather = 0;
2004 
2005 	count_vm_events(PGMIGRATE_SUCCESS, stats.nr_succeeded);
2006 	count_vm_events(PGMIGRATE_FAIL, stats.nr_failed_pages);
2007 	count_vm_events(THP_MIGRATION_SUCCESS, stats.nr_thp_succeeded);
2008 	count_vm_events(THP_MIGRATION_FAIL, stats.nr_thp_failed);
2009 	count_vm_events(THP_MIGRATION_SPLIT, stats.nr_thp_split);
2010 	trace_mm_migrate_pages(stats.nr_succeeded, stats.nr_failed_pages,
2011 			       stats.nr_thp_succeeded, stats.nr_thp_failed,
2012 			       stats.nr_thp_split, mode, reason);
2013 
2014 	if (ret_succeeded)
2015 		*ret_succeeded = stats.nr_succeeded;
2016 
2017 	return rc_gather;
2018 }
2019 
2020 struct page *alloc_migration_target(struct page *page, unsigned long private)
2021 {
2022 	struct folio *folio = page_folio(page);
2023 	struct migration_target_control *mtc;
2024 	gfp_t gfp_mask;
2025 	unsigned int order = 0;
2026 	struct folio *hugetlb_folio = NULL;
2027 	struct folio *new_folio = NULL;
2028 	int nid;
2029 	int zidx;
2030 
2031 	mtc = (struct migration_target_control *)private;
2032 	gfp_mask = mtc->gfp_mask;
2033 	nid = mtc->nid;
2034 	if (nid == NUMA_NO_NODE)
2035 		nid = folio_nid(folio);
2036 
2037 	if (folio_test_hugetlb(folio)) {
2038 		struct hstate *h = folio_hstate(folio);
2039 
2040 		gfp_mask = htlb_modify_alloc_mask(h, gfp_mask);
2041 		hugetlb_folio = alloc_hugetlb_folio_nodemask(h, nid,
2042 						mtc->nmask, gfp_mask);
2043 		return &hugetlb_folio->page;
2044 	}
2045 
2046 	if (folio_test_large(folio)) {
2047 		/*
2048 		 * clear __GFP_RECLAIM to make the migration callback
2049 		 * consistent with regular THP allocations.
2050 		 */
2051 		gfp_mask &= ~__GFP_RECLAIM;
2052 		gfp_mask |= GFP_TRANSHUGE;
2053 		order = folio_order(folio);
2054 	}
2055 	zidx = zone_idx(folio_zone(folio));
2056 	if (is_highmem_idx(zidx) || zidx == ZONE_MOVABLE)
2057 		gfp_mask |= __GFP_HIGHMEM;
2058 
2059 	new_folio = __folio_alloc(gfp_mask, order, nid, mtc->nmask);
2060 
2061 	return &new_folio->page;
2062 }
2063 
2064 #ifdef CONFIG_NUMA
2065 
2066 static int store_status(int __user *status, int start, int value, int nr)
2067 {
2068 	while (nr-- > 0) {
2069 		if (put_user(value, status + start))
2070 			return -EFAULT;
2071 		start++;
2072 	}
2073 
2074 	return 0;
2075 }
2076 
2077 static int do_move_pages_to_node(struct mm_struct *mm,
2078 		struct list_head *pagelist, int node)
2079 {
2080 	int err;
2081 	struct migration_target_control mtc = {
2082 		.nid = node,
2083 		.gfp_mask = GFP_HIGHUSER_MOVABLE | __GFP_THISNODE,
2084 	};
2085 
2086 	err = migrate_pages(pagelist, alloc_migration_target, NULL,
2087 		(unsigned long)&mtc, MIGRATE_SYNC, MR_SYSCALL, NULL);
2088 	if (err)
2089 		putback_movable_pages(pagelist);
2090 	return err;
2091 }
2092 
2093 /*
2094  * Resolves the given address to a struct page, isolates it from the LRU and
2095  * puts it to the given pagelist.
2096  * Returns:
2097  *     errno - if the page cannot be found/isolated
2098  *     0 - when it doesn't have to be migrated because it is already on the
2099  *         target node
2100  *     1 - when it has been queued
2101  */
2102 static int add_page_for_migration(struct mm_struct *mm, const void __user *p,
2103 		int node, struct list_head *pagelist, bool migrate_all)
2104 {
2105 	struct vm_area_struct *vma;
2106 	unsigned long addr;
2107 	struct page *page;
2108 	int err;
2109 	bool isolated;
2110 
2111 	mmap_read_lock(mm);
2112 	addr = (unsigned long)untagged_addr_remote(mm, p);
2113 
2114 	err = -EFAULT;
2115 	vma = vma_lookup(mm, addr);
2116 	if (!vma || !vma_migratable(vma))
2117 		goto out;
2118 
2119 	/* FOLL_DUMP to ignore special (like zero) pages */
2120 	page = follow_page(vma, addr, FOLL_GET | FOLL_DUMP);
2121 
2122 	err = PTR_ERR(page);
2123 	if (IS_ERR(page))
2124 		goto out;
2125 
2126 	err = -ENOENT;
2127 	if (!page)
2128 		goto out;
2129 
2130 	if (is_zone_device_page(page))
2131 		goto out_putpage;
2132 
2133 	err = 0;
2134 	if (page_to_nid(page) == node)
2135 		goto out_putpage;
2136 
2137 	err = -EACCES;
2138 	if (page_mapcount(page) > 1 && !migrate_all)
2139 		goto out_putpage;
2140 
2141 	if (PageHuge(page)) {
2142 		if (PageHead(page)) {
2143 			isolated = isolate_hugetlb(page_folio(page), pagelist);
2144 			err = isolated ? 1 : -EBUSY;
2145 		}
2146 	} else {
2147 		struct page *head;
2148 
2149 		head = compound_head(page);
2150 		isolated = isolate_lru_page(head);
2151 		if (!isolated) {
2152 			err = -EBUSY;
2153 			goto out_putpage;
2154 		}
2155 
2156 		err = 1;
2157 		list_add_tail(&head->lru, pagelist);
2158 		mod_node_page_state(page_pgdat(head),
2159 			NR_ISOLATED_ANON + page_is_file_lru(head),
2160 			thp_nr_pages(head));
2161 	}
2162 out_putpage:
2163 	/*
2164 	 * Either remove the duplicate refcount from
2165 	 * isolate_lru_page() or drop the page ref if it was
2166 	 * not isolated.
2167 	 */
2168 	put_page(page);
2169 out:
2170 	mmap_read_unlock(mm);
2171 	return err;
2172 }
2173 
2174 static int move_pages_and_store_status(struct mm_struct *mm, int node,
2175 		struct list_head *pagelist, int __user *status,
2176 		int start, int i, unsigned long nr_pages)
2177 {
2178 	int err;
2179 
2180 	if (list_empty(pagelist))
2181 		return 0;
2182 
2183 	err = do_move_pages_to_node(mm, pagelist, node);
2184 	if (err) {
2185 		/*
2186 		 * Positive err means the number of failed
2187 		 * pages to migrate.  Since we are going to
2188 		 * abort and return the number of non-migrated
2189 		 * pages, so need to include the rest of the
2190 		 * nr_pages that have not been attempted as
2191 		 * well.
2192 		 */
2193 		if (err > 0)
2194 			err += nr_pages - i;
2195 		return err;
2196 	}
2197 	return store_status(status, start, node, i - start);
2198 }
2199 
2200 /*
2201  * Migrate an array of page address onto an array of nodes and fill
2202  * the corresponding array of status.
2203  */
2204 static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes,
2205 			 unsigned long nr_pages,
2206 			 const void __user * __user *pages,
2207 			 const int __user *nodes,
2208 			 int __user *status, int flags)
2209 {
2210 	int current_node = NUMA_NO_NODE;
2211 	LIST_HEAD(pagelist);
2212 	int start, i;
2213 	int err = 0, err1;
2214 
2215 	lru_cache_disable();
2216 
2217 	for (i = start = 0; i < nr_pages; i++) {
2218 		const void __user *p;
2219 		int node;
2220 
2221 		err = -EFAULT;
2222 		if (get_user(p, pages + i))
2223 			goto out_flush;
2224 		if (get_user(node, nodes + i))
2225 			goto out_flush;
2226 
2227 		err = -ENODEV;
2228 		if (node < 0 || node >= MAX_NUMNODES)
2229 			goto out_flush;
2230 		if (!node_state(node, N_MEMORY))
2231 			goto out_flush;
2232 
2233 		err = -EACCES;
2234 		if (!node_isset(node, task_nodes))
2235 			goto out_flush;
2236 
2237 		if (current_node == NUMA_NO_NODE) {
2238 			current_node = node;
2239 			start = i;
2240 		} else if (node != current_node) {
2241 			err = move_pages_and_store_status(mm, current_node,
2242 					&pagelist, status, start, i, nr_pages);
2243 			if (err)
2244 				goto out;
2245 			start = i;
2246 			current_node = node;
2247 		}
2248 
2249 		/*
2250 		 * Errors in the page lookup or isolation are not fatal and we simply
2251 		 * report them via status
2252 		 */
2253 		err = add_page_for_migration(mm, p, current_node, &pagelist,
2254 					     flags & MPOL_MF_MOVE_ALL);
2255 
2256 		if (err > 0) {
2257 			/* The page is successfully queued for migration */
2258 			continue;
2259 		}
2260 
2261 		/*
2262 		 * The move_pages() man page does not have an -EEXIST choice, so
2263 		 * use -EFAULT instead.
2264 		 */
2265 		if (err == -EEXIST)
2266 			err = -EFAULT;
2267 
2268 		/*
2269 		 * If the page is already on the target node (!err), store the
2270 		 * node, otherwise, store the err.
2271 		 */
2272 		err = store_status(status, i, err ? : current_node, 1);
2273 		if (err)
2274 			goto out_flush;
2275 
2276 		err = move_pages_and_store_status(mm, current_node, &pagelist,
2277 				status, start, i, nr_pages);
2278 		if (err) {
2279 			/* We have accounted for page i */
2280 			if (err > 0)
2281 				err--;
2282 			goto out;
2283 		}
2284 		current_node = NUMA_NO_NODE;
2285 	}
2286 out_flush:
2287 	/* Make sure we do not overwrite the existing error */
2288 	err1 = move_pages_and_store_status(mm, current_node, &pagelist,
2289 				status, start, i, nr_pages);
2290 	if (err >= 0)
2291 		err = err1;
2292 out:
2293 	lru_cache_enable();
2294 	return err;
2295 }
2296 
2297 /*
2298  * Determine the nodes of an array of pages and store it in an array of status.
2299  */
2300 static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages,
2301 				const void __user **pages, int *status)
2302 {
2303 	unsigned long i;
2304 
2305 	mmap_read_lock(mm);
2306 
2307 	for (i = 0; i < nr_pages; i++) {
2308 		unsigned long addr = (unsigned long)(*pages);
2309 		struct vm_area_struct *vma;
2310 		struct page *page;
2311 		int err = -EFAULT;
2312 
2313 		vma = vma_lookup(mm, addr);
2314 		if (!vma)
2315 			goto set_status;
2316 
2317 		/* FOLL_DUMP to ignore special (like zero) pages */
2318 		page = follow_page(vma, addr, FOLL_GET | FOLL_DUMP);
2319 
2320 		err = PTR_ERR(page);
2321 		if (IS_ERR(page))
2322 			goto set_status;
2323 
2324 		err = -ENOENT;
2325 		if (!page)
2326 			goto set_status;
2327 
2328 		if (!is_zone_device_page(page))
2329 			err = page_to_nid(page);
2330 
2331 		put_page(page);
2332 set_status:
2333 		*status = err;
2334 
2335 		pages++;
2336 		status++;
2337 	}
2338 
2339 	mmap_read_unlock(mm);
2340 }
2341 
2342 static int get_compat_pages_array(const void __user *chunk_pages[],
2343 				  const void __user * __user *pages,
2344 				  unsigned long chunk_nr)
2345 {
2346 	compat_uptr_t __user *pages32 = (compat_uptr_t __user *)pages;
2347 	compat_uptr_t p;
2348 	int i;
2349 
2350 	for (i = 0; i < chunk_nr; i++) {
2351 		if (get_user(p, pages32 + i))
2352 			return -EFAULT;
2353 		chunk_pages[i] = compat_ptr(p);
2354 	}
2355 
2356 	return 0;
2357 }
2358 
2359 /*
2360  * Determine the nodes of a user array of pages and store it in
2361  * a user array of status.
2362  */
2363 static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages,
2364 			 const void __user * __user *pages,
2365 			 int __user *status)
2366 {
2367 #define DO_PAGES_STAT_CHUNK_NR 16UL
2368 	const void __user *chunk_pages[DO_PAGES_STAT_CHUNK_NR];
2369 	int chunk_status[DO_PAGES_STAT_CHUNK_NR];
2370 
2371 	while (nr_pages) {
2372 		unsigned long chunk_nr = min(nr_pages, DO_PAGES_STAT_CHUNK_NR);
2373 
2374 		if (in_compat_syscall()) {
2375 			if (get_compat_pages_array(chunk_pages, pages,
2376 						   chunk_nr))
2377 				break;
2378 		} else {
2379 			if (copy_from_user(chunk_pages, pages,
2380 				      chunk_nr * sizeof(*chunk_pages)))
2381 				break;
2382 		}
2383 
2384 		do_pages_stat_array(mm, chunk_nr, chunk_pages, chunk_status);
2385 
2386 		if (copy_to_user(status, chunk_status, chunk_nr * sizeof(*status)))
2387 			break;
2388 
2389 		pages += chunk_nr;
2390 		status += chunk_nr;
2391 		nr_pages -= chunk_nr;
2392 	}
2393 	return nr_pages ? -EFAULT : 0;
2394 }
2395 
2396 static struct mm_struct *find_mm_struct(pid_t pid, nodemask_t *mem_nodes)
2397 {
2398 	struct task_struct *task;
2399 	struct mm_struct *mm;
2400 
2401 	/*
2402 	 * There is no need to check if current process has the right to modify
2403 	 * the specified process when they are same.
2404 	 */
2405 	if (!pid) {
2406 		mmget(current->mm);
2407 		*mem_nodes = cpuset_mems_allowed(current);
2408 		return current->mm;
2409 	}
2410 
2411 	/* Find the mm_struct */
2412 	rcu_read_lock();
2413 	task = find_task_by_vpid(pid);
2414 	if (!task) {
2415 		rcu_read_unlock();
2416 		return ERR_PTR(-ESRCH);
2417 	}
2418 	get_task_struct(task);
2419 
2420 	/*
2421 	 * Check if this process has the right to modify the specified
2422 	 * process. Use the regular "ptrace_may_access()" checks.
2423 	 */
2424 	if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) {
2425 		rcu_read_unlock();
2426 		mm = ERR_PTR(-EPERM);
2427 		goto out;
2428 	}
2429 	rcu_read_unlock();
2430 
2431 	mm = ERR_PTR(security_task_movememory(task));
2432 	if (IS_ERR(mm))
2433 		goto out;
2434 	*mem_nodes = cpuset_mems_allowed(task);
2435 	mm = get_task_mm(task);
2436 out:
2437 	put_task_struct(task);
2438 	if (!mm)
2439 		mm = ERR_PTR(-EINVAL);
2440 	return mm;
2441 }
2442 
2443 /*
2444  * Move a list of pages in the address space of the currently executing
2445  * process.
2446  */
2447 static int kernel_move_pages(pid_t pid, unsigned long nr_pages,
2448 			     const void __user * __user *pages,
2449 			     const int __user *nodes,
2450 			     int __user *status, int flags)
2451 {
2452 	struct mm_struct *mm;
2453 	int err;
2454 	nodemask_t task_nodes;
2455 
2456 	/* Check flags */
2457 	if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL))
2458 		return -EINVAL;
2459 
2460 	if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
2461 		return -EPERM;
2462 
2463 	mm = find_mm_struct(pid, &task_nodes);
2464 	if (IS_ERR(mm))
2465 		return PTR_ERR(mm);
2466 
2467 	if (nodes)
2468 		err = do_pages_move(mm, task_nodes, nr_pages, pages,
2469 				    nodes, status, flags);
2470 	else
2471 		err = do_pages_stat(mm, nr_pages, pages, status);
2472 
2473 	mmput(mm);
2474 	return err;
2475 }
2476 
2477 SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages,
2478 		const void __user * __user *, pages,
2479 		const int __user *, nodes,
2480 		int __user *, status, int, flags)
2481 {
2482 	return kernel_move_pages(pid, nr_pages, pages, nodes, status, flags);
2483 }
2484 
2485 #ifdef CONFIG_NUMA_BALANCING
2486 /*
2487  * Returns true if this is a safe migration target node for misplaced NUMA
2488  * pages. Currently it only checks the watermarks which is crude.
2489  */
2490 static bool migrate_balanced_pgdat(struct pglist_data *pgdat,
2491 				   unsigned long nr_migrate_pages)
2492 {
2493 	int z;
2494 
2495 	for (z = pgdat->nr_zones - 1; z >= 0; z--) {
2496 		struct zone *zone = pgdat->node_zones + z;
2497 
2498 		if (!managed_zone(zone))
2499 			continue;
2500 
2501 		/* Avoid waking kswapd by allocating pages_to_migrate pages. */
2502 		if (!zone_watermark_ok(zone, 0,
2503 				       high_wmark_pages(zone) +
2504 				       nr_migrate_pages,
2505 				       ZONE_MOVABLE, 0))
2506 			continue;
2507 		return true;
2508 	}
2509 	return false;
2510 }
2511 
2512 static struct page *alloc_misplaced_dst_page(struct page *page,
2513 					   unsigned long data)
2514 {
2515 	int nid = (int) data;
2516 	int order = compound_order(page);
2517 	gfp_t gfp = __GFP_THISNODE;
2518 	struct folio *new;
2519 
2520 	if (order > 0)
2521 		gfp |= GFP_TRANSHUGE_LIGHT;
2522 	else {
2523 		gfp |= GFP_HIGHUSER_MOVABLE | __GFP_NOMEMALLOC | __GFP_NORETRY |
2524 			__GFP_NOWARN;
2525 		gfp &= ~__GFP_RECLAIM;
2526 	}
2527 	new = __folio_alloc_node(gfp, order, nid);
2528 
2529 	return &new->page;
2530 }
2531 
2532 static int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page)
2533 {
2534 	int nr_pages = thp_nr_pages(page);
2535 	int order = compound_order(page);
2536 
2537 	VM_BUG_ON_PAGE(order && !PageTransHuge(page), page);
2538 
2539 	/* Do not migrate THP mapped by multiple processes */
2540 	if (PageTransHuge(page) && total_mapcount(page) > 1)
2541 		return 0;
2542 
2543 	/* Avoid migrating to a node that is nearly full */
2544 	if (!migrate_balanced_pgdat(pgdat, nr_pages)) {
2545 		int z;
2546 
2547 		if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING))
2548 			return 0;
2549 		for (z = pgdat->nr_zones - 1; z >= 0; z--) {
2550 			if (managed_zone(pgdat->node_zones + z))
2551 				break;
2552 		}
2553 		wakeup_kswapd(pgdat->node_zones + z, 0, order, ZONE_MOVABLE);
2554 		return 0;
2555 	}
2556 
2557 	if (!isolate_lru_page(page))
2558 		return 0;
2559 
2560 	mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + page_is_file_lru(page),
2561 			    nr_pages);
2562 
2563 	/*
2564 	 * Isolating the page has taken another reference, so the
2565 	 * caller's reference can be safely dropped without the page
2566 	 * disappearing underneath us during migration.
2567 	 */
2568 	put_page(page);
2569 	return 1;
2570 }
2571 
2572 /*
2573  * Attempt to migrate a misplaced page to the specified destination
2574  * node. Caller is expected to have an elevated reference count on
2575  * the page that will be dropped by this function before returning.
2576  */
2577 int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma,
2578 			   int node)
2579 {
2580 	pg_data_t *pgdat = NODE_DATA(node);
2581 	int isolated;
2582 	int nr_remaining;
2583 	unsigned int nr_succeeded;
2584 	LIST_HEAD(migratepages);
2585 	int nr_pages = thp_nr_pages(page);
2586 
2587 	/*
2588 	 * Don't migrate file pages that are mapped in multiple processes
2589 	 * with execute permissions as they are probably shared libraries.
2590 	 */
2591 	if (page_mapcount(page) != 1 && page_is_file_lru(page) &&
2592 	    (vma->vm_flags & VM_EXEC))
2593 		goto out;
2594 
2595 	/*
2596 	 * Also do not migrate dirty pages as not all filesystems can move
2597 	 * dirty pages in MIGRATE_ASYNC mode which is a waste of cycles.
2598 	 */
2599 	if (page_is_file_lru(page) && PageDirty(page))
2600 		goto out;
2601 
2602 	isolated = numamigrate_isolate_page(pgdat, page);
2603 	if (!isolated)
2604 		goto out;
2605 
2606 	list_add(&page->lru, &migratepages);
2607 	nr_remaining = migrate_pages(&migratepages, alloc_misplaced_dst_page,
2608 				     NULL, node, MIGRATE_ASYNC,
2609 				     MR_NUMA_MISPLACED, &nr_succeeded);
2610 	if (nr_remaining) {
2611 		if (!list_empty(&migratepages)) {
2612 			list_del(&page->lru);
2613 			mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON +
2614 					page_is_file_lru(page), -nr_pages);
2615 			putback_lru_page(page);
2616 		}
2617 		isolated = 0;
2618 	}
2619 	if (nr_succeeded) {
2620 		count_vm_numa_events(NUMA_PAGE_MIGRATE, nr_succeeded);
2621 		if (!node_is_toptier(page_to_nid(page)) && node_is_toptier(node))
2622 			mod_node_page_state(pgdat, PGPROMOTE_SUCCESS,
2623 					    nr_succeeded);
2624 	}
2625 	BUG_ON(!list_empty(&migratepages));
2626 	return isolated;
2627 
2628 out:
2629 	put_page(page);
2630 	return 0;
2631 }
2632 #endif /* CONFIG_NUMA_BALANCING */
2633 #endif /* CONFIG_NUMA */
2634