1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Memory Migration functionality - linux/mm/migrate.c 4 * 5 * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter 6 * 7 * Page migration was first developed in the context of the memory hotplug 8 * project. The main authors of the migration code are: 9 * 10 * IWAMOTO Toshihiro <iwamoto@valinux.co.jp> 11 * Hirokazu Takahashi <taka@valinux.co.jp> 12 * Dave Hansen <haveblue@us.ibm.com> 13 * Christoph Lameter 14 */ 15 16 #include <linux/migrate.h> 17 #include <linux/export.h> 18 #include <linux/swap.h> 19 #include <linux/swapops.h> 20 #include <linux/pagemap.h> 21 #include <linux/buffer_head.h> 22 #include <linux/mm_inline.h> 23 #include <linux/nsproxy.h> 24 #include <linux/pagevec.h> 25 #include <linux/ksm.h> 26 #include <linux/rmap.h> 27 #include <linux/topology.h> 28 #include <linux/cpu.h> 29 #include <linux/cpuset.h> 30 #include <linux/writeback.h> 31 #include <linux/mempolicy.h> 32 #include <linux/vmalloc.h> 33 #include <linux/security.h> 34 #include <linux/backing-dev.h> 35 #include <linux/compaction.h> 36 #include <linux/syscalls.h> 37 #include <linux/compat.h> 38 #include <linux/hugetlb.h> 39 #include <linux/hugetlb_cgroup.h> 40 #include <linux/gfp.h> 41 #include <linux/pfn_t.h> 42 #include <linux/memremap.h> 43 #include <linux/userfaultfd_k.h> 44 #include <linux/balloon_compaction.h> 45 #include <linux/page_idle.h> 46 #include <linux/page_owner.h> 47 #include <linux/sched/mm.h> 48 #include <linux/ptrace.h> 49 #include <linux/oom.h> 50 #include <linux/memory.h> 51 #include <linux/random.h> 52 #include <linux/sched/sysctl.h> 53 #include <linux/memory-tiers.h> 54 55 #include <asm/tlbflush.h> 56 57 #include <trace/events/migrate.h> 58 59 #include "internal.h" 60 61 bool isolate_movable_page(struct page *page, isolate_mode_t mode) 62 { 63 struct folio *folio = folio_get_nontail_page(page); 64 const struct movable_operations *mops; 65 66 /* 67 * Avoid burning cycles with pages that are yet under __free_pages(), 68 * or just got freed under us. 69 * 70 * In case we 'win' a race for a movable page being freed under us and 71 * raise its refcount preventing __free_pages() from doing its job 72 * the put_page() at the end of this block will take care of 73 * release this page, thus avoiding a nasty leakage. 74 */ 75 if (!folio) 76 goto out; 77 78 if (unlikely(folio_test_slab(folio))) 79 goto out_putfolio; 80 /* Pairs with smp_wmb() in slab freeing, e.g. SLUB's __free_slab() */ 81 smp_rmb(); 82 /* 83 * Check movable flag before taking the page lock because 84 * we use non-atomic bitops on newly allocated page flags so 85 * unconditionally grabbing the lock ruins page's owner side. 86 */ 87 if (unlikely(!__folio_test_movable(folio))) 88 goto out_putfolio; 89 /* Pairs with smp_wmb() in slab allocation, e.g. SLUB's alloc_slab_page() */ 90 smp_rmb(); 91 if (unlikely(folio_test_slab(folio))) 92 goto out_putfolio; 93 94 /* 95 * As movable pages are not isolated from LRU lists, concurrent 96 * compaction threads can race against page migration functions 97 * as well as race against the releasing a page. 98 * 99 * In order to avoid having an already isolated movable page 100 * being (wrongly) re-isolated while it is under migration, 101 * or to avoid attempting to isolate pages being released, 102 * lets be sure we have the page lock 103 * before proceeding with the movable page isolation steps. 104 */ 105 if (unlikely(!folio_trylock(folio))) 106 goto out_putfolio; 107 108 if (!folio_test_movable(folio) || folio_test_isolated(folio)) 109 goto out_no_isolated; 110 111 mops = folio_movable_ops(folio); 112 VM_BUG_ON_FOLIO(!mops, folio); 113 114 if (!mops->isolate_page(&folio->page, mode)) 115 goto out_no_isolated; 116 117 /* Driver shouldn't use PG_isolated bit of page->flags */ 118 WARN_ON_ONCE(folio_test_isolated(folio)); 119 folio_set_isolated(folio); 120 folio_unlock(folio); 121 122 return true; 123 124 out_no_isolated: 125 folio_unlock(folio); 126 out_putfolio: 127 folio_put(folio); 128 out: 129 return false; 130 } 131 132 static void putback_movable_folio(struct folio *folio) 133 { 134 const struct movable_operations *mops = folio_movable_ops(folio); 135 136 mops->putback_page(&folio->page); 137 folio_clear_isolated(folio); 138 } 139 140 /* 141 * Put previously isolated pages back onto the appropriate lists 142 * from where they were once taken off for compaction/migration. 143 * 144 * This function shall be used whenever the isolated pageset has been 145 * built from lru, balloon, hugetlbfs page. See isolate_migratepages_range() 146 * and isolate_hugetlb(). 147 */ 148 void putback_movable_pages(struct list_head *l) 149 { 150 struct folio *folio; 151 struct folio *folio2; 152 153 list_for_each_entry_safe(folio, folio2, l, lru) { 154 if (unlikely(folio_test_hugetlb(folio))) { 155 folio_putback_active_hugetlb(folio); 156 continue; 157 } 158 list_del(&folio->lru); 159 /* 160 * We isolated non-lru movable folio so here we can use 161 * __PageMovable because LRU folio's mapping cannot have 162 * PAGE_MAPPING_MOVABLE. 163 */ 164 if (unlikely(__folio_test_movable(folio))) { 165 VM_BUG_ON_FOLIO(!folio_test_isolated(folio), folio); 166 folio_lock(folio); 167 if (folio_test_movable(folio)) 168 putback_movable_folio(folio); 169 else 170 folio_clear_isolated(folio); 171 folio_unlock(folio); 172 folio_put(folio); 173 } else { 174 node_stat_mod_folio(folio, NR_ISOLATED_ANON + 175 folio_is_file_lru(folio), -folio_nr_pages(folio)); 176 folio_putback_lru(folio); 177 } 178 } 179 } 180 181 /* 182 * Restore a potential migration pte to a working pte entry 183 */ 184 static bool remove_migration_pte(struct folio *folio, 185 struct vm_area_struct *vma, unsigned long addr, void *old) 186 { 187 DEFINE_FOLIO_VMA_WALK(pvmw, old, vma, addr, PVMW_SYNC | PVMW_MIGRATION); 188 189 while (page_vma_mapped_walk(&pvmw)) { 190 rmap_t rmap_flags = RMAP_NONE; 191 pte_t pte; 192 swp_entry_t entry; 193 struct page *new; 194 unsigned long idx = 0; 195 196 /* pgoff is invalid for ksm pages, but they are never large */ 197 if (folio_test_large(folio) && !folio_test_hugetlb(folio)) 198 idx = linear_page_index(vma, pvmw.address) - pvmw.pgoff; 199 new = folio_page(folio, idx); 200 201 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 202 /* PMD-mapped THP migration entry */ 203 if (!pvmw.pte) { 204 VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) || 205 !folio_test_pmd_mappable(folio), folio); 206 remove_migration_pmd(&pvmw, new); 207 continue; 208 } 209 #endif 210 211 folio_get(folio); 212 pte = mk_pte(new, READ_ONCE(vma->vm_page_prot)); 213 if (pte_swp_soft_dirty(*pvmw.pte)) 214 pte = pte_mksoft_dirty(pte); 215 216 /* 217 * Recheck VMA as permissions can change since migration started 218 */ 219 entry = pte_to_swp_entry(*pvmw.pte); 220 if (!is_migration_entry_young(entry)) 221 pte = pte_mkold(pte); 222 if (folio_test_dirty(folio) && is_migration_entry_dirty(entry)) 223 pte = pte_mkdirty(pte); 224 if (is_writable_migration_entry(entry)) 225 pte = maybe_mkwrite(pte, vma); 226 else if (pte_swp_uffd_wp(*pvmw.pte)) 227 pte = pte_mkuffd_wp(pte); 228 229 if (folio_test_anon(folio) && !is_readable_migration_entry(entry)) 230 rmap_flags |= RMAP_EXCLUSIVE; 231 232 if (unlikely(is_device_private_page(new))) { 233 if (pte_write(pte)) 234 entry = make_writable_device_private_entry( 235 page_to_pfn(new)); 236 else 237 entry = make_readable_device_private_entry( 238 page_to_pfn(new)); 239 pte = swp_entry_to_pte(entry); 240 if (pte_swp_soft_dirty(*pvmw.pte)) 241 pte = pte_swp_mksoft_dirty(pte); 242 if (pte_swp_uffd_wp(*pvmw.pte)) 243 pte = pte_swp_mkuffd_wp(pte); 244 } 245 246 #ifdef CONFIG_HUGETLB_PAGE 247 if (folio_test_hugetlb(folio)) { 248 unsigned int shift = huge_page_shift(hstate_vma(vma)); 249 250 pte = arch_make_huge_pte(pte, shift, vma->vm_flags); 251 if (folio_test_anon(folio)) 252 hugepage_add_anon_rmap(new, vma, pvmw.address, 253 rmap_flags); 254 else 255 page_dup_file_rmap(new, true); 256 set_huge_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte); 257 } else 258 #endif 259 { 260 if (folio_test_anon(folio)) 261 page_add_anon_rmap(new, vma, pvmw.address, 262 rmap_flags); 263 else 264 page_add_file_rmap(new, vma, false); 265 set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte); 266 } 267 if (vma->vm_flags & VM_LOCKED) 268 mlock_drain_local(); 269 270 trace_remove_migration_pte(pvmw.address, pte_val(pte), 271 compound_order(new)); 272 273 /* No need to invalidate - it was non-present before */ 274 update_mmu_cache(vma, pvmw.address, pvmw.pte); 275 } 276 277 return true; 278 } 279 280 /* 281 * Get rid of all migration entries and replace them by 282 * references to the indicated page. 283 */ 284 void remove_migration_ptes(struct folio *src, struct folio *dst, bool locked) 285 { 286 struct rmap_walk_control rwc = { 287 .rmap_one = remove_migration_pte, 288 .arg = src, 289 }; 290 291 if (locked) 292 rmap_walk_locked(dst, &rwc); 293 else 294 rmap_walk(dst, &rwc); 295 } 296 297 /* 298 * Something used the pte of a page under migration. We need to 299 * get to the page and wait until migration is finished. 300 * When we return from this function the fault will be retried. 301 */ 302 void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep, 303 spinlock_t *ptl) 304 { 305 pte_t pte; 306 swp_entry_t entry; 307 308 spin_lock(ptl); 309 pte = *ptep; 310 if (!is_swap_pte(pte)) 311 goto out; 312 313 entry = pte_to_swp_entry(pte); 314 if (!is_migration_entry(entry)) 315 goto out; 316 317 migration_entry_wait_on_locked(entry, ptep, ptl); 318 return; 319 out: 320 pte_unmap_unlock(ptep, ptl); 321 } 322 323 void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, 324 unsigned long address) 325 { 326 spinlock_t *ptl = pte_lockptr(mm, pmd); 327 pte_t *ptep = pte_offset_map(pmd, address); 328 __migration_entry_wait(mm, ptep, ptl); 329 } 330 331 #ifdef CONFIG_HUGETLB_PAGE 332 /* 333 * The vma read lock must be held upon entry. Holding that lock prevents either 334 * the pte or the ptl from being freed. 335 * 336 * This function will release the vma lock before returning. 337 */ 338 void __migration_entry_wait_huge(struct vm_area_struct *vma, 339 pte_t *ptep, spinlock_t *ptl) 340 { 341 pte_t pte; 342 343 hugetlb_vma_assert_locked(vma); 344 spin_lock(ptl); 345 pte = huge_ptep_get(ptep); 346 347 if (unlikely(!is_hugetlb_entry_migration(pte))) { 348 spin_unlock(ptl); 349 hugetlb_vma_unlock_read(vma); 350 } else { 351 /* 352 * If migration entry existed, safe to release vma lock 353 * here because the pgtable page won't be freed without the 354 * pgtable lock released. See comment right above pgtable 355 * lock release in migration_entry_wait_on_locked(). 356 */ 357 hugetlb_vma_unlock_read(vma); 358 migration_entry_wait_on_locked(pte_to_swp_entry(pte), NULL, ptl); 359 } 360 } 361 362 void migration_entry_wait_huge(struct vm_area_struct *vma, pte_t *pte) 363 { 364 spinlock_t *ptl = huge_pte_lockptr(hstate_vma(vma), vma->vm_mm, pte); 365 366 __migration_entry_wait_huge(vma, pte, ptl); 367 } 368 #endif 369 370 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 371 void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd) 372 { 373 spinlock_t *ptl; 374 375 ptl = pmd_lock(mm, pmd); 376 if (!is_pmd_migration_entry(*pmd)) 377 goto unlock; 378 migration_entry_wait_on_locked(pmd_to_swp_entry(*pmd), NULL, ptl); 379 return; 380 unlock: 381 spin_unlock(ptl); 382 } 383 #endif 384 385 static int folio_expected_refs(struct address_space *mapping, 386 struct folio *folio) 387 { 388 int refs = 1; 389 if (!mapping) 390 return refs; 391 392 refs += folio_nr_pages(folio); 393 if (folio_test_private(folio)) 394 refs++; 395 396 return refs; 397 } 398 399 /* 400 * Replace the page in the mapping. 401 * 402 * The number of remaining references must be: 403 * 1 for anonymous pages without a mapping 404 * 2 for pages with a mapping 405 * 3 for pages with a mapping and PagePrivate/PagePrivate2 set. 406 */ 407 int folio_migrate_mapping(struct address_space *mapping, 408 struct folio *newfolio, struct folio *folio, int extra_count) 409 { 410 XA_STATE(xas, &mapping->i_pages, folio_index(folio)); 411 struct zone *oldzone, *newzone; 412 int dirty; 413 int expected_count = folio_expected_refs(mapping, folio) + extra_count; 414 long nr = folio_nr_pages(folio); 415 416 if (!mapping) { 417 /* Anonymous page without mapping */ 418 if (folio_ref_count(folio) != expected_count) 419 return -EAGAIN; 420 421 /* No turning back from here */ 422 newfolio->index = folio->index; 423 newfolio->mapping = folio->mapping; 424 if (folio_test_swapbacked(folio)) 425 __folio_set_swapbacked(newfolio); 426 427 return MIGRATEPAGE_SUCCESS; 428 } 429 430 oldzone = folio_zone(folio); 431 newzone = folio_zone(newfolio); 432 433 xas_lock_irq(&xas); 434 if (!folio_ref_freeze(folio, expected_count)) { 435 xas_unlock_irq(&xas); 436 return -EAGAIN; 437 } 438 439 /* 440 * Now we know that no one else is looking at the folio: 441 * no turning back from here. 442 */ 443 newfolio->index = folio->index; 444 newfolio->mapping = folio->mapping; 445 folio_ref_add(newfolio, nr); /* add cache reference */ 446 if (folio_test_swapbacked(folio)) { 447 __folio_set_swapbacked(newfolio); 448 if (folio_test_swapcache(folio)) { 449 folio_set_swapcache(newfolio); 450 newfolio->private = folio_get_private(folio); 451 } 452 } else { 453 VM_BUG_ON_FOLIO(folio_test_swapcache(folio), folio); 454 } 455 456 /* Move dirty while page refs frozen and newpage not yet exposed */ 457 dirty = folio_test_dirty(folio); 458 if (dirty) { 459 folio_clear_dirty(folio); 460 folio_set_dirty(newfolio); 461 } 462 463 xas_store(&xas, newfolio); 464 465 /* 466 * Drop cache reference from old page by unfreezing 467 * to one less reference. 468 * We know this isn't the last reference. 469 */ 470 folio_ref_unfreeze(folio, expected_count - nr); 471 472 xas_unlock(&xas); 473 /* Leave irq disabled to prevent preemption while updating stats */ 474 475 /* 476 * If moved to a different zone then also account 477 * the page for that zone. Other VM counters will be 478 * taken care of when we establish references to the 479 * new page and drop references to the old page. 480 * 481 * Note that anonymous pages are accounted for 482 * via NR_FILE_PAGES and NR_ANON_MAPPED if they 483 * are mapped to swap space. 484 */ 485 if (newzone != oldzone) { 486 struct lruvec *old_lruvec, *new_lruvec; 487 struct mem_cgroup *memcg; 488 489 memcg = folio_memcg(folio); 490 old_lruvec = mem_cgroup_lruvec(memcg, oldzone->zone_pgdat); 491 new_lruvec = mem_cgroup_lruvec(memcg, newzone->zone_pgdat); 492 493 __mod_lruvec_state(old_lruvec, NR_FILE_PAGES, -nr); 494 __mod_lruvec_state(new_lruvec, NR_FILE_PAGES, nr); 495 if (folio_test_swapbacked(folio) && !folio_test_swapcache(folio)) { 496 __mod_lruvec_state(old_lruvec, NR_SHMEM, -nr); 497 __mod_lruvec_state(new_lruvec, NR_SHMEM, nr); 498 } 499 #ifdef CONFIG_SWAP 500 if (folio_test_swapcache(folio)) { 501 __mod_lruvec_state(old_lruvec, NR_SWAPCACHE, -nr); 502 __mod_lruvec_state(new_lruvec, NR_SWAPCACHE, nr); 503 } 504 #endif 505 if (dirty && mapping_can_writeback(mapping)) { 506 __mod_lruvec_state(old_lruvec, NR_FILE_DIRTY, -nr); 507 __mod_zone_page_state(oldzone, NR_ZONE_WRITE_PENDING, -nr); 508 __mod_lruvec_state(new_lruvec, NR_FILE_DIRTY, nr); 509 __mod_zone_page_state(newzone, NR_ZONE_WRITE_PENDING, nr); 510 } 511 } 512 local_irq_enable(); 513 514 return MIGRATEPAGE_SUCCESS; 515 } 516 EXPORT_SYMBOL(folio_migrate_mapping); 517 518 /* 519 * The expected number of remaining references is the same as that 520 * of folio_migrate_mapping(). 521 */ 522 int migrate_huge_page_move_mapping(struct address_space *mapping, 523 struct folio *dst, struct folio *src) 524 { 525 XA_STATE(xas, &mapping->i_pages, folio_index(src)); 526 int expected_count; 527 528 xas_lock_irq(&xas); 529 expected_count = 2 + folio_has_private(src); 530 if (!folio_ref_freeze(src, expected_count)) { 531 xas_unlock_irq(&xas); 532 return -EAGAIN; 533 } 534 535 dst->index = src->index; 536 dst->mapping = src->mapping; 537 538 folio_get(dst); 539 540 xas_store(&xas, dst); 541 542 folio_ref_unfreeze(src, expected_count - 1); 543 544 xas_unlock_irq(&xas); 545 546 return MIGRATEPAGE_SUCCESS; 547 } 548 549 /* 550 * Copy the flags and some other ancillary information 551 */ 552 void folio_migrate_flags(struct folio *newfolio, struct folio *folio) 553 { 554 int cpupid; 555 556 if (folio_test_error(folio)) 557 folio_set_error(newfolio); 558 if (folio_test_referenced(folio)) 559 folio_set_referenced(newfolio); 560 if (folio_test_uptodate(folio)) 561 folio_mark_uptodate(newfolio); 562 if (folio_test_clear_active(folio)) { 563 VM_BUG_ON_FOLIO(folio_test_unevictable(folio), folio); 564 folio_set_active(newfolio); 565 } else if (folio_test_clear_unevictable(folio)) 566 folio_set_unevictable(newfolio); 567 if (folio_test_workingset(folio)) 568 folio_set_workingset(newfolio); 569 if (folio_test_checked(folio)) 570 folio_set_checked(newfolio); 571 /* 572 * PG_anon_exclusive (-> PG_mappedtodisk) is always migrated via 573 * migration entries. We can still have PG_anon_exclusive set on an 574 * effectively unmapped and unreferenced first sub-pages of an 575 * anonymous THP: we can simply copy it here via PG_mappedtodisk. 576 */ 577 if (folio_test_mappedtodisk(folio)) 578 folio_set_mappedtodisk(newfolio); 579 580 /* Move dirty on pages not done by folio_migrate_mapping() */ 581 if (folio_test_dirty(folio)) 582 folio_set_dirty(newfolio); 583 584 if (folio_test_young(folio)) 585 folio_set_young(newfolio); 586 if (folio_test_idle(folio)) 587 folio_set_idle(newfolio); 588 589 /* 590 * Copy NUMA information to the new page, to prevent over-eager 591 * future migrations of this same page. 592 */ 593 cpupid = page_cpupid_xchg_last(&folio->page, -1); 594 /* 595 * For memory tiering mode, when migrate between slow and fast 596 * memory node, reset cpupid, because that is used to record 597 * page access time in slow memory node. 598 */ 599 if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) { 600 bool f_toptier = node_is_toptier(page_to_nid(&folio->page)); 601 bool t_toptier = node_is_toptier(page_to_nid(&newfolio->page)); 602 603 if (f_toptier != t_toptier) 604 cpupid = -1; 605 } 606 page_cpupid_xchg_last(&newfolio->page, cpupid); 607 608 folio_migrate_ksm(newfolio, folio); 609 /* 610 * Please do not reorder this without considering how mm/ksm.c's 611 * get_ksm_page() depends upon ksm_migrate_page() and PageSwapCache(). 612 */ 613 if (folio_test_swapcache(folio)) 614 folio_clear_swapcache(folio); 615 folio_clear_private(folio); 616 617 /* page->private contains hugetlb specific flags */ 618 if (!folio_test_hugetlb(folio)) 619 folio->private = NULL; 620 621 /* 622 * If any waiters have accumulated on the new page then 623 * wake them up. 624 */ 625 if (folio_test_writeback(newfolio)) 626 folio_end_writeback(newfolio); 627 628 /* 629 * PG_readahead shares the same bit with PG_reclaim. The above 630 * end_page_writeback() may clear PG_readahead mistakenly, so set the 631 * bit after that. 632 */ 633 if (folio_test_readahead(folio)) 634 folio_set_readahead(newfolio); 635 636 folio_copy_owner(newfolio, folio); 637 638 if (!folio_test_hugetlb(folio)) 639 mem_cgroup_migrate(folio, newfolio); 640 } 641 EXPORT_SYMBOL(folio_migrate_flags); 642 643 void folio_migrate_copy(struct folio *newfolio, struct folio *folio) 644 { 645 folio_copy(newfolio, folio); 646 folio_migrate_flags(newfolio, folio); 647 } 648 EXPORT_SYMBOL(folio_migrate_copy); 649 650 /************************************************************ 651 * Migration functions 652 ***********************************************************/ 653 654 int migrate_folio_extra(struct address_space *mapping, struct folio *dst, 655 struct folio *src, enum migrate_mode mode, int extra_count) 656 { 657 int rc; 658 659 BUG_ON(folio_test_writeback(src)); /* Writeback must be complete */ 660 661 rc = folio_migrate_mapping(mapping, dst, src, extra_count); 662 663 if (rc != MIGRATEPAGE_SUCCESS) 664 return rc; 665 666 if (mode != MIGRATE_SYNC_NO_COPY) 667 folio_migrate_copy(dst, src); 668 else 669 folio_migrate_flags(dst, src); 670 return MIGRATEPAGE_SUCCESS; 671 } 672 673 /** 674 * migrate_folio() - Simple folio migration. 675 * @mapping: The address_space containing the folio. 676 * @dst: The folio to migrate the data to. 677 * @src: The folio containing the current data. 678 * @mode: How to migrate the page. 679 * 680 * Common logic to directly migrate a single LRU folio suitable for 681 * folios that do not use PagePrivate/PagePrivate2. 682 * 683 * Folios are locked upon entry and exit. 684 */ 685 int migrate_folio(struct address_space *mapping, struct folio *dst, 686 struct folio *src, enum migrate_mode mode) 687 { 688 return migrate_folio_extra(mapping, dst, src, mode, 0); 689 } 690 EXPORT_SYMBOL(migrate_folio); 691 692 #ifdef CONFIG_BLOCK 693 /* Returns true if all buffers are successfully locked */ 694 static bool buffer_migrate_lock_buffers(struct buffer_head *head, 695 enum migrate_mode mode) 696 { 697 struct buffer_head *bh = head; 698 699 /* Simple case, sync compaction */ 700 if (mode != MIGRATE_ASYNC) { 701 do { 702 lock_buffer(bh); 703 bh = bh->b_this_page; 704 705 } while (bh != head); 706 707 return true; 708 } 709 710 /* async case, we cannot block on lock_buffer so use trylock_buffer */ 711 do { 712 if (!trylock_buffer(bh)) { 713 /* 714 * We failed to lock the buffer and cannot stall in 715 * async migration. Release the taken locks 716 */ 717 struct buffer_head *failed_bh = bh; 718 bh = head; 719 while (bh != failed_bh) { 720 unlock_buffer(bh); 721 bh = bh->b_this_page; 722 } 723 return false; 724 } 725 726 bh = bh->b_this_page; 727 } while (bh != head); 728 return true; 729 } 730 731 static int __buffer_migrate_folio(struct address_space *mapping, 732 struct folio *dst, struct folio *src, enum migrate_mode mode, 733 bool check_refs) 734 { 735 struct buffer_head *bh, *head; 736 int rc; 737 int expected_count; 738 739 head = folio_buffers(src); 740 if (!head) 741 return migrate_folio(mapping, dst, src, mode); 742 743 /* Check whether page does not have extra refs before we do more work */ 744 expected_count = folio_expected_refs(mapping, src); 745 if (folio_ref_count(src) != expected_count) 746 return -EAGAIN; 747 748 if (!buffer_migrate_lock_buffers(head, mode)) 749 return -EAGAIN; 750 751 if (check_refs) { 752 bool busy; 753 bool invalidated = false; 754 755 recheck_buffers: 756 busy = false; 757 spin_lock(&mapping->private_lock); 758 bh = head; 759 do { 760 if (atomic_read(&bh->b_count)) { 761 busy = true; 762 break; 763 } 764 bh = bh->b_this_page; 765 } while (bh != head); 766 if (busy) { 767 if (invalidated) { 768 rc = -EAGAIN; 769 goto unlock_buffers; 770 } 771 spin_unlock(&mapping->private_lock); 772 invalidate_bh_lrus(); 773 invalidated = true; 774 goto recheck_buffers; 775 } 776 } 777 778 rc = folio_migrate_mapping(mapping, dst, src, 0); 779 if (rc != MIGRATEPAGE_SUCCESS) 780 goto unlock_buffers; 781 782 folio_attach_private(dst, folio_detach_private(src)); 783 784 bh = head; 785 do { 786 set_bh_page(bh, &dst->page, bh_offset(bh)); 787 bh = bh->b_this_page; 788 } while (bh != head); 789 790 if (mode != MIGRATE_SYNC_NO_COPY) 791 folio_migrate_copy(dst, src); 792 else 793 folio_migrate_flags(dst, src); 794 795 rc = MIGRATEPAGE_SUCCESS; 796 unlock_buffers: 797 if (check_refs) 798 spin_unlock(&mapping->private_lock); 799 bh = head; 800 do { 801 unlock_buffer(bh); 802 bh = bh->b_this_page; 803 } while (bh != head); 804 805 return rc; 806 } 807 808 /** 809 * buffer_migrate_folio() - Migration function for folios with buffers. 810 * @mapping: The address space containing @src. 811 * @dst: The folio to migrate to. 812 * @src: The folio to migrate from. 813 * @mode: How to migrate the folio. 814 * 815 * This function can only be used if the underlying filesystem guarantees 816 * that no other references to @src exist. For example attached buffer 817 * heads are accessed only under the folio lock. If your filesystem cannot 818 * provide this guarantee, buffer_migrate_folio_norefs() may be more 819 * appropriate. 820 * 821 * Return: 0 on success or a negative errno on failure. 822 */ 823 int buffer_migrate_folio(struct address_space *mapping, 824 struct folio *dst, struct folio *src, enum migrate_mode mode) 825 { 826 return __buffer_migrate_folio(mapping, dst, src, mode, false); 827 } 828 EXPORT_SYMBOL(buffer_migrate_folio); 829 830 /** 831 * buffer_migrate_folio_norefs() - Migration function for folios with buffers. 832 * @mapping: The address space containing @src. 833 * @dst: The folio to migrate to. 834 * @src: The folio to migrate from. 835 * @mode: How to migrate the folio. 836 * 837 * Like buffer_migrate_folio() except that this variant is more careful 838 * and checks that there are also no buffer head references. This function 839 * is the right one for mappings where buffer heads are directly looked 840 * up and referenced (such as block device mappings). 841 * 842 * Return: 0 on success or a negative errno on failure. 843 */ 844 int buffer_migrate_folio_norefs(struct address_space *mapping, 845 struct folio *dst, struct folio *src, enum migrate_mode mode) 846 { 847 return __buffer_migrate_folio(mapping, dst, src, mode, true); 848 } 849 EXPORT_SYMBOL_GPL(buffer_migrate_folio_norefs); 850 #endif 851 852 int filemap_migrate_folio(struct address_space *mapping, 853 struct folio *dst, struct folio *src, enum migrate_mode mode) 854 { 855 int ret; 856 857 ret = folio_migrate_mapping(mapping, dst, src, 0); 858 if (ret != MIGRATEPAGE_SUCCESS) 859 return ret; 860 861 if (folio_get_private(src)) 862 folio_attach_private(dst, folio_detach_private(src)); 863 864 if (mode != MIGRATE_SYNC_NO_COPY) 865 folio_migrate_copy(dst, src); 866 else 867 folio_migrate_flags(dst, src); 868 return MIGRATEPAGE_SUCCESS; 869 } 870 EXPORT_SYMBOL_GPL(filemap_migrate_folio); 871 872 /* 873 * Writeback a folio to clean the dirty state 874 */ 875 static int writeout(struct address_space *mapping, struct folio *folio) 876 { 877 struct writeback_control wbc = { 878 .sync_mode = WB_SYNC_NONE, 879 .nr_to_write = 1, 880 .range_start = 0, 881 .range_end = LLONG_MAX, 882 .for_reclaim = 1 883 }; 884 int rc; 885 886 if (!mapping->a_ops->writepage) 887 /* No write method for the address space */ 888 return -EINVAL; 889 890 if (!folio_clear_dirty_for_io(folio)) 891 /* Someone else already triggered a write */ 892 return -EAGAIN; 893 894 /* 895 * A dirty folio may imply that the underlying filesystem has 896 * the folio on some queue. So the folio must be clean for 897 * migration. Writeout may mean we lose the lock and the 898 * folio state is no longer what we checked for earlier. 899 * At this point we know that the migration attempt cannot 900 * be successful. 901 */ 902 remove_migration_ptes(folio, folio, false); 903 904 rc = mapping->a_ops->writepage(&folio->page, &wbc); 905 906 if (rc != AOP_WRITEPAGE_ACTIVATE) 907 /* unlocked. Relock */ 908 folio_lock(folio); 909 910 return (rc < 0) ? -EIO : -EAGAIN; 911 } 912 913 /* 914 * Default handling if a filesystem does not provide a migration function. 915 */ 916 static int fallback_migrate_folio(struct address_space *mapping, 917 struct folio *dst, struct folio *src, enum migrate_mode mode) 918 { 919 if (folio_test_dirty(src)) { 920 /* Only writeback folios in full synchronous migration */ 921 switch (mode) { 922 case MIGRATE_SYNC: 923 case MIGRATE_SYNC_NO_COPY: 924 break; 925 default: 926 return -EBUSY; 927 } 928 return writeout(mapping, src); 929 } 930 931 /* 932 * Buffers may be managed in a filesystem specific way. 933 * We must have no buffers or drop them. 934 */ 935 if (folio_test_private(src) && 936 !filemap_release_folio(src, GFP_KERNEL)) 937 return mode == MIGRATE_SYNC ? -EAGAIN : -EBUSY; 938 939 return migrate_folio(mapping, dst, src, mode); 940 } 941 942 /* 943 * Move a page to a newly allocated page 944 * The page is locked and all ptes have been successfully removed. 945 * 946 * The new page will have replaced the old page if this function 947 * is successful. 948 * 949 * Return value: 950 * < 0 - error code 951 * MIGRATEPAGE_SUCCESS - success 952 */ 953 static int move_to_new_folio(struct folio *dst, struct folio *src, 954 enum migrate_mode mode) 955 { 956 int rc = -EAGAIN; 957 bool is_lru = !__PageMovable(&src->page); 958 959 VM_BUG_ON_FOLIO(!folio_test_locked(src), src); 960 VM_BUG_ON_FOLIO(!folio_test_locked(dst), dst); 961 962 if (likely(is_lru)) { 963 struct address_space *mapping = folio_mapping(src); 964 965 if (!mapping) 966 rc = migrate_folio(mapping, dst, src, mode); 967 else if (mapping->a_ops->migrate_folio) 968 /* 969 * Most folios have a mapping and most filesystems 970 * provide a migrate_folio callback. Anonymous folios 971 * are part of swap space which also has its own 972 * migrate_folio callback. This is the most common path 973 * for page migration. 974 */ 975 rc = mapping->a_ops->migrate_folio(mapping, dst, src, 976 mode); 977 else 978 rc = fallback_migrate_folio(mapping, dst, src, mode); 979 } else { 980 const struct movable_operations *mops; 981 982 /* 983 * In case of non-lru page, it could be released after 984 * isolation step. In that case, we shouldn't try migration. 985 */ 986 VM_BUG_ON_FOLIO(!folio_test_isolated(src), src); 987 if (!folio_test_movable(src)) { 988 rc = MIGRATEPAGE_SUCCESS; 989 folio_clear_isolated(src); 990 goto out; 991 } 992 993 mops = folio_movable_ops(src); 994 rc = mops->migrate_page(&dst->page, &src->page, mode); 995 WARN_ON_ONCE(rc == MIGRATEPAGE_SUCCESS && 996 !folio_test_isolated(src)); 997 } 998 999 /* 1000 * When successful, old pagecache src->mapping must be cleared before 1001 * src is freed; but stats require that PageAnon be left as PageAnon. 1002 */ 1003 if (rc == MIGRATEPAGE_SUCCESS) { 1004 if (__PageMovable(&src->page)) { 1005 VM_BUG_ON_FOLIO(!folio_test_isolated(src), src); 1006 1007 /* 1008 * We clear PG_movable under page_lock so any compactor 1009 * cannot try to migrate this page. 1010 */ 1011 folio_clear_isolated(src); 1012 } 1013 1014 /* 1015 * Anonymous and movable src->mapping will be cleared by 1016 * free_pages_prepare so don't reset it here for keeping 1017 * the type to work PageAnon, for example. 1018 */ 1019 if (!folio_mapping_flags(src)) 1020 src->mapping = NULL; 1021 1022 if (likely(!folio_is_zone_device(dst))) 1023 flush_dcache_folio(dst); 1024 } 1025 out: 1026 return rc; 1027 } 1028 1029 /* 1030 * To record some information during migration, we use some unused 1031 * fields (mapping and private) of struct folio of the newly allocated 1032 * destination folio. This is safe because nobody is using them 1033 * except us. 1034 */ 1035 union migration_ptr { 1036 struct anon_vma *anon_vma; 1037 struct address_space *mapping; 1038 }; 1039 static void __migrate_folio_record(struct folio *dst, 1040 unsigned long page_was_mapped, 1041 struct anon_vma *anon_vma) 1042 { 1043 union migration_ptr ptr = { .anon_vma = anon_vma }; 1044 dst->mapping = ptr.mapping; 1045 dst->private = (void *)page_was_mapped; 1046 } 1047 1048 static void __migrate_folio_extract(struct folio *dst, 1049 int *page_was_mappedp, 1050 struct anon_vma **anon_vmap) 1051 { 1052 union migration_ptr ptr = { .mapping = dst->mapping }; 1053 *anon_vmap = ptr.anon_vma; 1054 *page_was_mappedp = (unsigned long)dst->private; 1055 dst->mapping = NULL; 1056 dst->private = NULL; 1057 } 1058 1059 /* Restore the source folio to the original state upon failure */ 1060 static void migrate_folio_undo_src(struct folio *src, 1061 int page_was_mapped, 1062 struct anon_vma *anon_vma, 1063 bool locked, 1064 struct list_head *ret) 1065 { 1066 if (page_was_mapped) 1067 remove_migration_ptes(src, src, false); 1068 /* Drop an anon_vma reference if we took one */ 1069 if (anon_vma) 1070 put_anon_vma(anon_vma); 1071 if (locked) 1072 folio_unlock(src); 1073 if (ret) 1074 list_move_tail(&src->lru, ret); 1075 } 1076 1077 /* Restore the destination folio to the original state upon failure */ 1078 static void migrate_folio_undo_dst(struct folio *dst, 1079 bool locked, 1080 free_page_t put_new_page, 1081 unsigned long private) 1082 { 1083 if (locked) 1084 folio_unlock(dst); 1085 if (put_new_page) 1086 put_new_page(&dst->page, private); 1087 else 1088 folio_put(dst); 1089 } 1090 1091 /* Cleanup src folio upon migration success */ 1092 static void migrate_folio_done(struct folio *src, 1093 enum migrate_reason reason) 1094 { 1095 /* 1096 * Compaction can migrate also non-LRU pages which are 1097 * not accounted to NR_ISOLATED_*. They can be recognized 1098 * as __PageMovable 1099 */ 1100 if (likely(!__folio_test_movable(src))) 1101 mod_node_page_state(folio_pgdat(src), NR_ISOLATED_ANON + 1102 folio_is_file_lru(src), -folio_nr_pages(src)); 1103 1104 if (reason != MR_MEMORY_FAILURE) 1105 /* We release the page in page_handle_poison. */ 1106 folio_put(src); 1107 } 1108 1109 /* Obtain the lock on page, remove all ptes. */ 1110 static int migrate_folio_unmap(new_page_t get_new_page, free_page_t put_new_page, 1111 unsigned long private, struct folio *src, 1112 struct folio **dstp, enum migrate_mode mode, 1113 enum migrate_reason reason, struct list_head *ret) 1114 { 1115 struct folio *dst; 1116 int rc = -EAGAIN; 1117 struct page *newpage = NULL; 1118 int page_was_mapped = 0; 1119 struct anon_vma *anon_vma = NULL; 1120 bool is_lru = !__PageMovable(&src->page); 1121 bool locked = false; 1122 bool dst_locked = false; 1123 1124 if (folio_ref_count(src) == 1) { 1125 /* Folio was freed from under us. So we are done. */ 1126 folio_clear_active(src); 1127 folio_clear_unevictable(src); 1128 /* free_pages_prepare() will clear PG_isolated. */ 1129 list_del(&src->lru); 1130 migrate_folio_done(src, reason); 1131 return MIGRATEPAGE_SUCCESS; 1132 } 1133 1134 newpage = get_new_page(&src->page, private); 1135 if (!newpage) 1136 return -ENOMEM; 1137 dst = page_folio(newpage); 1138 *dstp = dst; 1139 1140 dst->private = NULL; 1141 1142 if (!folio_trylock(src)) { 1143 if (mode == MIGRATE_ASYNC) 1144 goto out; 1145 1146 /* 1147 * It's not safe for direct compaction to call lock_page. 1148 * For example, during page readahead pages are added locked 1149 * to the LRU. Later, when the IO completes the pages are 1150 * marked uptodate and unlocked. However, the queueing 1151 * could be merging multiple pages for one bio (e.g. 1152 * mpage_readahead). If an allocation happens for the 1153 * second or third page, the process can end up locking 1154 * the same page twice and deadlocking. Rather than 1155 * trying to be clever about what pages can be locked, 1156 * avoid the use of lock_page for direct compaction 1157 * altogether. 1158 */ 1159 if (current->flags & PF_MEMALLOC) 1160 goto out; 1161 1162 folio_lock(src); 1163 } 1164 locked = true; 1165 1166 if (folio_test_writeback(src)) { 1167 /* 1168 * Only in the case of a full synchronous migration is it 1169 * necessary to wait for PageWriteback. In the async case, 1170 * the retry loop is too short and in the sync-light case, 1171 * the overhead of stalling is too much 1172 */ 1173 switch (mode) { 1174 case MIGRATE_SYNC: 1175 case MIGRATE_SYNC_NO_COPY: 1176 break; 1177 default: 1178 rc = -EBUSY; 1179 goto out; 1180 } 1181 folio_wait_writeback(src); 1182 } 1183 1184 /* 1185 * By try_to_migrate(), src->mapcount goes down to 0 here. In this case, 1186 * we cannot notice that anon_vma is freed while we migrate a page. 1187 * This get_anon_vma() delays freeing anon_vma pointer until the end 1188 * of migration. File cache pages are no problem because of page_lock() 1189 * File Caches may use write_page() or lock_page() in migration, then, 1190 * just care Anon page here. 1191 * 1192 * Only folio_get_anon_vma() understands the subtleties of 1193 * getting a hold on an anon_vma from outside one of its mms. 1194 * But if we cannot get anon_vma, then we won't need it anyway, 1195 * because that implies that the anon page is no longer mapped 1196 * (and cannot be remapped so long as we hold the page lock). 1197 */ 1198 if (folio_test_anon(src) && !folio_test_ksm(src)) 1199 anon_vma = folio_get_anon_vma(src); 1200 1201 /* 1202 * Block others from accessing the new page when we get around to 1203 * establishing additional references. We are usually the only one 1204 * holding a reference to dst at this point. We used to have a BUG 1205 * here if folio_trylock(dst) fails, but would like to allow for 1206 * cases where there might be a race with the previous use of dst. 1207 * This is much like races on refcount of oldpage: just don't BUG(). 1208 */ 1209 if (unlikely(!folio_trylock(dst))) 1210 goto out; 1211 dst_locked = true; 1212 1213 if (unlikely(!is_lru)) { 1214 __migrate_folio_record(dst, page_was_mapped, anon_vma); 1215 return MIGRATEPAGE_UNMAP; 1216 } 1217 1218 /* 1219 * Corner case handling: 1220 * 1. When a new swap-cache page is read into, it is added to the LRU 1221 * and treated as swapcache but it has no rmap yet. 1222 * Calling try_to_unmap() against a src->mapping==NULL page will 1223 * trigger a BUG. So handle it here. 1224 * 2. An orphaned page (see truncate_cleanup_page) might have 1225 * fs-private metadata. The page can be picked up due to memory 1226 * offlining. Everywhere else except page reclaim, the page is 1227 * invisible to the vm, so the page can not be migrated. So try to 1228 * free the metadata, so the page can be freed. 1229 */ 1230 if (!src->mapping) { 1231 if (folio_test_private(src)) { 1232 try_to_free_buffers(src); 1233 goto out; 1234 } 1235 } else if (folio_mapped(src)) { 1236 /* Establish migration ptes */ 1237 VM_BUG_ON_FOLIO(folio_test_anon(src) && 1238 !folio_test_ksm(src) && !anon_vma, src); 1239 try_to_migrate(src, mode == MIGRATE_ASYNC ? TTU_BATCH_FLUSH : 0); 1240 page_was_mapped = 1; 1241 } 1242 1243 if (!folio_mapped(src)) { 1244 __migrate_folio_record(dst, page_was_mapped, anon_vma); 1245 return MIGRATEPAGE_UNMAP; 1246 } 1247 1248 out: 1249 /* 1250 * A folio that has not been unmapped will be restored to 1251 * right list unless we want to retry. 1252 */ 1253 if (rc == -EAGAIN) 1254 ret = NULL; 1255 1256 migrate_folio_undo_src(src, page_was_mapped, anon_vma, locked, ret); 1257 migrate_folio_undo_dst(dst, dst_locked, put_new_page, private); 1258 1259 return rc; 1260 } 1261 1262 /* Migrate the folio to the newly allocated folio in dst. */ 1263 static int migrate_folio_move(free_page_t put_new_page, unsigned long private, 1264 struct folio *src, struct folio *dst, 1265 enum migrate_mode mode, enum migrate_reason reason, 1266 struct list_head *ret) 1267 { 1268 int rc; 1269 int page_was_mapped = 0; 1270 struct anon_vma *anon_vma = NULL; 1271 bool is_lru = !__PageMovable(&src->page); 1272 struct list_head *prev; 1273 1274 __migrate_folio_extract(dst, &page_was_mapped, &anon_vma); 1275 prev = dst->lru.prev; 1276 list_del(&dst->lru); 1277 1278 rc = move_to_new_folio(dst, src, mode); 1279 if (rc) 1280 goto out; 1281 1282 if (unlikely(!is_lru)) 1283 goto out_unlock_both; 1284 1285 /* 1286 * When successful, push dst to LRU immediately: so that if it 1287 * turns out to be an mlocked page, remove_migration_ptes() will 1288 * automatically build up the correct dst->mlock_count for it. 1289 * 1290 * We would like to do something similar for the old page, when 1291 * unsuccessful, and other cases when a page has been temporarily 1292 * isolated from the unevictable LRU: but this case is the easiest. 1293 */ 1294 folio_add_lru(dst); 1295 if (page_was_mapped) 1296 lru_add_drain(); 1297 1298 if (page_was_mapped) 1299 remove_migration_ptes(src, dst, false); 1300 1301 out_unlock_both: 1302 folio_unlock(dst); 1303 set_page_owner_migrate_reason(&dst->page, reason); 1304 /* 1305 * If migration is successful, decrease refcount of dst, 1306 * which will not free the page because new page owner increased 1307 * refcounter. 1308 */ 1309 folio_put(dst); 1310 1311 /* 1312 * A folio that has been migrated has all references removed 1313 * and will be freed. 1314 */ 1315 list_del(&src->lru); 1316 /* Drop an anon_vma reference if we took one */ 1317 if (anon_vma) 1318 put_anon_vma(anon_vma); 1319 folio_unlock(src); 1320 migrate_folio_done(src, reason); 1321 1322 return rc; 1323 out: 1324 /* 1325 * A folio that has not been migrated will be restored to 1326 * right list unless we want to retry. 1327 */ 1328 if (rc == -EAGAIN) { 1329 list_add(&dst->lru, prev); 1330 __migrate_folio_record(dst, page_was_mapped, anon_vma); 1331 return rc; 1332 } 1333 1334 migrate_folio_undo_src(src, page_was_mapped, anon_vma, true, ret); 1335 migrate_folio_undo_dst(dst, true, put_new_page, private); 1336 1337 return rc; 1338 } 1339 1340 /* 1341 * Counterpart of unmap_and_move_page() for hugepage migration. 1342 * 1343 * This function doesn't wait the completion of hugepage I/O 1344 * because there is no race between I/O and migration for hugepage. 1345 * Note that currently hugepage I/O occurs only in direct I/O 1346 * where no lock is held and PG_writeback is irrelevant, 1347 * and writeback status of all subpages are counted in the reference 1348 * count of the head page (i.e. if all subpages of a 2MB hugepage are 1349 * under direct I/O, the reference of the head page is 512 and a bit more.) 1350 * This means that when we try to migrate hugepage whose subpages are 1351 * doing direct I/O, some references remain after try_to_unmap() and 1352 * hugepage migration fails without data corruption. 1353 * 1354 * There is also no race when direct I/O is issued on the page under migration, 1355 * because then pte is replaced with migration swap entry and direct I/O code 1356 * will wait in the page fault for migration to complete. 1357 */ 1358 static int unmap_and_move_huge_page(new_page_t get_new_page, 1359 free_page_t put_new_page, unsigned long private, 1360 struct page *hpage, int force, 1361 enum migrate_mode mode, int reason, 1362 struct list_head *ret) 1363 { 1364 struct folio *dst, *src = page_folio(hpage); 1365 int rc = -EAGAIN; 1366 int page_was_mapped = 0; 1367 struct page *new_hpage; 1368 struct anon_vma *anon_vma = NULL; 1369 struct address_space *mapping = NULL; 1370 1371 if (folio_ref_count(src) == 1) { 1372 /* page was freed from under us. So we are done. */ 1373 folio_putback_active_hugetlb(src); 1374 return MIGRATEPAGE_SUCCESS; 1375 } 1376 1377 new_hpage = get_new_page(hpage, private); 1378 if (!new_hpage) 1379 return -ENOMEM; 1380 dst = page_folio(new_hpage); 1381 1382 if (!folio_trylock(src)) { 1383 if (!force) 1384 goto out; 1385 switch (mode) { 1386 case MIGRATE_SYNC: 1387 case MIGRATE_SYNC_NO_COPY: 1388 break; 1389 default: 1390 goto out; 1391 } 1392 folio_lock(src); 1393 } 1394 1395 /* 1396 * Check for pages which are in the process of being freed. Without 1397 * folio_mapping() set, hugetlbfs specific move page routine will not 1398 * be called and we could leak usage counts for subpools. 1399 */ 1400 if (hugetlb_folio_subpool(src) && !folio_mapping(src)) { 1401 rc = -EBUSY; 1402 goto out_unlock; 1403 } 1404 1405 if (folio_test_anon(src)) 1406 anon_vma = folio_get_anon_vma(src); 1407 1408 if (unlikely(!folio_trylock(dst))) 1409 goto put_anon; 1410 1411 if (folio_mapped(src)) { 1412 enum ttu_flags ttu = 0; 1413 1414 if (!folio_test_anon(src)) { 1415 /* 1416 * In shared mappings, try_to_unmap could potentially 1417 * call huge_pmd_unshare. Because of this, take 1418 * semaphore in write mode here and set TTU_RMAP_LOCKED 1419 * to let lower levels know we have taken the lock. 1420 */ 1421 mapping = hugetlb_page_mapping_lock_write(hpage); 1422 if (unlikely(!mapping)) 1423 goto unlock_put_anon; 1424 1425 ttu = TTU_RMAP_LOCKED; 1426 } 1427 1428 try_to_migrate(src, ttu); 1429 page_was_mapped = 1; 1430 1431 if (ttu & TTU_RMAP_LOCKED) 1432 i_mmap_unlock_write(mapping); 1433 } 1434 1435 if (!folio_mapped(src)) 1436 rc = move_to_new_folio(dst, src, mode); 1437 1438 if (page_was_mapped) 1439 remove_migration_ptes(src, 1440 rc == MIGRATEPAGE_SUCCESS ? dst : src, false); 1441 1442 unlock_put_anon: 1443 folio_unlock(dst); 1444 1445 put_anon: 1446 if (anon_vma) 1447 put_anon_vma(anon_vma); 1448 1449 if (rc == MIGRATEPAGE_SUCCESS) { 1450 move_hugetlb_state(src, dst, reason); 1451 put_new_page = NULL; 1452 } 1453 1454 out_unlock: 1455 folio_unlock(src); 1456 out: 1457 if (rc == MIGRATEPAGE_SUCCESS) 1458 folio_putback_active_hugetlb(src); 1459 else if (rc != -EAGAIN) 1460 list_move_tail(&src->lru, ret); 1461 1462 /* 1463 * If migration was not successful and there's a freeing callback, use 1464 * it. Otherwise, put_page() will drop the reference grabbed during 1465 * isolation. 1466 */ 1467 if (put_new_page) 1468 put_new_page(new_hpage, private); 1469 else 1470 folio_putback_active_hugetlb(dst); 1471 1472 return rc; 1473 } 1474 1475 static inline int try_split_folio(struct folio *folio, struct list_head *split_folios) 1476 { 1477 int rc; 1478 1479 folio_lock(folio); 1480 rc = split_folio_to_list(folio, split_folios); 1481 folio_unlock(folio); 1482 if (!rc) 1483 list_move_tail(&folio->lru, split_folios); 1484 1485 return rc; 1486 } 1487 1488 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1489 #define NR_MAX_BATCHED_MIGRATION HPAGE_PMD_NR 1490 #else 1491 #define NR_MAX_BATCHED_MIGRATION 512 1492 #endif 1493 #define NR_MAX_MIGRATE_PAGES_RETRY 10 1494 #define NR_MAX_MIGRATE_ASYNC_RETRY 3 1495 #define NR_MAX_MIGRATE_SYNC_RETRY \ 1496 (NR_MAX_MIGRATE_PAGES_RETRY - NR_MAX_MIGRATE_ASYNC_RETRY) 1497 1498 struct migrate_pages_stats { 1499 int nr_succeeded; /* Normal and large folios migrated successfully, in 1500 units of base pages */ 1501 int nr_failed_pages; /* Normal and large folios failed to be migrated, in 1502 units of base pages. Untried folios aren't counted */ 1503 int nr_thp_succeeded; /* THP migrated successfully */ 1504 int nr_thp_failed; /* THP failed to be migrated */ 1505 int nr_thp_split; /* THP split before migrating */ 1506 }; 1507 1508 /* 1509 * Returns the number of hugetlb folios that were not migrated, or an error code 1510 * after NR_MAX_MIGRATE_PAGES_RETRY attempts or if no hugetlb folios are movable 1511 * any more because the list has become empty or no retryable hugetlb folios 1512 * exist any more. It is caller's responsibility to call putback_movable_pages() 1513 * only if ret != 0. 1514 */ 1515 static int migrate_hugetlbs(struct list_head *from, new_page_t get_new_page, 1516 free_page_t put_new_page, unsigned long private, 1517 enum migrate_mode mode, int reason, 1518 struct migrate_pages_stats *stats, 1519 struct list_head *ret_folios) 1520 { 1521 int retry = 1; 1522 int nr_failed = 0; 1523 int nr_retry_pages = 0; 1524 int pass = 0; 1525 struct folio *folio, *folio2; 1526 int rc, nr_pages; 1527 1528 for (pass = 0; pass < NR_MAX_MIGRATE_PAGES_RETRY && retry; pass++) { 1529 retry = 0; 1530 nr_retry_pages = 0; 1531 1532 list_for_each_entry_safe(folio, folio2, from, lru) { 1533 if (!folio_test_hugetlb(folio)) 1534 continue; 1535 1536 nr_pages = folio_nr_pages(folio); 1537 1538 cond_resched(); 1539 1540 /* 1541 * Migratability of hugepages depends on architectures and 1542 * their size. This check is necessary because some callers 1543 * of hugepage migration like soft offline and memory 1544 * hotremove don't walk through page tables or check whether 1545 * the hugepage is pmd-based or not before kicking migration. 1546 */ 1547 if (!hugepage_migration_supported(folio_hstate(folio))) { 1548 nr_failed++; 1549 stats->nr_failed_pages += nr_pages; 1550 list_move_tail(&folio->lru, ret_folios); 1551 continue; 1552 } 1553 1554 rc = unmap_and_move_huge_page(get_new_page, 1555 put_new_page, private, 1556 &folio->page, pass > 2, mode, 1557 reason, ret_folios); 1558 /* 1559 * The rules are: 1560 * Success: hugetlb folio will be put back 1561 * -EAGAIN: stay on the from list 1562 * -ENOMEM: stay on the from list 1563 * Other errno: put on ret_folios list 1564 */ 1565 switch(rc) { 1566 case -ENOMEM: 1567 /* 1568 * When memory is low, don't bother to try to migrate 1569 * other folios, just exit. 1570 */ 1571 stats->nr_failed_pages += nr_pages + nr_retry_pages; 1572 return -ENOMEM; 1573 case -EAGAIN: 1574 retry++; 1575 nr_retry_pages += nr_pages; 1576 break; 1577 case MIGRATEPAGE_SUCCESS: 1578 stats->nr_succeeded += nr_pages; 1579 break; 1580 default: 1581 /* 1582 * Permanent failure (-EBUSY, etc.): 1583 * unlike -EAGAIN case, the failed folio is 1584 * removed from migration folio list and not 1585 * retried in the next outer loop. 1586 */ 1587 nr_failed++; 1588 stats->nr_failed_pages += nr_pages; 1589 break; 1590 } 1591 } 1592 } 1593 /* 1594 * nr_failed is number of hugetlb folios failed to be migrated. After 1595 * NR_MAX_MIGRATE_PAGES_RETRY attempts, give up and count retried hugetlb 1596 * folios as failed. 1597 */ 1598 nr_failed += retry; 1599 stats->nr_failed_pages += nr_retry_pages; 1600 1601 return nr_failed; 1602 } 1603 1604 /* 1605 * migrate_pages_batch() first unmaps folios in the from list as many as 1606 * possible, then move the unmapped folios. 1607 * 1608 * We only batch migration if mode == MIGRATE_ASYNC to avoid to wait a 1609 * lock or bit when we have locked more than one folio. Which may cause 1610 * deadlock (e.g., for loop device). So, if mode != MIGRATE_ASYNC, the 1611 * length of the from list must be <= 1. 1612 */ 1613 static int migrate_pages_batch(struct list_head *from, new_page_t get_new_page, 1614 free_page_t put_new_page, unsigned long private, 1615 enum migrate_mode mode, int reason, struct list_head *ret_folios, 1616 struct list_head *split_folios, struct migrate_pages_stats *stats, 1617 int nr_pass) 1618 { 1619 int retry = 1; 1620 int large_retry = 1; 1621 int thp_retry = 1; 1622 int nr_failed = 0; 1623 int nr_retry_pages = 0; 1624 int nr_large_failed = 0; 1625 int pass = 0; 1626 bool is_large = false; 1627 bool is_thp = false; 1628 struct folio *folio, *folio2, *dst = NULL, *dst2; 1629 int rc, rc_saved = 0, nr_pages; 1630 LIST_HEAD(unmap_folios); 1631 LIST_HEAD(dst_folios); 1632 bool nosplit = (reason == MR_NUMA_MISPLACED); 1633 1634 VM_WARN_ON_ONCE(mode != MIGRATE_ASYNC && 1635 !list_empty(from) && !list_is_singular(from)); 1636 1637 for (pass = 0; pass < nr_pass && (retry || large_retry); pass++) { 1638 retry = 0; 1639 large_retry = 0; 1640 thp_retry = 0; 1641 nr_retry_pages = 0; 1642 1643 list_for_each_entry_safe(folio, folio2, from, lru) { 1644 /* 1645 * Large folio statistics is based on the source large 1646 * folio. Capture required information that might get 1647 * lost during migration. 1648 */ 1649 is_large = folio_test_large(folio); 1650 is_thp = is_large && folio_test_pmd_mappable(folio); 1651 nr_pages = folio_nr_pages(folio); 1652 1653 cond_resched(); 1654 1655 /* 1656 * Large folio migration might be unsupported or 1657 * the allocation might be failed so we should retry 1658 * on the same folio with the large folio split 1659 * to normal folios. 1660 * 1661 * Split folios are put in split_folios, and 1662 * we will migrate them after the rest of the 1663 * list is processed. 1664 */ 1665 if (!thp_migration_supported() && is_thp) { 1666 nr_large_failed++; 1667 stats->nr_thp_failed++; 1668 if (!try_split_folio(folio, split_folios)) { 1669 stats->nr_thp_split++; 1670 continue; 1671 } 1672 stats->nr_failed_pages += nr_pages; 1673 list_move_tail(&folio->lru, ret_folios); 1674 continue; 1675 } 1676 1677 rc = migrate_folio_unmap(get_new_page, put_new_page, private, 1678 folio, &dst, mode, reason, ret_folios); 1679 /* 1680 * The rules are: 1681 * Success: folio will be freed 1682 * Unmap: folio will be put on unmap_folios list, 1683 * dst folio put on dst_folios list 1684 * -EAGAIN: stay on the from list 1685 * -ENOMEM: stay on the from list 1686 * Other errno: put on ret_folios list 1687 */ 1688 switch(rc) { 1689 case -ENOMEM: 1690 /* 1691 * When memory is low, don't bother to try to migrate 1692 * other folios, move unmapped folios, then exit. 1693 */ 1694 if (is_large) { 1695 nr_large_failed++; 1696 stats->nr_thp_failed += is_thp; 1697 /* Large folio NUMA faulting doesn't split to retry. */ 1698 if (!nosplit) { 1699 int ret = try_split_folio(folio, split_folios); 1700 1701 if (!ret) { 1702 stats->nr_thp_split += is_thp; 1703 break; 1704 } else if (reason == MR_LONGTERM_PIN && 1705 ret == -EAGAIN) { 1706 /* 1707 * Try again to split large folio to 1708 * mitigate the failure of longterm pinning. 1709 */ 1710 large_retry++; 1711 thp_retry += is_thp; 1712 nr_retry_pages += nr_pages; 1713 /* Undo duplicated failure counting. */ 1714 nr_large_failed--; 1715 stats->nr_thp_failed -= is_thp; 1716 break; 1717 } 1718 } 1719 } else { 1720 nr_failed++; 1721 } 1722 1723 stats->nr_failed_pages += nr_pages + nr_retry_pages; 1724 /* nr_failed isn't updated for not used */ 1725 nr_large_failed += large_retry; 1726 stats->nr_thp_failed += thp_retry; 1727 rc_saved = rc; 1728 if (list_empty(&unmap_folios)) 1729 goto out; 1730 else 1731 goto move; 1732 case -EAGAIN: 1733 if (is_large) { 1734 large_retry++; 1735 thp_retry += is_thp; 1736 } else { 1737 retry++; 1738 } 1739 nr_retry_pages += nr_pages; 1740 break; 1741 case MIGRATEPAGE_SUCCESS: 1742 stats->nr_succeeded += nr_pages; 1743 stats->nr_thp_succeeded += is_thp; 1744 break; 1745 case MIGRATEPAGE_UNMAP: 1746 list_move_tail(&folio->lru, &unmap_folios); 1747 list_add_tail(&dst->lru, &dst_folios); 1748 break; 1749 default: 1750 /* 1751 * Permanent failure (-EBUSY, etc.): 1752 * unlike -EAGAIN case, the failed folio is 1753 * removed from migration folio list and not 1754 * retried in the next outer loop. 1755 */ 1756 if (is_large) { 1757 nr_large_failed++; 1758 stats->nr_thp_failed += is_thp; 1759 } else { 1760 nr_failed++; 1761 } 1762 1763 stats->nr_failed_pages += nr_pages; 1764 break; 1765 } 1766 } 1767 } 1768 nr_failed += retry; 1769 nr_large_failed += large_retry; 1770 stats->nr_thp_failed += thp_retry; 1771 stats->nr_failed_pages += nr_retry_pages; 1772 move: 1773 /* Flush TLBs for all unmapped folios */ 1774 try_to_unmap_flush(); 1775 1776 retry = 1; 1777 for (pass = 0; pass < nr_pass && (retry || large_retry); pass++) { 1778 retry = 0; 1779 large_retry = 0; 1780 thp_retry = 0; 1781 nr_retry_pages = 0; 1782 1783 dst = list_first_entry(&dst_folios, struct folio, lru); 1784 dst2 = list_next_entry(dst, lru); 1785 list_for_each_entry_safe(folio, folio2, &unmap_folios, lru) { 1786 is_large = folio_test_large(folio); 1787 is_thp = is_large && folio_test_pmd_mappable(folio); 1788 nr_pages = folio_nr_pages(folio); 1789 1790 cond_resched(); 1791 1792 rc = migrate_folio_move(put_new_page, private, 1793 folio, dst, mode, 1794 reason, ret_folios); 1795 /* 1796 * The rules are: 1797 * Success: folio will be freed 1798 * -EAGAIN: stay on the unmap_folios list 1799 * Other errno: put on ret_folios list 1800 */ 1801 switch(rc) { 1802 case -EAGAIN: 1803 if (is_large) { 1804 large_retry++; 1805 thp_retry += is_thp; 1806 } else { 1807 retry++; 1808 } 1809 nr_retry_pages += nr_pages; 1810 break; 1811 case MIGRATEPAGE_SUCCESS: 1812 stats->nr_succeeded += nr_pages; 1813 stats->nr_thp_succeeded += is_thp; 1814 break; 1815 default: 1816 if (is_large) { 1817 nr_large_failed++; 1818 stats->nr_thp_failed += is_thp; 1819 } else { 1820 nr_failed++; 1821 } 1822 1823 stats->nr_failed_pages += nr_pages; 1824 break; 1825 } 1826 dst = dst2; 1827 dst2 = list_next_entry(dst, lru); 1828 } 1829 } 1830 nr_failed += retry; 1831 nr_large_failed += large_retry; 1832 stats->nr_thp_failed += thp_retry; 1833 stats->nr_failed_pages += nr_retry_pages; 1834 1835 if (rc_saved) 1836 rc = rc_saved; 1837 else 1838 rc = nr_failed + nr_large_failed; 1839 out: 1840 /* Cleanup remaining folios */ 1841 dst = list_first_entry(&dst_folios, struct folio, lru); 1842 dst2 = list_next_entry(dst, lru); 1843 list_for_each_entry_safe(folio, folio2, &unmap_folios, lru) { 1844 int page_was_mapped = 0; 1845 struct anon_vma *anon_vma = NULL; 1846 1847 __migrate_folio_extract(dst, &page_was_mapped, &anon_vma); 1848 migrate_folio_undo_src(folio, page_was_mapped, anon_vma, 1849 true, ret_folios); 1850 list_del(&dst->lru); 1851 migrate_folio_undo_dst(dst, true, put_new_page, private); 1852 dst = dst2; 1853 dst2 = list_next_entry(dst, lru); 1854 } 1855 1856 return rc; 1857 } 1858 1859 static int migrate_pages_sync(struct list_head *from, new_page_t get_new_page, 1860 free_page_t put_new_page, unsigned long private, 1861 enum migrate_mode mode, int reason, struct list_head *ret_folios, 1862 struct list_head *split_folios, struct migrate_pages_stats *stats) 1863 { 1864 int rc, nr_failed = 0; 1865 LIST_HEAD(folios); 1866 struct migrate_pages_stats astats; 1867 1868 memset(&astats, 0, sizeof(astats)); 1869 /* Try to migrate in batch with MIGRATE_ASYNC mode firstly */ 1870 rc = migrate_pages_batch(from, get_new_page, put_new_page, private, MIGRATE_ASYNC, 1871 reason, &folios, split_folios, &astats, 1872 NR_MAX_MIGRATE_ASYNC_RETRY); 1873 stats->nr_succeeded += astats.nr_succeeded; 1874 stats->nr_thp_succeeded += astats.nr_thp_succeeded; 1875 stats->nr_thp_split += astats.nr_thp_split; 1876 if (rc < 0) { 1877 stats->nr_failed_pages += astats.nr_failed_pages; 1878 stats->nr_thp_failed += astats.nr_thp_failed; 1879 list_splice_tail(&folios, ret_folios); 1880 return rc; 1881 } 1882 stats->nr_thp_failed += astats.nr_thp_split; 1883 nr_failed += astats.nr_thp_split; 1884 /* 1885 * Fall back to migrate all failed folios one by one synchronously. All 1886 * failed folios except split THPs will be retried, so their failure 1887 * isn't counted 1888 */ 1889 list_splice_tail_init(&folios, from); 1890 while (!list_empty(from)) { 1891 list_move(from->next, &folios); 1892 rc = migrate_pages_batch(&folios, get_new_page, put_new_page, 1893 private, mode, reason, ret_folios, 1894 split_folios, stats, NR_MAX_MIGRATE_SYNC_RETRY); 1895 list_splice_tail_init(&folios, ret_folios); 1896 if (rc < 0) 1897 return rc; 1898 nr_failed += rc; 1899 } 1900 1901 return nr_failed; 1902 } 1903 1904 /* 1905 * migrate_pages - migrate the folios specified in a list, to the free folios 1906 * supplied as the target for the page migration 1907 * 1908 * @from: The list of folios to be migrated. 1909 * @get_new_page: The function used to allocate free folios to be used 1910 * as the target of the folio migration. 1911 * @put_new_page: The function used to free target folios if migration 1912 * fails, or NULL if no special handling is necessary. 1913 * @private: Private data to be passed on to get_new_page() 1914 * @mode: The migration mode that specifies the constraints for 1915 * folio migration, if any. 1916 * @reason: The reason for folio migration. 1917 * @ret_succeeded: Set to the number of folios migrated successfully if 1918 * the caller passes a non-NULL pointer. 1919 * 1920 * The function returns after NR_MAX_MIGRATE_PAGES_RETRY attempts or if no folios 1921 * are movable any more because the list has become empty or no retryable folios 1922 * exist any more. It is caller's responsibility to call putback_movable_pages() 1923 * only if ret != 0. 1924 * 1925 * Returns the number of {normal folio, large folio, hugetlb} that were not 1926 * migrated, or an error code. The number of large folio splits will be 1927 * considered as the number of non-migrated large folio, no matter how many 1928 * split folios of the large folio are migrated successfully. 1929 */ 1930 int migrate_pages(struct list_head *from, new_page_t get_new_page, 1931 free_page_t put_new_page, unsigned long private, 1932 enum migrate_mode mode, int reason, unsigned int *ret_succeeded) 1933 { 1934 int rc, rc_gather; 1935 int nr_pages; 1936 struct folio *folio, *folio2; 1937 LIST_HEAD(folios); 1938 LIST_HEAD(ret_folios); 1939 LIST_HEAD(split_folios); 1940 struct migrate_pages_stats stats; 1941 1942 trace_mm_migrate_pages_start(mode, reason); 1943 1944 memset(&stats, 0, sizeof(stats)); 1945 1946 rc_gather = migrate_hugetlbs(from, get_new_page, put_new_page, private, 1947 mode, reason, &stats, &ret_folios); 1948 if (rc_gather < 0) 1949 goto out; 1950 1951 again: 1952 nr_pages = 0; 1953 list_for_each_entry_safe(folio, folio2, from, lru) { 1954 /* Retried hugetlb folios will be kept in list */ 1955 if (folio_test_hugetlb(folio)) { 1956 list_move_tail(&folio->lru, &ret_folios); 1957 continue; 1958 } 1959 1960 nr_pages += folio_nr_pages(folio); 1961 if (nr_pages >= NR_MAX_BATCHED_MIGRATION) 1962 break; 1963 } 1964 if (nr_pages >= NR_MAX_BATCHED_MIGRATION) 1965 list_cut_before(&folios, from, &folio2->lru); 1966 else 1967 list_splice_init(from, &folios); 1968 if (mode == MIGRATE_ASYNC) 1969 rc = migrate_pages_batch(&folios, get_new_page, put_new_page, private, 1970 mode, reason, &ret_folios, &split_folios, &stats, 1971 NR_MAX_MIGRATE_PAGES_RETRY); 1972 else 1973 rc = migrate_pages_sync(&folios, get_new_page, put_new_page, private, 1974 mode, reason, &ret_folios, &split_folios, &stats); 1975 list_splice_tail_init(&folios, &ret_folios); 1976 if (rc < 0) { 1977 rc_gather = rc; 1978 list_splice_tail(&split_folios, &ret_folios); 1979 goto out; 1980 } 1981 if (!list_empty(&split_folios)) { 1982 /* 1983 * Failure isn't counted since all split folios of a large folio 1984 * is counted as 1 failure already. And, we only try to migrate 1985 * with minimal effort, force MIGRATE_ASYNC mode and retry once. 1986 */ 1987 migrate_pages_batch(&split_folios, get_new_page, put_new_page, private, 1988 MIGRATE_ASYNC, reason, &ret_folios, NULL, &stats, 1); 1989 list_splice_tail_init(&split_folios, &ret_folios); 1990 } 1991 rc_gather += rc; 1992 if (!list_empty(from)) 1993 goto again; 1994 out: 1995 /* 1996 * Put the permanent failure folio back to migration list, they 1997 * will be put back to the right list by the caller. 1998 */ 1999 list_splice(&ret_folios, from); 2000 2001 /* 2002 * Return 0 in case all split folios of fail-to-migrate large folios 2003 * are migrated successfully. 2004 */ 2005 if (list_empty(from)) 2006 rc_gather = 0; 2007 2008 count_vm_events(PGMIGRATE_SUCCESS, stats.nr_succeeded); 2009 count_vm_events(PGMIGRATE_FAIL, stats.nr_failed_pages); 2010 count_vm_events(THP_MIGRATION_SUCCESS, stats.nr_thp_succeeded); 2011 count_vm_events(THP_MIGRATION_FAIL, stats.nr_thp_failed); 2012 count_vm_events(THP_MIGRATION_SPLIT, stats.nr_thp_split); 2013 trace_mm_migrate_pages(stats.nr_succeeded, stats.nr_failed_pages, 2014 stats.nr_thp_succeeded, stats.nr_thp_failed, 2015 stats.nr_thp_split, mode, reason); 2016 2017 if (ret_succeeded) 2018 *ret_succeeded = stats.nr_succeeded; 2019 2020 return rc_gather; 2021 } 2022 2023 struct page *alloc_migration_target(struct page *page, unsigned long private) 2024 { 2025 struct folio *folio = page_folio(page); 2026 struct migration_target_control *mtc; 2027 gfp_t gfp_mask; 2028 unsigned int order = 0; 2029 struct folio *hugetlb_folio = NULL; 2030 struct folio *new_folio = NULL; 2031 int nid; 2032 int zidx; 2033 2034 mtc = (struct migration_target_control *)private; 2035 gfp_mask = mtc->gfp_mask; 2036 nid = mtc->nid; 2037 if (nid == NUMA_NO_NODE) 2038 nid = folio_nid(folio); 2039 2040 if (folio_test_hugetlb(folio)) { 2041 struct hstate *h = folio_hstate(folio); 2042 2043 gfp_mask = htlb_modify_alloc_mask(h, gfp_mask); 2044 hugetlb_folio = alloc_hugetlb_folio_nodemask(h, nid, 2045 mtc->nmask, gfp_mask); 2046 return &hugetlb_folio->page; 2047 } 2048 2049 if (folio_test_large(folio)) { 2050 /* 2051 * clear __GFP_RECLAIM to make the migration callback 2052 * consistent with regular THP allocations. 2053 */ 2054 gfp_mask &= ~__GFP_RECLAIM; 2055 gfp_mask |= GFP_TRANSHUGE; 2056 order = folio_order(folio); 2057 } 2058 zidx = zone_idx(folio_zone(folio)); 2059 if (is_highmem_idx(zidx) || zidx == ZONE_MOVABLE) 2060 gfp_mask |= __GFP_HIGHMEM; 2061 2062 new_folio = __folio_alloc(gfp_mask, order, nid, mtc->nmask); 2063 2064 return &new_folio->page; 2065 } 2066 2067 #ifdef CONFIG_NUMA 2068 2069 static int store_status(int __user *status, int start, int value, int nr) 2070 { 2071 while (nr-- > 0) { 2072 if (put_user(value, status + start)) 2073 return -EFAULT; 2074 start++; 2075 } 2076 2077 return 0; 2078 } 2079 2080 static int do_move_pages_to_node(struct mm_struct *mm, 2081 struct list_head *pagelist, int node) 2082 { 2083 int err; 2084 struct migration_target_control mtc = { 2085 .nid = node, 2086 .gfp_mask = GFP_HIGHUSER_MOVABLE | __GFP_THISNODE, 2087 }; 2088 2089 err = migrate_pages(pagelist, alloc_migration_target, NULL, 2090 (unsigned long)&mtc, MIGRATE_SYNC, MR_SYSCALL, NULL); 2091 if (err) 2092 putback_movable_pages(pagelist); 2093 return err; 2094 } 2095 2096 /* 2097 * Resolves the given address to a struct page, isolates it from the LRU and 2098 * puts it to the given pagelist. 2099 * Returns: 2100 * errno - if the page cannot be found/isolated 2101 * 0 - when it doesn't have to be migrated because it is already on the 2102 * target node 2103 * 1 - when it has been queued 2104 */ 2105 static int add_page_for_migration(struct mm_struct *mm, unsigned long addr, 2106 int node, struct list_head *pagelist, bool migrate_all) 2107 { 2108 struct vm_area_struct *vma; 2109 struct page *page; 2110 int err; 2111 bool isolated; 2112 2113 mmap_read_lock(mm); 2114 err = -EFAULT; 2115 vma = vma_lookup(mm, addr); 2116 if (!vma || !vma_migratable(vma)) 2117 goto out; 2118 2119 /* FOLL_DUMP to ignore special (like zero) pages */ 2120 page = follow_page(vma, addr, FOLL_GET | FOLL_DUMP); 2121 2122 err = PTR_ERR(page); 2123 if (IS_ERR(page)) 2124 goto out; 2125 2126 err = -ENOENT; 2127 if (!page) 2128 goto out; 2129 2130 if (is_zone_device_page(page)) 2131 goto out_putpage; 2132 2133 err = 0; 2134 if (page_to_nid(page) == node) 2135 goto out_putpage; 2136 2137 err = -EACCES; 2138 if (page_mapcount(page) > 1 && !migrate_all) 2139 goto out_putpage; 2140 2141 if (PageHuge(page)) { 2142 if (PageHead(page)) { 2143 isolated = isolate_hugetlb(page_folio(page), pagelist); 2144 err = isolated ? 1 : -EBUSY; 2145 } 2146 } else { 2147 struct page *head; 2148 2149 head = compound_head(page); 2150 isolated = isolate_lru_page(head); 2151 if (!isolated) { 2152 err = -EBUSY; 2153 goto out_putpage; 2154 } 2155 2156 err = 1; 2157 list_add_tail(&head->lru, pagelist); 2158 mod_node_page_state(page_pgdat(head), 2159 NR_ISOLATED_ANON + page_is_file_lru(head), 2160 thp_nr_pages(head)); 2161 } 2162 out_putpage: 2163 /* 2164 * Either remove the duplicate refcount from 2165 * isolate_lru_page() or drop the page ref if it was 2166 * not isolated. 2167 */ 2168 put_page(page); 2169 out: 2170 mmap_read_unlock(mm); 2171 return err; 2172 } 2173 2174 static int move_pages_and_store_status(struct mm_struct *mm, int node, 2175 struct list_head *pagelist, int __user *status, 2176 int start, int i, unsigned long nr_pages) 2177 { 2178 int err; 2179 2180 if (list_empty(pagelist)) 2181 return 0; 2182 2183 err = do_move_pages_to_node(mm, pagelist, node); 2184 if (err) { 2185 /* 2186 * Positive err means the number of failed 2187 * pages to migrate. Since we are going to 2188 * abort and return the number of non-migrated 2189 * pages, so need to include the rest of the 2190 * nr_pages that have not been attempted as 2191 * well. 2192 */ 2193 if (err > 0) 2194 err += nr_pages - i; 2195 return err; 2196 } 2197 return store_status(status, start, node, i - start); 2198 } 2199 2200 /* 2201 * Migrate an array of page address onto an array of nodes and fill 2202 * the corresponding array of status. 2203 */ 2204 static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes, 2205 unsigned long nr_pages, 2206 const void __user * __user *pages, 2207 const int __user *nodes, 2208 int __user *status, int flags) 2209 { 2210 int current_node = NUMA_NO_NODE; 2211 LIST_HEAD(pagelist); 2212 int start, i; 2213 int err = 0, err1; 2214 2215 lru_cache_disable(); 2216 2217 for (i = start = 0; i < nr_pages; i++) { 2218 const void __user *p; 2219 unsigned long addr; 2220 int node; 2221 2222 err = -EFAULT; 2223 if (get_user(p, pages + i)) 2224 goto out_flush; 2225 if (get_user(node, nodes + i)) 2226 goto out_flush; 2227 addr = (unsigned long)untagged_addr(p); 2228 2229 err = -ENODEV; 2230 if (node < 0 || node >= MAX_NUMNODES) 2231 goto out_flush; 2232 if (!node_state(node, N_MEMORY)) 2233 goto out_flush; 2234 2235 err = -EACCES; 2236 if (!node_isset(node, task_nodes)) 2237 goto out_flush; 2238 2239 if (current_node == NUMA_NO_NODE) { 2240 current_node = node; 2241 start = i; 2242 } else if (node != current_node) { 2243 err = move_pages_and_store_status(mm, current_node, 2244 &pagelist, status, start, i, nr_pages); 2245 if (err) 2246 goto out; 2247 start = i; 2248 current_node = node; 2249 } 2250 2251 /* 2252 * Errors in the page lookup or isolation are not fatal and we simply 2253 * report them via status 2254 */ 2255 err = add_page_for_migration(mm, addr, current_node, 2256 &pagelist, flags & MPOL_MF_MOVE_ALL); 2257 2258 if (err > 0) { 2259 /* The page is successfully queued for migration */ 2260 continue; 2261 } 2262 2263 /* 2264 * The move_pages() man page does not have an -EEXIST choice, so 2265 * use -EFAULT instead. 2266 */ 2267 if (err == -EEXIST) 2268 err = -EFAULT; 2269 2270 /* 2271 * If the page is already on the target node (!err), store the 2272 * node, otherwise, store the err. 2273 */ 2274 err = store_status(status, i, err ? : current_node, 1); 2275 if (err) 2276 goto out_flush; 2277 2278 err = move_pages_and_store_status(mm, current_node, &pagelist, 2279 status, start, i, nr_pages); 2280 if (err) { 2281 /* We have accounted for page i */ 2282 if (err > 0) 2283 err--; 2284 goto out; 2285 } 2286 current_node = NUMA_NO_NODE; 2287 } 2288 out_flush: 2289 /* Make sure we do not overwrite the existing error */ 2290 err1 = move_pages_and_store_status(mm, current_node, &pagelist, 2291 status, start, i, nr_pages); 2292 if (err >= 0) 2293 err = err1; 2294 out: 2295 lru_cache_enable(); 2296 return err; 2297 } 2298 2299 /* 2300 * Determine the nodes of an array of pages and store it in an array of status. 2301 */ 2302 static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages, 2303 const void __user **pages, int *status) 2304 { 2305 unsigned long i; 2306 2307 mmap_read_lock(mm); 2308 2309 for (i = 0; i < nr_pages; i++) { 2310 unsigned long addr = (unsigned long)(*pages); 2311 struct vm_area_struct *vma; 2312 struct page *page; 2313 int err = -EFAULT; 2314 2315 vma = vma_lookup(mm, addr); 2316 if (!vma) 2317 goto set_status; 2318 2319 /* FOLL_DUMP to ignore special (like zero) pages */ 2320 page = follow_page(vma, addr, FOLL_GET | FOLL_DUMP); 2321 2322 err = PTR_ERR(page); 2323 if (IS_ERR(page)) 2324 goto set_status; 2325 2326 err = -ENOENT; 2327 if (!page) 2328 goto set_status; 2329 2330 if (!is_zone_device_page(page)) 2331 err = page_to_nid(page); 2332 2333 put_page(page); 2334 set_status: 2335 *status = err; 2336 2337 pages++; 2338 status++; 2339 } 2340 2341 mmap_read_unlock(mm); 2342 } 2343 2344 static int get_compat_pages_array(const void __user *chunk_pages[], 2345 const void __user * __user *pages, 2346 unsigned long chunk_nr) 2347 { 2348 compat_uptr_t __user *pages32 = (compat_uptr_t __user *)pages; 2349 compat_uptr_t p; 2350 int i; 2351 2352 for (i = 0; i < chunk_nr; i++) { 2353 if (get_user(p, pages32 + i)) 2354 return -EFAULT; 2355 chunk_pages[i] = compat_ptr(p); 2356 } 2357 2358 return 0; 2359 } 2360 2361 /* 2362 * Determine the nodes of a user array of pages and store it in 2363 * a user array of status. 2364 */ 2365 static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages, 2366 const void __user * __user *pages, 2367 int __user *status) 2368 { 2369 #define DO_PAGES_STAT_CHUNK_NR 16UL 2370 const void __user *chunk_pages[DO_PAGES_STAT_CHUNK_NR]; 2371 int chunk_status[DO_PAGES_STAT_CHUNK_NR]; 2372 2373 while (nr_pages) { 2374 unsigned long chunk_nr = min(nr_pages, DO_PAGES_STAT_CHUNK_NR); 2375 2376 if (in_compat_syscall()) { 2377 if (get_compat_pages_array(chunk_pages, pages, 2378 chunk_nr)) 2379 break; 2380 } else { 2381 if (copy_from_user(chunk_pages, pages, 2382 chunk_nr * sizeof(*chunk_pages))) 2383 break; 2384 } 2385 2386 do_pages_stat_array(mm, chunk_nr, chunk_pages, chunk_status); 2387 2388 if (copy_to_user(status, chunk_status, chunk_nr * sizeof(*status))) 2389 break; 2390 2391 pages += chunk_nr; 2392 status += chunk_nr; 2393 nr_pages -= chunk_nr; 2394 } 2395 return nr_pages ? -EFAULT : 0; 2396 } 2397 2398 static struct mm_struct *find_mm_struct(pid_t pid, nodemask_t *mem_nodes) 2399 { 2400 struct task_struct *task; 2401 struct mm_struct *mm; 2402 2403 /* 2404 * There is no need to check if current process has the right to modify 2405 * the specified process when they are same. 2406 */ 2407 if (!pid) { 2408 mmget(current->mm); 2409 *mem_nodes = cpuset_mems_allowed(current); 2410 return current->mm; 2411 } 2412 2413 /* Find the mm_struct */ 2414 rcu_read_lock(); 2415 task = find_task_by_vpid(pid); 2416 if (!task) { 2417 rcu_read_unlock(); 2418 return ERR_PTR(-ESRCH); 2419 } 2420 get_task_struct(task); 2421 2422 /* 2423 * Check if this process has the right to modify the specified 2424 * process. Use the regular "ptrace_may_access()" checks. 2425 */ 2426 if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) { 2427 rcu_read_unlock(); 2428 mm = ERR_PTR(-EPERM); 2429 goto out; 2430 } 2431 rcu_read_unlock(); 2432 2433 mm = ERR_PTR(security_task_movememory(task)); 2434 if (IS_ERR(mm)) 2435 goto out; 2436 *mem_nodes = cpuset_mems_allowed(task); 2437 mm = get_task_mm(task); 2438 out: 2439 put_task_struct(task); 2440 if (!mm) 2441 mm = ERR_PTR(-EINVAL); 2442 return mm; 2443 } 2444 2445 /* 2446 * Move a list of pages in the address space of the currently executing 2447 * process. 2448 */ 2449 static int kernel_move_pages(pid_t pid, unsigned long nr_pages, 2450 const void __user * __user *pages, 2451 const int __user *nodes, 2452 int __user *status, int flags) 2453 { 2454 struct mm_struct *mm; 2455 int err; 2456 nodemask_t task_nodes; 2457 2458 /* Check flags */ 2459 if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL)) 2460 return -EINVAL; 2461 2462 if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE)) 2463 return -EPERM; 2464 2465 mm = find_mm_struct(pid, &task_nodes); 2466 if (IS_ERR(mm)) 2467 return PTR_ERR(mm); 2468 2469 if (nodes) 2470 err = do_pages_move(mm, task_nodes, nr_pages, pages, 2471 nodes, status, flags); 2472 else 2473 err = do_pages_stat(mm, nr_pages, pages, status); 2474 2475 mmput(mm); 2476 return err; 2477 } 2478 2479 SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages, 2480 const void __user * __user *, pages, 2481 const int __user *, nodes, 2482 int __user *, status, int, flags) 2483 { 2484 return kernel_move_pages(pid, nr_pages, pages, nodes, status, flags); 2485 } 2486 2487 #ifdef CONFIG_NUMA_BALANCING 2488 /* 2489 * Returns true if this is a safe migration target node for misplaced NUMA 2490 * pages. Currently it only checks the watermarks which is crude. 2491 */ 2492 static bool migrate_balanced_pgdat(struct pglist_data *pgdat, 2493 unsigned long nr_migrate_pages) 2494 { 2495 int z; 2496 2497 for (z = pgdat->nr_zones - 1; z >= 0; z--) { 2498 struct zone *zone = pgdat->node_zones + z; 2499 2500 if (!managed_zone(zone)) 2501 continue; 2502 2503 /* Avoid waking kswapd by allocating pages_to_migrate pages. */ 2504 if (!zone_watermark_ok(zone, 0, 2505 high_wmark_pages(zone) + 2506 nr_migrate_pages, 2507 ZONE_MOVABLE, 0)) 2508 continue; 2509 return true; 2510 } 2511 return false; 2512 } 2513 2514 static struct page *alloc_misplaced_dst_page(struct page *page, 2515 unsigned long data) 2516 { 2517 int nid = (int) data; 2518 int order = compound_order(page); 2519 gfp_t gfp = __GFP_THISNODE; 2520 struct folio *new; 2521 2522 if (order > 0) 2523 gfp |= GFP_TRANSHUGE_LIGHT; 2524 else { 2525 gfp |= GFP_HIGHUSER_MOVABLE | __GFP_NOMEMALLOC | __GFP_NORETRY | 2526 __GFP_NOWARN; 2527 gfp &= ~__GFP_RECLAIM; 2528 } 2529 new = __folio_alloc_node(gfp, order, nid); 2530 2531 return &new->page; 2532 } 2533 2534 static int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page) 2535 { 2536 int nr_pages = thp_nr_pages(page); 2537 int order = compound_order(page); 2538 2539 VM_BUG_ON_PAGE(order && !PageTransHuge(page), page); 2540 2541 /* Do not migrate THP mapped by multiple processes */ 2542 if (PageTransHuge(page) && total_mapcount(page) > 1) 2543 return 0; 2544 2545 /* Avoid migrating to a node that is nearly full */ 2546 if (!migrate_balanced_pgdat(pgdat, nr_pages)) { 2547 int z; 2548 2549 if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING)) 2550 return 0; 2551 for (z = pgdat->nr_zones - 1; z >= 0; z--) { 2552 if (managed_zone(pgdat->node_zones + z)) 2553 break; 2554 } 2555 wakeup_kswapd(pgdat->node_zones + z, 0, order, ZONE_MOVABLE); 2556 return 0; 2557 } 2558 2559 if (!isolate_lru_page(page)) 2560 return 0; 2561 2562 mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + page_is_file_lru(page), 2563 nr_pages); 2564 2565 /* 2566 * Isolating the page has taken another reference, so the 2567 * caller's reference can be safely dropped without the page 2568 * disappearing underneath us during migration. 2569 */ 2570 put_page(page); 2571 return 1; 2572 } 2573 2574 /* 2575 * Attempt to migrate a misplaced page to the specified destination 2576 * node. Caller is expected to have an elevated reference count on 2577 * the page that will be dropped by this function before returning. 2578 */ 2579 int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma, 2580 int node) 2581 { 2582 pg_data_t *pgdat = NODE_DATA(node); 2583 int isolated; 2584 int nr_remaining; 2585 unsigned int nr_succeeded; 2586 LIST_HEAD(migratepages); 2587 int nr_pages = thp_nr_pages(page); 2588 2589 /* 2590 * Don't migrate file pages that are mapped in multiple processes 2591 * with execute permissions as they are probably shared libraries. 2592 */ 2593 if (page_mapcount(page) != 1 && page_is_file_lru(page) && 2594 (vma->vm_flags & VM_EXEC)) 2595 goto out; 2596 2597 /* 2598 * Also do not migrate dirty pages as not all filesystems can move 2599 * dirty pages in MIGRATE_ASYNC mode which is a waste of cycles. 2600 */ 2601 if (page_is_file_lru(page) && PageDirty(page)) 2602 goto out; 2603 2604 isolated = numamigrate_isolate_page(pgdat, page); 2605 if (!isolated) 2606 goto out; 2607 2608 list_add(&page->lru, &migratepages); 2609 nr_remaining = migrate_pages(&migratepages, alloc_misplaced_dst_page, 2610 NULL, node, MIGRATE_ASYNC, 2611 MR_NUMA_MISPLACED, &nr_succeeded); 2612 if (nr_remaining) { 2613 if (!list_empty(&migratepages)) { 2614 list_del(&page->lru); 2615 mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + 2616 page_is_file_lru(page), -nr_pages); 2617 putback_lru_page(page); 2618 } 2619 isolated = 0; 2620 } 2621 if (nr_succeeded) { 2622 count_vm_numa_events(NUMA_PAGE_MIGRATE, nr_succeeded); 2623 if (!node_is_toptier(page_to_nid(page)) && node_is_toptier(node)) 2624 mod_node_page_state(pgdat, PGPROMOTE_SUCCESS, 2625 nr_succeeded); 2626 } 2627 BUG_ON(!list_empty(&migratepages)); 2628 return isolated; 2629 2630 out: 2631 put_page(page); 2632 return 0; 2633 } 2634 #endif /* CONFIG_NUMA_BALANCING */ 2635 #endif /* CONFIG_NUMA */ 2636