1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Memory Migration functionality - linux/mm/migrate.c 4 * 5 * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter 6 * 7 * Page migration was first developed in the context of the memory hotplug 8 * project. The main authors of the migration code are: 9 * 10 * IWAMOTO Toshihiro <iwamoto@valinux.co.jp> 11 * Hirokazu Takahashi <taka@valinux.co.jp> 12 * Dave Hansen <haveblue@us.ibm.com> 13 * Christoph Lameter 14 */ 15 16 #include <linux/migrate.h> 17 #include <linux/export.h> 18 #include <linux/swap.h> 19 #include <linux/swapops.h> 20 #include <linux/pagemap.h> 21 #include <linux/buffer_head.h> 22 #include <linux/mm_inline.h> 23 #include <linux/nsproxy.h> 24 #include <linux/pagevec.h> 25 #include <linux/ksm.h> 26 #include <linux/rmap.h> 27 #include <linux/topology.h> 28 #include <linux/cpu.h> 29 #include <linux/cpuset.h> 30 #include <linux/writeback.h> 31 #include <linux/mempolicy.h> 32 #include <linux/vmalloc.h> 33 #include <linux/security.h> 34 #include <linux/backing-dev.h> 35 #include <linux/compaction.h> 36 #include <linux/syscalls.h> 37 #include <linux/compat.h> 38 #include <linux/hugetlb.h> 39 #include <linux/hugetlb_cgroup.h> 40 #include <linux/gfp.h> 41 #include <linux/pfn_t.h> 42 #include <linux/memremap.h> 43 #include <linux/userfaultfd_k.h> 44 #include <linux/balloon_compaction.h> 45 #include <linux/page_idle.h> 46 #include <linux/page_owner.h> 47 #include <linux/sched/mm.h> 48 #include <linux/ptrace.h> 49 #include <linux/oom.h> 50 #include <linux/memory.h> 51 #include <linux/random.h> 52 #include <linux/sched/sysctl.h> 53 #include <linux/memory-tiers.h> 54 55 #include <asm/tlbflush.h> 56 57 #include <trace/events/migrate.h> 58 59 #include "internal.h" 60 61 int isolate_movable_page(struct page *page, isolate_mode_t mode) 62 { 63 const struct movable_operations *mops; 64 65 /* 66 * Avoid burning cycles with pages that are yet under __free_pages(), 67 * or just got freed under us. 68 * 69 * In case we 'win' a race for a movable page being freed under us and 70 * raise its refcount preventing __free_pages() from doing its job 71 * the put_page() at the end of this block will take care of 72 * release this page, thus avoiding a nasty leakage. 73 */ 74 if (unlikely(!get_page_unless_zero(page))) 75 goto out; 76 77 if (unlikely(PageSlab(page))) 78 goto out_putpage; 79 /* Pairs with smp_wmb() in slab freeing, e.g. SLUB's __free_slab() */ 80 smp_rmb(); 81 /* 82 * Check movable flag before taking the page lock because 83 * we use non-atomic bitops on newly allocated page flags so 84 * unconditionally grabbing the lock ruins page's owner side. 85 */ 86 if (unlikely(!__PageMovable(page))) 87 goto out_putpage; 88 /* Pairs with smp_wmb() in slab allocation, e.g. SLUB's alloc_slab_page() */ 89 smp_rmb(); 90 if (unlikely(PageSlab(page))) 91 goto out_putpage; 92 93 /* 94 * As movable pages are not isolated from LRU lists, concurrent 95 * compaction threads can race against page migration functions 96 * as well as race against the releasing a page. 97 * 98 * In order to avoid having an already isolated movable page 99 * being (wrongly) re-isolated while it is under migration, 100 * or to avoid attempting to isolate pages being released, 101 * lets be sure we have the page lock 102 * before proceeding with the movable page isolation steps. 103 */ 104 if (unlikely(!trylock_page(page))) 105 goto out_putpage; 106 107 if (!PageMovable(page) || PageIsolated(page)) 108 goto out_no_isolated; 109 110 mops = page_movable_ops(page); 111 VM_BUG_ON_PAGE(!mops, page); 112 113 if (!mops->isolate_page(page, mode)) 114 goto out_no_isolated; 115 116 /* Driver shouldn't use PG_isolated bit of page->flags */ 117 WARN_ON_ONCE(PageIsolated(page)); 118 SetPageIsolated(page); 119 unlock_page(page); 120 121 return 0; 122 123 out_no_isolated: 124 unlock_page(page); 125 out_putpage: 126 put_page(page); 127 out: 128 return -EBUSY; 129 } 130 131 static void putback_movable_page(struct page *page) 132 { 133 const struct movable_operations *mops = page_movable_ops(page); 134 135 mops->putback_page(page); 136 ClearPageIsolated(page); 137 } 138 139 /* 140 * Put previously isolated pages back onto the appropriate lists 141 * from where they were once taken off for compaction/migration. 142 * 143 * This function shall be used whenever the isolated pageset has been 144 * built from lru, balloon, hugetlbfs page. See isolate_migratepages_range() 145 * and isolate_hugetlb(). 146 */ 147 void putback_movable_pages(struct list_head *l) 148 { 149 struct page *page; 150 struct page *page2; 151 152 list_for_each_entry_safe(page, page2, l, lru) { 153 if (unlikely(PageHuge(page))) { 154 putback_active_hugepage(page); 155 continue; 156 } 157 list_del(&page->lru); 158 /* 159 * We isolated non-lru movable page so here we can use 160 * __PageMovable because LRU page's mapping cannot have 161 * PAGE_MAPPING_MOVABLE. 162 */ 163 if (unlikely(__PageMovable(page))) { 164 VM_BUG_ON_PAGE(!PageIsolated(page), page); 165 lock_page(page); 166 if (PageMovable(page)) 167 putback_movable_page(page); 168 else 169 ClearPageIsolated(page); 170 unlock_page(page); 171 put_page(page); 172 } else { 173 mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + 174 page_is_file_lru(page), -thp_nr_pages(page)); 175 putback_lru_page(page); 176 } 177 } 178 } 179 180 /* 181 * Restore a potential migration pte to a working pte entry 182 */ 183 static bool remove_migration_pte(struct folio *folio, 184 struct vm_area_struct *vma, unsigned long addr, void *old) 185 { 186 DEFINE_FOLIO_VMA_WALK(pvmw, old, vma, addr, PVMW_SYNC | PVMW_MIGRATION); 187 188 while (page_vma_mapped_walk(&pvmw)) { 189 rmap_t rmap_flags = RMAP_NONE; 190 pte_t pte; 191 swp_entry_t entry; 192 struct page *new; 193 unsigned long idx = 0; 194 195 /* pgoff is invalid for ksm pages, but they are never large */ 196 if (folio_test_large(folio) && !folio_test_hugetlb(folio)) 197 idx = linear_page_index(vma, pvmw.address) - pvmw.pgoff; 198 new = folio_page(folio, idx); 199 200 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 201 /* PMD-mapped THP migration entry */ 202 if (!pvmw.pte) { 203 VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) || 204 !folio_test_pmd_mappable(folio), folio); 205 remove_migration_pmd(&pvmw, new); 206 continue; 207 } 208 #endif 209 210 folio_get(folio); 211 pte = mk_pte(new, READ_ONCE(vma->vm_page_prot)); 212 if (pte_swp_soft_dirty(*pvmw.pte)) 213 pte = pte_mksoft_dirty(pte); 214 215 /* 216 * Recheck VMA as permissions can change since migration started 217 */ 218 entry = pte_to_swp_entry(*pvmw.pte); 219 if (!is_migration_entry_young(entry)) 220 pte = pte_mkold(pte); 221 if (folio_test_dirty(folio) && is_migration_entry_dirty(entry)) 222 pte = pte_mkdirty(pte); 223 if (is_writable_migration_entry(entry)) 224 pte = maybe_mkwrite(pte, vma); 225 else if (pte_swp_uffd_wp(*pvmw.pte)) 226 pte = pte_mkuffd_wp(pte); 227 228 if (folio_test_anon(folio) && !is_readable_migration_entry(entry)) 229 rmap_flags |= RMAP_EXCLUSIVE; 230 231 if (unlikely(is_device_private_page(new))) { 232 if (pte_write(pte)) 233 entry = make_writable_device_private_entry( 234 page_to_pfn(new)); 235 else 236 entry = make_readable_device_private_entry( 237 page_to_pfn(new)); 238 pte = swp_entry_to_pte(entry); 239 if (pte_swp_soft_dirty(*pvmw.pte)) 240 pte = pte_swp_mksoft_dirty(pte); 241 if (pte_swp_uffd_wp(*pvmw.pte)) 242 pte = pte_swp_mkuffd_wp(pte); 243 } 244 245 #ifdef CONFIG_HUGETLB_PAGE 246 if (folio_test_hugetlb(folio)) { 247 unsigned int shift = huge_page_shift(hstate_vma(vma)); 248 249 pte = pte_mkhuge(pte); 250 pte = arch_make_huge_pte(pte, shift, vma->vm_flags); 251 if (folio_test_anon(folio)) 252 hugepage_add_anon_rmap(new, vma, pvmw.address, 253 rmap_flags); 254 else 255 page_dup_file_rmap(new, true); 256 set_huge_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte); 257 } else 258 #endif 259 { 260 if (folio_test_anon(folio)) 261 page_add_anon_rmap(new, vma, pvmw.address, 262 rmap_flags); 263 else 264 page_add_file_rmap(new, vma, false); 265 set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte); 266 } 267 if (vma->vm_flags & VM_LOCKED) 268 mlock_page_drain_local(); 269 270 trace_remove_migration_pte(pvmw.address, pte_val(pte), 271 compound_order(new)); 272 273 /* No need to invalidate - it was non-present before */ 274 update_mmu_cache(vma, pvmw.address, pvmw.pte); 275 } 276 277 return true; 278 } 279 280 /* 281 * Get rid of all migration entries and replace them by 282 * references to the indicated page. 283 */ 284 void remove_migration_ptes(struct folio *src, struct folio *dst, bool locked) 285 { 286 struct rmap_walk_control rwc = { 287 .rmap_one = remove_migration_pte, 288 .arg = src, 289 }; 290 291 if (locked) 292 rmap_walk_locked(dst, &rwc); 293 else 294 rmap_walk(dst, &rwc); 295 } 296 297 /* 298 * Something used the pte of a page under migration. We need to 299 * get to the page and wait until migration is finished. 300 * When we return from this function the fault will be retried. 301 */ 302 void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep, 303 spinlock_t *ptl) 304 { 305 pte_t pte; 306 swp_entry_t entry; 307 308 spin_lock(ptl); 309 pte = *ptep; 310 if (!is_swap_pte(pte)) 311 goto out; 312 313 entry = pte_to_swp_entry(pte); 314 if (!is_migration_entry(entry)) 315 goto out; 316 317 migration_entry_wait_on_locked(entry, ptep, ptl); 318 return; 319 out: 320 pte_unmap_unlock(ptep, ptl); 321 } 322 323 void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, 324 unsigned long address) 325 { 326 spinlock_t *ptl = pte_lockptr(mm, pmd); 327 pte_t *ptep = pte_offset_map(pmd, address); 328 __migration_entry_wait(mm, ptep, ptl); 329 } 330 331 #ifdef CONFIG_HUGETLB_PAGE 332 void __migration_entry_wait_huge(pte_t *ptep, spinlock_t *ptl) 333 { 334 pte_t pte; 335 336 spin_lock(ptl); 337 pte = huge_ptep_get(ptep); 338 339 if (unlikely(!is_hugetlb_entry_migration(pte))) 340 spin_unlock(ptl); 341 else 342 migration_entry_wait_on_locked(pte_to_swp_entry(pte), NULL, ptl); 343 } 344 345 void migration_entry_wait_huge(struct vm_area_struct *vma, pte_t *pte) 346 { 347 spinlock_t *ptl = huge_pte_lockptr(hstate_vma(vma), vma->vm_mm, pte); 348 349 __migration_entry_wait_huge(pte, ptl); 350 } 351 #endif 352 353 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 354 void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd) 355 { 356 spinlock_t *ptl; 357 358 ptl = pmd_lock(mm, pmd); 359 if (!is_pmd_migration_entry(*pmd)) 360 goto unlock; 361 migration_entry_wait_on_locked(pmd_to_swp_entry(*pmd), NULL, ptl); 362 return; 363 unlock: 364 spin_unlock(ptl); 365 } 366 #endif 367 368 static int folio_expected_refs(struct address_space *mapping, 369 struct folio *folio) 370 { 371 int refs = 1; 372 if (!mapping) 373 return refs; 374 375 refs += folio_nr_pages(folio); 376 if (folio_test_private(folio)) 377 refs++; 378 379 return refs; 380 } 381 382 /* 383 * Replace the page in the mapping. 384 * 385 * The number of remaining references must be: 386 * 1 for anonymous pages without a mapping 387 * 2 for pages with a mapping 388 * 3 for pages with a mapping and PagePrivate/PagePrivate2 set. 389 */ 390 int folio_migrate_mapping(struct address_space *mapping, 391 struct folio *newfolio, struct folio *folio, int extra_count) 392 { 393 XA_STATE(xas, &mapping->i_pages, folio_index(folio)); 394 struct zone *oldzone, *newzone; 395 int dirty; 396 int expected_count = folio_expected_refs(mapping, folio) + extra_count; 397 long nr = folio_nr_pages(folio); 398 399 if (!mapping) { 400 /* Anonymous page without mapping */ 401 if (folio_ref_count(folio) != expected_count) 402 return -EAGAIN; 403 404 /* No turning back from here */ 405 newfolio->index = folio->index; 406 newfolio->mapping = folio->mapping; 407 if (folio_test_swapbacked(folio)) 408 __folio_set_swapbacked(newfolio); 409 410 return MIGRATEPAGE_SUCCESS; 411 } 412 413 oldzone = folio_zone(folio); 414 newzone = folio_zone(newfolio); 415 416 xas_lock_irq(&xas); 417 if (!folio_ref_freeze(folio, expected_count)) { 418 xas_unlock_irq(&xas); 419 return -EAGAIN; 420 } 421 422 /* 423 * Now we know that no one else is looking at the folio: 424 * no turning back from here. 425 */ 426 newfolio->index = folio->index; 427 newfolio->mapping = folio->mapping; 428 folio_ref_add(newfolio, nr); /* add cache reference */ 429 if (folio_test_swapbacked(folio)) { 430 __folio_set_swapbacked(newfolio); 431 if (folio_test_swapcache(folio)) { 432 folio_set_swapcache(newfolio); 433 newfolio->private = folio_get_private(folio); 434 } 435 } else { 436 VM_BUG_ON_FOLIO(folio_test_swapcache(folio), folio); 437 } 438 439 /* Move dirty while page refs frozen and newpage not yet exposed */ 440 dirty = folio_test_dirty(folio); 441 if (dirty) { 442 folio_clear_dirty(folio); 443 folio_set_dirty(newfolio); 444 } 445 446 xas_store(&xas, newfolio); 447 448 /* 449 * Drop cache reference from old page by unfreezing 450 * to one less reference. 451 * We know this isn't the last reference. 452 */ 453 folio_ref_unfreeze(folio, expected_count - nr); 454 455 xas_unlock(&xas); 456 /* Leave irq disabled to prevent preemption while updating stats */ 457 458 /* 459 * If moved to a different zone then also account 460 * the page for that zone. Other VM counters will be 461 * taken care of when we establish references to the 462 * new page and drop references to the old page. 463 * 464 * Note that anonymous pages are accounted for 465 * via NR_FILE_PAGES and NR_ANON_MAPPED if they 466 * are mapped to swap space. 467 */ 468 if (newzone != oldzone) { 469 struct lruvec *old_lruvec, *new_lruvec; 470 struct mem_cgroup *memcg; 471 472 memcg = folio_memcg(folio); 473 old_lruvec = mem_cgroup_lruvec(memcg, oldzone->zone_pgdat); 474 new_lruvec = mem_cgroup_lruvec(memcg, newzone->zone_pgdat); 475 476 __mod_lruvec_state(old_lruvec, NR_FILE_PAGES, -nr); 477 __mod_lruvec_state(new_lruvec, NR_FILE_PAGES, nr); 478 if (folio_test_swapbacked(folio) && !folio_test_swapcache(folio)) { 479 __mod_lruvec_state(old_lruvec, NR_SHMEM, -nr); 480 __mod_lruvec_state(new_lruvec, NR_SHMEM, nr); 481 } 482 #ifdef CONFIG_SWAP 483 if (folio_test_swapcache(folio)) { 484 __mod_lruvec_state(old_lruvec, NR_SWAPCACHE, -nr); 485 __mod_lruvec_state(new_lruvec, NR_SWAPCACHE, nr); 486 } 487 #endif 488 if (dirty && mapping_can_writeback(mapping)) { 489 __mod_lruvec_state(old_lruvec, NR_FILE_DIRTY, -nr); 490 __mod_zone_page_state(oldzone, NR_ZONE_WRITE_PENDING, -nr); 491 __mod_lruvec_state(new_lruvec, NR_FILE_DIRTY, nr); 492 __mod_zone_page_state(newzone, NR_ZONE_WRITE_PENDING, nr); 493 } 494 } 495 local_irq_enable(); 496 497 return MIGRATEPAGE_SUCCESS; 498 } 499 EXPORT_SYMBOL(folio_migrate_mapping); 500 501 /* 502 * The expected number of remaining references is the same as that 503 * of folio_migrate_mapping(). 504 */ 505 int migrate_huge_page_move_mapping(struct address_space *mapping, 506 struct folio *dst, struct folio *src) 507 { 508 XA_STATE(xas, &mapping->i_pages, folio_index(src)); 509 int expected_count; 510 511 xas_lock_irq(&xas); 512 expected_count = 2 + folio_has_private(src); 513 if (!folio_ref_freeze(src, expected_count)) { 514 xas_unlock_irq(&xas); 515 return -EAGAIN; 516 } 517 518 dst->index = src->index; 519 dst->mapping = src->mapping; 520 521 folio_get(dst); 522 523 xas_store(&xas, dst); 524 525 folio_ref_unfreeze(src, expected_count - 1); 526 527 xas_unlock_irq(&xas); 528 529 return MIGRATEPAGE_SUCCESS; 530 } 531 532 /* 533 * Copy the flags and some other ancillary information 534 */ 535 void folio_migrate_flags(struct folio *newfolio, struct folio *folio) 536 { 537 int cpupid; 538 539 if (folio_test_error(folio)) 540 folio_set_error(newfolio); 541 if (folio_test_referenced(folio)) 542 folio_set_referenced(newfolio); 543 if (folio_test_uptodate(folio)) 544 folio_mark_uptodate(newfolio); 545 if (folio_test_clear_active(folio)) { 546 VM_BUG_ON_FOLIO(folio_test_unevictable(folio), folio); 547 folio_set_active(newfolio); 548 } else if (folio_test_clear_unevictable(folio)) 549 folio_set_unevictable(newfolio); 550 if (folio_test_workingset(folio)) 551 folio_set_workingset(newfolio); 552 if (folio_test_checked(folio)) 553 folio_set_checked(newfolio); 554 /* 555 * PG_anon_exclusive (-> PG_mappedtodisk) is always migrated via 556 * migration entries. We can still have PG_anon_exclusive set on an 557 * effectively unmapped and unreferenced first sub-pages of an 558 * anonymous THP: we can simply copy it here via PG_mappedtodisk. 559 */ 560 if (folio_test_mappedtodisk(folio)) 561 folio_set_mappedtodisk(newfolio); 562 563 /* Move dirty on pages not done by folio_migrate_mapping() */ 564 if (folio_test_dirty(folio)) 565 folio_set_dirty(newfolio); 566 567 if (folio_test_young(folio)) 568 folio_set_young(newfolio); 569 if (folio_test_idle(folio)) 570 folio_set_idle(newfolio); 571 572 /* 573 * Copy NUMA information to the new page, to prevent over-eager 574 * future migrations of this same page. 575 */ 576 cpupid = page_cpupid_xchg_last(&folio->page, -1); 577 /* 578 * For memory tiering mode, when migrate between slow and fast 579 * memory node, reset cpupid, because that is used to record 580 * page access time in slow memory node. 581 */ 582 if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) { 583 bool f_toptier = node_is_toptier(page_to_nid(&folio->page)); 584 bool t_toptier = node_is_toptier(page_to_nid(&newfolio->page)); 585 586 if (f_toptier != t_toptier) 587 cpupid = -1; 588 } 589 page_cpupid_xchg_last(&newfolio->page, cpupid); 590 591 folio_migrate_ksm(newfolio, folio); 592 /* 593 * Please do not reorder this without considering how mm/ksm.c's 594 * get_ksm_page() depends upon ksm_migrate_page() and PageSwapCache(). 595 */ 596 if (folio_test_swapcache(folio)) 597 folio_clear_swapcache(folio); 598 folio_clear_private(folio); 599 600 /* page->private contains hugetlb specific flags */ 601 if (!folio_test_hugetlb(folio)) 602 folio->private = NULL; 603 604 /* 605 * If any waiters have accumulated on the new page then 606 * wake them up. 607 */ 608 if (folio_test_writeback(newfolio)) 609 folio_end_writeback(newfolio); 610 611 /* 612 * PG_readahead shares the same bit with PG_reclaim. The above 613 * end_page_writeback() may clear PG_readahead mistakenly, so set the 614 * bit after that. 615 */ 616 if (folio_test_readahead(folio)) 617 folio_set_readahead(newfolio); 618 619 folio_copy_owner(newfolio, folio); 620 621 if (!folio_test_hugetlb(folio)) 622 mem_cgroup_migrate(folio, newfolio); 623 } 624 EXPORT_SYMBOL(folio_migrate_flags); 625 626 void folio_migrate_copy(struct folio *newfolio, struct folio *folio) 627 { 628 folio_copy(newfolio, folio); 629 folio_migrate_flags(newfolio, folio); 630 } 631 EXPORT_SYMBOL(folio_migrate_copy); 632 633 /************************************************************ 634 * Migration functions 635 ***********************************************************/ 636 637 int migrate_folio_extra(struct address_space *mapping, struct folio *dst, 638 struct folio *src, enum migrate_mode mode, int extra_count) 639 { 640 int rc; 641 642 BUG_ON(folio_test_writeback(src)); /* Writeback must be complete */ 643 644 rc = folio_migrate_mapping(mapping, dst, src, extra_count); 645 646 if (rc != MIGRATEPAGE_SUCCESS) 647 return rc; 648 649 if (mode != MIGRATE_SYNC_NO_COPY) 650 folio_migrate_copy(dst, src); 651 else 652 folio_migrate_flags(dst, src); 653 return MIGRATEPAGE_SUCCESS; 654 } 655 656 /** 657 * migrate_folio() - Simple folio migration. 658 * @mapping: The address_space containing the folio. 659 * @dst: The folio to migrate the data to. 660 * @src: The folio containing the current data. 661 * @mode: How to migrate the page. 662 * 663 * Common logic to directly migrate a single LRU folio suitable for 664 * folios that do not use PagePrivate/PagePrivate2. 665 * 666 * Folios are locked upon entry and exit. 667 */ 668 int migrate_folio(struct address_space *mapping, struct folio *dst, 669 struct folio *src, enum migrate_mode mode) 670 { 671 return migrate_folio_extra(mapping, dst, src, mode, 0); 672 } 673 EXPORT_SYMBOL(migrate_folio); 674 675 #ifdef CONFIG_BLOCK 676 /* Returns true if all buffers are successfully locked */ 677 static bool buffer_migrate_lock_buffers(struct buffer_head *head, 678 enum migrate_mode mode) 679 { 680 struct buffer_head *bh = head; 681 682 /* Simple case, sync compaction */ 683 if (mode != MIGRATE_ASYNC) { 684 do { 685 lock_buffer(bh); 686 bh = bh->b_this_page; 687 688 } while (bh != head); 689 690 return true; 691 } 692 693 /* async case, we cannot block on lock_buffer so use trylock_buffer */ 694 do { 695 if (!trylock_buffer(bh)) { 696 /* 697 * We failed to lock the buffer and cannot stall in 698 * async migration. Release the taken locks 699 */ 700 struct buffer_head *failed_bh = bh; 701 bh = head; 702 while (bh != failed_bh) { 703 unlock_buffer(bh); 704 bh = bh->b_this_page; 705 } 706 return false; 707 } 708 709 bh = bh->b_this_page; 710 } while (bh != head); 711 return true; 712 } 713 714 static int __buffer_migrate_folio(struct address_space *mapping, 715 struct folio *dst, struct folio *src, enum migrate_mode mode, 716 bool check_refs) 717 { 718 struct buffer_head *bh, *head; 719 int rc; 720 int expected_count; 721 722 head = folio_buffers(src); 723 if (!head) 724 return migrate_folio(mapping, dst, src, mode); 725 726 /* Check whether page does not have extra refs before we do more work */ 727 expected_count = folio_expected_refs(mapping, src); 728 if (folio_ref_count(src) != expected_count) 729 return -EAGAIN; 730 731 if (!buffer_migrate_lock_buffers(head, mode)) 732 return -EAGAIN; 733 734 if (check_refs) { 735 bool busy; 736 bool invalidated = false; 737 738 recheck_buffers: 739 busy = false; 740 spin_lock(&mapping->private_lock); 741 bh = head; 742 do { 743 if (atomic_read(&bh->b_count)) { 744 busy = true; 745 break; 746 } 747 bh = bh->b_this_page; 748 } while (bh != head); 749 if (busy) { 750 if (invalidated) { 751 rc = -EAGAIN; 752 goto unlock_buffers; 753 } 754 spin_unlock(&mapping->private_lock); 755 invalidate_bh_lrus(); 756 invalidated = true; 757 goto recheck_buffers; 758 } 759 } 760 761 rc = folio_migrate_mapping(mapping, dst, src, 0); 762 if (rc != MIGRATEPAGE_SUCCESS) 763 goto unlock_buffers; 764 765 folio_attach_private(dst, folio_detach_private(src)); 766 767 bh = head; 768 do { 769 set_bh_page(bh, &dst->page, bh_offset(bh)); 770 bh = bh->b_this_page; 771 } while (bh != head); 772 773 if (mode != MIGRATE_SYNC_NO_COPY) 774 folio_migrate_copy(dst, src); 775 else 776 folio_migrate_flags(dst, src); 777 778 rc = MIGRATEPAGE_SUCCESS; 779 unlock_buffers: 780 if (check_refs) 781 spin_unlock(&mapping->private_lock); 782 bh = head; 783 do { 784 unlock_buffer(bh); 785 bh = bh->b_this_page; 786 } while (bh != head); 787 788 return rc; 789 } 790 791 /** 792 * buffer_migrate_folio() - Migration function for folios with buffers. 793 * @mapping: The address space containing @src. 794 * @dst: The folio to migrate to. 795 * @src: The folio to migrate from. 796 * @mode: How to migrate the folio. 797 * 798 * This function can only be used if the underlying filesystem guarantees 799 * that no other references to @src exist. For example attached buffer 800 * heads are accessed only under the folio lock. If your filesystem cannot 801 * provide this guarantee, buffer_migrate_folio_norefs() may be more 802 * appropriate. 803 * 804 * Return: 0 on success or a negative errno on failure. 805 */ 806 int buffer_migrate_folio(struct address_space *mapping, 807 struct folio *dst, struct folio *src, enum migrate_mode mode) 808 { 809 return __buffer_migrate_folio(mapping, dst, src, mode, false); 810 } 811 EXPORT_SYMBOL(buffer_migrate_folio); 812 813 /** 814 * buffer_migrate_folio_norefs() - Migration function for folios with buffers. 815 * @mapping: The address space containing @src. 816 * @dst: The folio to migrate to. 817 * @src: The folio to migrate from. 818 * @mode: How to migrate the folio. 819 * 820 * Like buffer_migrate_folio() except that this variant is more careful 821 * and checks that there are also no buffer head references. This function 822 * is the right one for mappings where buffer heads are directly looked 823 * up and referenced (such as block device mappings). 824 * 825 * Return: 0 on success or a negative errno on failure. 826 */ 827 int buffer_migrate_folio_norefs(struct address_space *mapping, 828 struct folio *dst, struct folio *src, enum migrate_mode mode) 829 { 830 return __buffer_migrate_folio(mapping, dst, src, mode, true); 831 } 832 #endif 833 834 int filemap_migrate_folio(struct address_space *mapping, 835 struct folio *dst, struct folio *src, enum migrate_mode mode) 836 { 837 int ret; 838 839 ret = folio_migrate_mapping(mapping, dst, src, 0); 840 if (ret != MIGRATEPAGE_SUCCESS) 841 return ret; 842 843 if (folio_get_private(src)) 844 folio_attach_private(dst, folio_detach_private(src)); 845 846 if (mode != MIGRATE_SYNC_NO_COPY) 847 folio_migrate_copy(dst, src); 848 else 849 folio_migrate_flags(dst, src); 850 return MIGRATEPAGE_SUCCESS; 851 } 852 EXPORT_SYMBOL_GPL(filemap_migrate_folio); 853 854 /* 855 * Writeback a folio to clean the dirty state 856 */ 857 static int writeout(struct address_space *mapping, struct folio *folio) 858 { 859 struct writeback_control wbc = { 860 .sync_mode = WB_SYNC_NONE, 861 .nr_to_write = 1, 862 .range_start = 0, 863 .range_end = LLONG_MAX, 864 .for_reclaim = 1 865 }; 866 int rc; 867 868 if (!mapping->a_ops->writepage) 869 /* No write method for the address space */ 870 return -EINVAL; 871 872 if (!folio_clear_dirty_for_io(folio)) 873 /* Someone else already triggered a write */ 874 return -EAGAIN; 875 876 /* 877 * A dirty folio may imply that the underlying filesystem has 878 * the folio on some queue. So the folio must be clean for 879 * migration. Writeout may mean we lose the lock and the 880 * folio state is no longer what we checked for earlier. 881 * At this point we know that the migration attempt cannot 882 * be successful. 883 */ 884 remove_migration_ptes(folio, folio, false); 885 886 rc = mapping->a_ops->writepage(&folio->page, &wbc); 887 888 if (rc != AOP_WRITEPAGE_ACTIVATE) 889 /* unlocked. Relock */ 890 folio_lock(folio); 891 892 return (rc < 0) ? -EIO : -EAGAIN; 893 } 894 895 /* 896 * Default handling if a filesystem does not provide a migration function. 897 */ 898 static int fallback_migrate_folio(struct address_space *mapping, 899 struct folio *dst, struct folio *src, enum migrate_mode mode) 900 { 901 if (folio_test_dirty(src)) { 902 /* Only writeback folios in full synchronous migration */ 903 switch (mode) { 904 case MIGRATE_SYNC: 905 case MIGRATE_SYNC_NO_COPY: 906 break; 907 default: 908 return -EBUSY; 909 } 910 return writeout(mapping, src); 911 } 912 913 /* 914 * Buffers may be managed in a filesystem specific way. 915 * We must have no buffers or drop them. 916 */ 917 if (folio_test_private(src) && 918 !filemap_release_folio(src, GFP_KERNEL)) 919 return mode == MIGRATE_SYNC ? -EAGAIN : -EBUSY; 920 921 return migrate_folio(mapping, dst, src, mode); 922 } 923 924 /* 925 * Move a page to a newly allocated page 926 * The page is locked and all ptes have been successfully removed. 927 * 928 * The new page will have replaced the old page if this function 929 * is successful. 930 * 931 * Return value: 932 * < 0 - error code 933 * MIGRATEPAGE_SUCCESS - success 934 */ 935 static int move_to_new_folio(struct folio *dst, struct folio *src, 936 enum migrate_mode mode) 937 { 938 int rc = -EAGAIN; 939 bool is_lru = !__PageMovable(&src->page); 940 941 VM_BUG_ON_FOLIO(!folio_test_locked(src), src); 942 VM_BUG_ON_FOLIO(!folio_test_locked(dst), dst); 943 944 if (likely(is_lru)) { 945 struct address_space *mapping = folio_mapping(src); 946 947 if (!mapping) 948 rc = migrate_folio(mapping, dst, src, mode); 949 else if (mapping->a_ops->migrate_folio) 950 /* 951 * Most folios have a mapping and most filesystems 952 * provide a migrate_folio callback. Anonymous folios 953 * are part of swap space which also has its own 954 * migrate_folio callback. This is the most common path 955 * for page migration. 956 */ 957 rc = mapping->a_ops->migrate_folio(mapping, dst, src, 958 mode); 959 else 960 rc = fallback_migrate_folio(mapping, dst, src, mode); 961 } else { 962 const struct movable_operations *mops; 963 964 /* 965 * In case of non-lru page, it could be released after 966 * isolation step. In that case, we shouldn't try migration. 967 */ 968 VM_BUG_ON_FOLIO(!folio_test_isolated(src), src); 969 if (!folio_test_movable(src)) { 970 rc = MIGRATEPAGE_SUCCESS; 971 folio_clear_isolated(src); 972 goto out; 973 } 974 975 mops = page_movable_ops(&src->page); 976 rc = mops->migrate_page(&dst->page, &src->page, mode); 977 WARN_ON_ONCE(rc == MIGRATEPAGE_SUCCESS && 978 !folio_test_isolated(src)); 979 } 980 981 /* 982 * When successful, old pagecache src->mapping must be cleared before 983 * src is freed; but stats require that PageAnon be left as PageAnon. 984 */ 985 if (rc == MIGRATEPAGE_SUCCESS) { 986 if (__PageMovable(&src->page)) { 987 VM_BUG_ON_FOLIO(!folio_test_isolated(src), src); 988 989 /* 990 * We clear PG_movable under page_lock so any compactor 991 * cannot try to migrate this page. 992 */ 993 folio_clear_isolated(src); 994 } 995 996 /* 997 * Anonymous and movable src->mapping will be cleared by 998 * free_pages_prepare so don't reset it here for keeping 999 * the type to work PageAnon, for example. 1000 */ 1001 if (!folio_mapping_flags(src)) 1002 src->mapping = NULL; 1003 1004 if (likely(!folio_is_zone_device(dst))) 1005 flush_dcache_folio(dst); 1006 } 1007 out: 1008 return rc; 1009 } 1010 1011 static int __unmap_and_move(struct folio *src, struct folio *dst, 1012 int force, enum migrate_mode mode) 1013 { 1014 int rc = -EAGAIN; 1015 bool page_was_mapped = false; 1016 struct anon_vma *anon_vma = NULL; 1017 bool is_lru = !__PageMovable(&src->page); 1018 1019 if (!folio_trylock(src)) { 1020 if (!force || mode == MIGRATE_ASYNC) 1021 goto out; 1022 1023 /* 1024 * It's not safe for direct compaction to call lock_page. 1025 * For example, during page readahead pages are added locked 1026 * to the LRU. Later, when the IO completes the pages are 1027 * marked uptodate and unlocked. However, the queueing 1028 * could be merging multiple pages for one bio (e.g. 1029 * mpage_readahead). If an allocation happens for the 1030 * second or third page, the process can end up locking 1031 * the same page twice and deadlocking. Rather than 1032 * trying to be clever about what pages can be locked, 1033 * avoid the use of lock_page for direct compaction 1034 * altogether. 1035 */ 1036 if (current->flags & PF_MEMALLOC) 1037 goto out; 1038 1039 folio_lock(src); 1040 } 1041 1042 if (folio_test_writeback(src)) { 1043 /* 1044 * Only in the case of a full synchronous migration is it 1045 * necessary to wait for PageWriteback. In the async case, 1046 * the retry loop is too short and in the sync-light case, 1047 * the overhead of stalling is too much 1048 */ 1049 switch (mode) { 1050 case MIGRATE_SYNC: 1051 case MIGRATE_SYNC_NO_COPY: 1052 break; 1053 default: 1054 rc = -EBUSY; 1055 goto out_unlock; 1056 } 1057 if (!force) 1058 goto out_unlock; 1059 folio_wait_writeback(src); 1060 } 1061 1062 /* 1063 * By try_to_migrate(), src->mapcount goes down to 0 here. In this case, 1064 * we cannot notice that anon_vma is freed while we migrate a page. 1065 * This get_anon_vma() delays freeing anon_vma pointer until the end 1066 * of migration. File cache pages are no problem because of page_lock() 1067 * File Caches may use write_page() or lock_page() in migration, then, 1068 * just care Anon page here. 1069 * 1070 * Only folio_get_anon_vma() understands the subtleties of 1071 * getting a hold on an anon_vma from outside one of its mms. 1072 * But if we cannot get anon_vma, then we won't need it anyway, 1073 * because that implies that the anon page is no longer mapped 1074 * (and cannot be remapped so long as we hold the page lock). 1075 */ 1076 if (folio_test_anon(src) && !folio_test_ksm(src)) 1077 anon_vma = folio_get_anon_vma(src); 1078 1079 /* 1080 * Block others from accessing the new page when we get around to 1081 * establishing additional references. We are usually the only one 1082 * holding a reference to dst at this point. We used to have a BUG 1083 * here if folio_trylock(dst) fails, but would like to allow for 1084 * cases where there might be a race with the previous use of dst. 1085 * This is much like races on refcount of oldpage: just don't BUG(). 1086 */ 1087 if (unlikely(!folio_trylock(dst))) 1088 goto out_unlock; 1089 1090 if (unlikely(!is_lru)) { 1091 rc = move_to_new_folio(dst, src, mode); 1092 goto out_unlock_both; 1093 } 1094 1095 /* 1096 * Corner case handling: 1097 * 1. When a new swap-cache page is read into, it is added to the LRU 1098 * and treated as swapcache but it has no rmap yet. 1099 * Calling try_to_unmap() against a src->mapping==NULL page will 1100 * trigger a BUG. So handle it here. 1101 * 2. An orphaned page (see truncate_cleanup_page) might have 1102 * fs-private metadata. The page can be picked up due to memory 1103 * offlining. Everywhere else except page reclaim, the page is 1104 * invisible to the vm, so the page can not be migrated. So try to 1105 * free the metadata, so the page can be freed. 1106 */ 1107 if (!src->mapping) { 1108 if (folio_test_private(src)) { 1109 try_to_free_buffers(src); 1110 goto out_unlock_both; 1111 } 1112 } else if (folio_mapped(src)) { 1113 /* Establish migration ptes */ 1114 VM_BUG_ON_FOLIO(folio_test_anon(src) && 1115 !folio_test_ksm(src) && !anon_vma, src); 1116 try_to_migrate(src, 0); 1117 page_was_mapped = true; 1118 } 1119 1120 if (!folio_mapped(src)) 1121 rc = move_to_new_folio(dst, src, mode); 1122 1123 /* 1124 * When successful, push dst to LRU immediately: so that if it 1125 * turns out to be an mlocked page, remove_migration_ptes() will 1126 * automatically build up the correct dst->mlock_count for it. 1127 * 1128 * We would like to do something similar for the old page, when 1129 * unsuccessful, and other cases when a page has been temporarily 1130 * isolated from the unevictable LRU: but this case is the easiest. 1131 */ 1132 if (rc == MIGRATEPAGE_SUCCESS) { 1133 folio_add_lru(dst); 1134 if (page_was_mapped) 1135 lru_add_drain(); 1136 } 1137 1138 if (page_was_mapped) 1139 remove_migration_ptes(src, 1140 rc == MIGRATEPAGE_SUCCESS ? dst : src, false); 1141 1142 out_unlock_both: 1143 folio_unlock(dst); 1144 out_unlock: 1145 /* Drop an anon_vma reference if we took one */ 1146 if (anon_vma) 1147 put_anon_vma(anon_vma); 1148 folio_unlock(src); 1149 out: 1150 /* 1151 * If migration is successful, decrease refcount of dst, 1152 * which will not free the page because new page owner increased 1153 * refcounter. 1154 */ 1155 if (rc == MIGRATEPAGE_SUCCESS) 1156 folio_put(dst); 1157 1158 return rc; 1159 } 1160 1161 /* 1162 * Obtain the lock on page, remove all ptes and migrate the page 1163 * to the newly allocated page in newpage. 1164 */ 1165 static int unmap_and_move(new_page_t get_new_page, 1166 free_page_t put_new_page, 1167 unsigned long private, struct page *page, 1168 int force, enum migrate_mode mode, 1169 enum migrate_reason reason, 1170 struct list_head *ret) 1171 { 1172 struct folio *dst, *src = page_folio(page); 1173 int rc = MIGRATEPAGE_SUCCESS; 1174 struct page *newpage = NULL; 1175 1176 if (!thp_migration_supported() && PageTransHuge(page)) 1177 return -ENOSYS; 1178 1179 if (page_count(page) == 1) { 1180 /* Page was freed from under us. So we are done. */ 1181 ClearPageActive(page); 1182 ClearPageUnevictable(page); 1183 /* free_pages_prepare() will clear PG_isolated. */ 1184 goto out; 1185 } 1186 1187 newpage = get_new_page(page, private); 1188 if (!newpage) 1189 return -ENOMEM; 1190 dst = page_folio(newpage); 1191 1192 newpage->private = 0; 1193 rc = __unmap_and_move(src, dst, force, mode); 1194 if (rc == MIGRATEPAGE_SUCCESS) 1195 set_page_owner_migrate_reason(newpage, reason); 1196 1197 out: 1198 if (rc != -EAGAIN) { 1199 /* 1200 * A page that has been migrated has all references 1201 * removed and will be freed. A page that has not been 1202 * migrated will have kept its references and be restored. 1203 */ 1204 list_del(&page->lru); 1205 } 1206 1207 /* 1208 * If migration is successful, releases reference grabbed during 1209 * isolation. Otherwise, restore the page to right list unless 1210 * we want to retry. 1211 */ 1212 if (rc == MIGRATEPAGE_SUCCESS) { 1213 /* 1214 * Compaction can migrate also non-LRU pages which are 1215 * not accounted to NR_ISOLATED_*. They can be recognized 1216 * as __PageMovable 1217 */ 1218 if (likely(!__PageMovable(page))) 1219 mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + 1220 page_is_file_lru(page), -thp_nr_pages(page)); 1221 1222 if (reason != MR_MEMORY_FAILURE) 1223 /* 1224 * We release the page in page_handle_poison. 1225 */ 1226 put_page(page); 1227 } else { 1228 if (rc != -EAGAIN) 1229 list_add_tail(&page->lru, ret); 1230 1231 if (put_new_page) 1232 put_new_page(newpage, private); 1233 else 1234 put_page(newpage); 1235 } 1236 1237 return rc; 1238 } 1239 1240 /* 1241 * Counterpart of unmap_and_move_page() for hugepage migration. 1242 * 1243 * This function doesn't wait the completion of hugepage I/O 1244 * because there is no race between I/O and migration for hugepage. 1245 * Note that currently hugepage I/O occurs only in direct I/O 1246 * where no lock is held and PG_writeback is irrelevant, 1247 * and writeback status of all subpages are counted in the reference 1248 * count of the head page (i.e. if all subpages of a 2MB hugepage are 1249 * under direct I/O, the reference of the head page is 512 and a bit more.) 1250 * This means that when we try to migrate hugepage whose subpages are 1251 * doing direct I/O, some references remain after try_to_unmap() and 1252 * hugepage migration fails without data corruption. 1253 * 1254 * There is also no race when direct I/O is issued on the page under migration, 1255 * because then pte is replaced with migration swap entry and direct I/O code 1256 * will wait in the page fault for migration to complete. 1257 */ 1258 static int unmap_and_move_huge_page(new_page_t get_new_page, 1259 free_page_t put_new_page, unsigned long private, 1260 struct page *hpage, int force, 1261 enum migrate_mode mode, int reason, 1262 struct list_head *ret) 1263 { 1264 struct folio *dst, *src = page_folio(hpage); 1265 int rc = -EAGAIN; 1266 int page_was_mapped = 0; 1267 struct page *new_hpage; 1268 struct anon_vma *anon_vma = NULL; 1269 struct address_space *mapping = NULL; 1270 1271 /* 1272 * Migratability of hugepages depends on architectures and their size. 1273 * This check is necessary because some callers of hugepage migration 1274 * like soft offline and memory hotremove don't walk through page 1275 * tables or check whether the hugepage is pmd-based or not before 1276 * kicking migration. 1277 */ 1278 if (!hugepage_migration_supported(page_hstate(hpage))) 1279 return -ENOSYS; 1280 1281 if (folio_ref_count(src) == 1) { 1282 /* page was freed from under us. So we are done. */ 1283 putback_active_hugepage(hpage); 1284 return MIGRATEPAGE_SUCCESS; 1285 } 1286 1287 new_hpage = get_new_page(hpage, private); 1288 if (!new_hpage) 1289 return -ENOMEM; 1290 dst = page_folio(new_hpage); 1291 1292 if (!folio_trylock(src)) { 1293 if (!force) 1294 goto out; 1295 switch (mode) { 1296 case MIGRATE_SYNC: 1297 case MIGRATE_SYNC_NO_COPY: 1298 break; 1299 default: 1300 goto out; 1301 } 1302 folio_lock(src); 1303 } 1304 1305 /* 1306 * Check for pages which are in the process of being freed. Without 1307 * folio_mapping() set, hugetlbfs specific move page routine will not 1308 * be called and we could leak usage counts for subpools. 1309 */ 1310 if (hugetlb_page_subpool(hpage) && !folio_mapping(src)) { 1311 rc = -EBUSY; 1312 goto out_unlock; 1313 } 1314 1315 if (folio_test_anon(src)) 1316 anon_vma = folio_get_anon_vma(src); 1317 1318 if (unlikely(!folio_trylock(dst))) 1319 goto put_anon; 1320 1321 if (folio_mapped(src)) { 1322 enum ttu_flags ttu = 0; 1323 1324 if (!folio_test_anon(src)) { 1325 /* 1326 * In shared mappings, try_to_unmap could potentially 1327 * call huge_pmd_unshare. Because of this, take 1328 * semaphore in write mode here and set TTU_RMAP_LOCKED 1329 * to let lower levels know we have taken the lock. 1330 */ 1331 mapping = hugetlb_page_mapping_lock_write(hpage); 1332 if (unlikely(!mapping)) 1333 goto unlock_put_anon; 1334 1335 ttu = TTU_RMAP_LOCKED; 1336 } 1337 1338 try_to_migrate(src, ttu); 1339 page_was_mapped = 1; 1340 1341 if (ttu & TTU_RMAP_LOCKED) 1342 i_mmap_unlock_write(mapping); 1343 } 1344 1345 if (!folio_mapped(src)) 1346 rc = move_to_new_folio(dst, src, mode); 1347 1348 if (page_was_mapped) 1349 remove_migration_ptes(src, 1350 rc == MIGRATEPAGE_SUCCESS ? dst : src, false); 1351 1352 unlock_put_anon: 1353 folio_unlock(dst); 1354 1355 put_anon: 1356 if (anon_vma) 1357 put_anon_vma(anon_vma); 1358 1359 if (rc == MIGRATEPAGE_SUCCESS) { 1360 move_hugetlb_state(hpage, new_hpage, reason); 1361 put_new_page = NULL; 1362 } 1363 1364 out_unlock: 1365 folio_unlock(src); 1366 out: 1367 if (rc == MIGRATEPAGE_SUCCESS) 1368 putback_active_hugepage(hpage); 1369 else if (rc != -EAGAIN) 1370 list_move_tail(&src->lru, ret); 1371 1372 /* 1373 * If migration was not successful and there's a freeing callback, use 1374 * it. Otherwise, put_page() will drop the reference grabbed during 1375 * isolation. 1376 */ 1377 if (put_new_page) 1378 put_new_page(new_hpage, private); 1379 else 1380 putback_active_hugepage(new_hpage); 1381 1382 return rc; 1383 } 1384 1385 static inline int try_split_thp(struct page *page, struct list_head *split_pages) 1386 { 1387 int rc; 1388 1389 lock_page(page); 1390 rc = split_huge_page_to_list(page, split_pages); 1391 unlock_page(page); 1392 if (!rc) 1393 list_move_tail(&page->lru, split_pages); 1394 1395 return rc; 1396 } 1397 1398 /* 1399 * migrate_pages - migrate the pages specified in a list, to the free pages 1400 * supplied as the target for the page migration 1401 * 1402 * @from: The list of pages to be migrated. 1403 * @get_new_page: The function used to allocate free pages to be used 1404 * as the target of the page migration. 1405 * @put_new_page: The function used to free target pages if migration 1406 * fails, or NULL if no special handling is necessary. 1407 * @private: Private data to be passed on to get_new_page() 1408 * @mode: The migration mode that specifies the constraints for 1409 * page migration, if any. 1410 * @reason: The reason for page migration. 1411 * @ret_succeeded: Set to the number of normal pages migrated successfully if 1412 * the caller passes a non-NULL pointer. 1413 * 1414 * The function returns after 10 attempts or if no pages are movable any more 1415 * because the list has become empty or no retryable pages exist any more. 1416 * It is caller's responsibility to call putback_movable_pages() to return pages 1417 * to the LRU or free list only if ret != 0. 1418 * 1419 * Returns the number of {normal page, THP, hugetlb} that were not migrated, or 1420 * an error code. The number of THP splits will be considered as the number of 1421 * non-migrated THP, no matter how many subpages of the THP are migrated successfully. 1422 */ 1423 int migrate_pages(struct list_head *from, new_page_t get_new_page, 1424 free_page_t put_new_page, unsigned long private, 1425 enum migrate_mode mode, int reason, unsigned int *ret_succeeded) 1426 { 1427 int retry = 1; 1428 int thp_retry = 1; 1429 int nr_failed = 0; 1430 int nr_failed_pages = 0; 1431 int nr_retry_pages = 0; 1432 int nr_succeeded = 0; 1433 int nr_thp_succeeded = 0; 1434 int nr_thp_failed = 0; 1435 int nr_thp_split = 0; 1436 int pass = 0; 1437 bool is_thp = false; 1438 struct page *page; 1439 struct page *page2; 1440 int rc, nr_subpages; 1441 LIST_HEAD(ret_pages); 1442 LIST_HEAD(thp_split_pages); 1443 bool nosplit = (reason == MR_NUMA_MISPLACED); 1444 bool no_subpage_counting = false; 1445 1446 trace_mm_migrate_pages_start(mode, reason); 1447 1448 thp_subpage_migration: 1449 for (pass = 0; pass < 10 && (retry || thp_retry); pass++) { 1450 retry = 0; 1451 thp_retry = 0; 1452 nr_retry_pages = 0; 1453 1454 list_for_each_entry_safe(page, page2, from, lru) { 1455 /* 1456 * THP statistics is based on the source huge page. 1457 * Capture required information that might get lost 1458 * during migration. 1459 */ 1460 is_thp = PageTransHuge(page) && !PageHuge(page); 1461 nr_subpages = compound_nr(page); 1462 cond_resched(); 1463 1464 if (PageHuge(page)) 1465 rc = unmap_and_move_huge_page(get_new_page, 1466 put_new_page, private, page, 1467 pass > 2, mode, reason, 1468 &ret_pages); 1469 else 1470 rc = unmap_and_move(get_new_page, put_new_page, 1471 private, page, pass > 2, mode, 1472 reason, &ret_pages); 1473 /* 1474 * The rules are: 1475 * Success: non hugetlb page will be freed, hugetlb 1476 * page will be put back 1477 * -EAGAIN: stay on the from list 1478 * -ENOMEM: stay on the from list 1479 * -ENOSYS: stay on the from list 1480 * Other errno: put on ret_pages list then splice to 1481 * from list 1482 */ 1483 switch(rc) { 1484 /* 1485 * THP migration might be unsupported or the 1486 * allocation could've failed so we should 1487 * retry on the same page with the THP split 1488 * to base pages. 1489 * 1490 * Sub-pages are put in thp_split_pages, and 1491 * we will migrate them after the rest of the 1492 * list is processed. 1493 */ 1494 case -ENOSYS: 1495 /* THP migration is unsupported */ 1496 if (is_thp) { 1497 nr_thp_failed++; 1498 if (!try_split_thp(page, &thp_split_pages)) { 1499 nr_thp_split++; 1500 break; 1501 } 1502 /* Hugetlb migration is unsupported */ 1503 } else if (!no_subpage_counting) { 1504 nr_failed++; 1505 } 1506 1507 nr_failed_pages += nr_subpages; 1508 list_move_tail(&page->lru, &ret_pages); 1509 break; 1510 case -ENOMEM: 1511 /* 1512 * When memory is low, don't bother to try to migrate 1513 * other pages, just exit. 1514 */ 1515 if (is_thp) { 1516 nr_thp_failed++; 1517 /* THP NUMA faulting doesn't split THP to retry. */ 1518 if (!nosplit && !try_split_thp(page, &thp_split_pages)) { 1519 nr_thp_split++; 1520 break; 1521 } 1522 } else if (!no_subpage_counting) { 1523 nr_failed++; 1524 } 1525 1526 nr_failed_pages += nr_subpages + nr_retry_pages; 1527 /* 1528 * There might be some subpages of fail-to-migrate THPs 1529 * left in thp_split_pages list. Move them back to migration 1530 * list so that they could be put back to the right list by 1531 * the caller otherwise the page refcnt will be leaked. 1532 */ 1533 list_splice_init(&thp_split_pages, from); 1534 /* nr_failed isn't updated for not used */ 1535 nr_thp_failed += thp_retry; 1536 goto out; 1537 case -EAGAIN: 1538 if (is_thp) 1539 thp_retry++; 1540 else if (!no_subpage_counting) 1541 retry++; 1542 nr_retry_pages += nr_subpages; 1543 break; 1544 case MIGRATEPAGE_SUCCESS: 1545 nr_succeeded += nr_subpages; 1546 if (is_thp) 1547 nr_thp_succeeded++; 1548 break; 1549 default: 1550 /* 1551 * Permanent failure (-EBUSY, etc.): 1552 * unlike -EAGAIN case, the failed page is 1553 * removed from migration page list and not 1554 * retried in the next outer loop. 1555 */ 1556 if (is_thp) 1557 nr_thp_failed++; 1558 else if (!no_subpage_counting) 1559 nr_failed++; 1560 1561 nr_failed_pages += nr_subpages; 1562 break; 1563 } 1564 } 1565 } 1566 nr_failed += retry; 1567 nr_thp_failed += thp_retry; 1568 nr_failed_pages += nr_retry_pages; 1569 /* 1570 * Try to migrate subpages of fail-to-migrate THPs, no nr_failed 1571 * counting in this round, since all subpages of a THP is counted 1572 * as 1 failure in the first round. 1573 */ 1574 if (!list_empty(&thp_split_pages)) { 1575 /* 1576 * Move non-migrated pages (after 10 retries) to ret_pages 1577 * to avoid migrating them again. 1578 */ 1579 list_splice_init(from, &ret_pages); 1580 list_splice_init(&thp_split_pages, from); 1581 no_subpage_counting = true; 1582 retry = 1; 1583 goto thp_subpage_migration; 1584 } 1585 1586 rc = nr_failed + nr_thp_failed; 1587 out: 1588 /* 1589 * Put the permanent failure page back to migration list, they 1590 * will be put back to the right list by the caller. 1591 */ 1592 list_splice(&ret_pages, from); 1593 1594 /* 1595 * Return 0 in case all subpages of fail-to-migrate THPs are 1596 * migrated successfully. 1597 */ 1598 if (list_empty(from)) 1599 rc = 0; 1600 1601 count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded); 1602 count_vm_events(PGMIGRATE_FAIL, nr_failed_pages); 1603 count_vm_events(THP_MIGRATION_SUCCESS, nr_thp_succeeded); 1604 count_vm_events(THP_MIGRATION_FAIL, nr_thp_failed); 1605 count_vm_events(THP_MIGRATION_SPLIT, nr_thp_split); 1606 trace_mm_migrate_pages(nr_succeeded, nr_failed_pages, nr_thp_succeeded, 1607 nr_thp_failed, nr_thp_split, mode, reason); 1608 1609 if (ret_succeeded) 1610 *ret_succeeded = nr_succeeded; 1611 1612 return rc; 1613 } 1614 1615 struct page *alloc_migration_target(struct page *page, unsigned long private) 1616 { 1617 struct folio *folio = page_folio(page); 1618 struct migration_target_control *mtc; 1619 gfp_t gfp_mask; 1620 unsigned int order = 0; 1621 struct folio *new_folio = NULL; 1622 int nid; 1623 int zidx; 1624 1625 mtc = (struct migration_target_control *)private; 1626 gfp_mask = mtc->gfp_mask; 1627 nid = mtc->nid; 1628 if (nid == NUMA_NO_NODE) 1629 nid = folio_nid(folio); 1630 1631 if (folio_test_hugetlb(folio)) { 1632 struct hstate *h = page_hstate(&folio->page); 1633 1634 gfp_mask = htlb_modify_alloc_mask(h, gfp_mask); 1635 return alloc_huge_page_nodemask(h, nid, mtc->nmask, gfp_mask); 1636 } 1637 1638 if (folio_test_large(folio)) { 1639 /* 1640 * clear __GFP_RECLAIM to make the migration callback 1641 * consistent with regular THP allocations. 1642 */ 1643 gfp_mask &= ~__GFP_RECLAIM; 1644 gfp_mask |= GFP_TRANSHUGE; 1645 order = folio_order(folio); 1646 } 1647 zidx = zone_idx(folio_zone(folio)); 1648 if (is_highmem_idx(zidx) || zidx == ZONE_MOVABLE) 1649 gfp_mask |= __GFP_HIGHMEM; 1650 1651 new_folio = __folio_alloc(gfp_mask, order, nid, mtc->nmask); 1652 1653 return &new_folio->page; 1654 } 1655 1656 #ifdef CONFIG_NUMA 1657 1658 static int store_status(int __user *status, int start, int value, int nr) 1659 { 1660 while (nr-- > 0) { 1661 if (put_user(value, status + start)) 1662 return -EFAULT; 1663 start++; 1664 } 1665 1666 return 0; 1667 } 1668 1669 static int do_move_pages_to_node(struct mm_struct *mm, 1670 struct list_head *pagelist, int node) 1671 { 1672 int err; 1673 struct migration_target_control mtc = { 1674 .nid = node, 1675 .gfp_mask = GFP_HIGHUSER_MOVABLE | __GFP_THISNODE, 1676 }; 1677 1678 err = migrate_pages(pagelist, alloc_migration_target, NULL, 1679 (unsigned long)&mtc, MIGRATE_SYNC, MR_SYSCALL, NULL); 1680 if (err) 1681 putback_movable_pages(pagelist); 1682 return err; 1683 } 1684 1685 /* 1686 * Resolves the given address to a struct page, isolates it from the LRU and 1687 * puts it to the given pagelist. 1688 * Returns: 1689 * errno - if the page cannot be found/isolated 1690 * 0 - when it doesn't have to be migrated because it is already on the 1691 * target node 1692 * 1 - when it has been queued 1693 */ 1694 static int add_page_for_migration(struct mm_struct *mm, unsigned long addr, 1695 int node, struct list_head *pagelist, bool migrate_all) 1696 { 1697 struct vm_area_struct *vma; 1698 struct page *page; 1699 int err; 1700 1701 mmap_read_lock(mm); 1702 err = -EFAULT; 1703 vma = vma_lookup(mm, addr); 1704 if (!vma || !vma_migratable(vma)) 1705 goto out; 1706 1707 /* FOLL_DUMP to ignore special (like zero) pages */ 1708 page = follow_page(vma, addr, FOLL_GET | FOLL_DUMP); 1709 1710 err = PTR_ERR(page); 1711 if (IS_ERR(page)) 1712 goto out; 1713 1714 err = -ENOENT; 1715 if (!page) 1716 goto out; 1717 1718 if (is_zone_device_page(page)) 1719 goto out_putpage; 1720 1721 err = 0; 1722 if (page_to_nid(page) == node) 1723 goto out_putpage; 1724 1725 err = -EACCES; 1726 if (page_mapcount(page) > 1 && !migrate_all) 1727 goto out_putpage; 1728 1729 if (PageHuge(page)) { 1730 if (PageHead(page)) { 1731 err = isolate_hugetlb(page, pagelist); 1732 if (!err) 1733 err = 1; 1734 } 1735 } else { 1736 struct page *head; 1737 1738 head = compound_head(page); 1739 err = isolate_lru_page(head); 1740 if (err) 1741 goto out_putpage; 1742 1743 err = 1; 1744 list_add_tail(&head->lru, pagelist); 1745 mod_node_page_state(page_pgdat(head), 1746 NR_ISOLATED_ANON + page_is_file_lru(head), 1747 thp_nr_pages(head)); 1748 } 1749 out_putpage: 1750 /* 1751 * Either remove the duplicate refcount from 1752 * isolate_lru_page() or drop the page ref if it was 1753 * not isolated. 1754 */ 1755 put_page(page); 1756 out: 1757 mmap_read_unlock(mm); 1758 return err; 1759 } 1760 1761 static int move_pages_and_store_status(struct mm_struct *mm, int node, 1762 struct list_head *pagelist, int __user *status, 1763 int start, int i, unsigned long nr_pages) 1764 { 1765 int err; 1766 1767 if (list_empty(pagelist)) 1768 return 0; 1769 1770 err = do_move_pages_to_node(mm, pagelist, node); 1771 if (err) { 1772 /* 1773 * Positive err means the number of failed 1774 * pages to migrate. Since we are going to 1775 * abort and return the number of non-migrated 1776 * pages, so need to include the rest of the 1777 * nr_pages that have not been attempted as 1778 * well. 1779 */ 1780 if (err > 0) 1781 err += nr_pages - i; 1782 return err; 1783 } 1784 return store_status(status, start, node, i - start); 1785 } 1786 1787 /* 1788 * Migrate an array of page address onto an array of nodes and fill 1789 * the corresponding array of status. 1790 */ 1791 static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes, 1792 unsigned long nr_pages, 1793 const void __user * __user *pages, 1794 const int __user *nodes, 1795 int __user *status, int flags) 1796 { 1797 int current_node = NUMA_NO_NODE; 1798 LIST_HEAD(pagelist); 1799 int start, i; 1800 int err = 0, err1; 1801 1802 lru_cache_disable(); 1803 1804 for (i = start = 0; i < nr_pages; i++) { 1805 const void __user *p; 1806 unsigned long addr; 1807 int node; 1808 1809 err = -EFAULT; 1810 if (get_user(p, pages + i)) 1811 goto out_flush; 1812 if (get_user(node, nodes + i)) 1813 goto out_flush; 1814 addr = (unsigned long)untagged_addr(p); 1815 1816 err = -ENODEV; 1817 if (node < 0 || node >= MAX_NUMNODES) 1818 goto out_flush; 1819 if (!node_state(node, N_MEMORY)) 1820 goto out_flush; 1821 1822 err = -EACCES; 1823 if (!node_isset(node, task_nodes)) 1824 goto out_flush; 1825 1826 if (current_node == NUMA_NO_NODE) { 1827 current_node = node; 1828 start = i; 1829 } else if (node != current_node) { 1830 err = move_pages_and_store_status(mm, current_node, 1831 &pagelist, status, start, i, nr_pages); 1832 if (err) 1833 goto out; 1834 start = i; 1835 current_node = node; 1836 } 1837 1838 /* 1839 * Errors in the page lookup or isolation are not fatal and we simply 1840 * report them via status 1841 */ 1842 err = add_page_for_migration(mm, addr, current_node, 1843 &pagelist, flags & MPOL_MF_MOVE_ALL); 1844 1845 if (err > 0) { 1846 /* The page is successfully queued for migration */ 1847 continue; 1848 } 1849 1850 /* 1851 * The move_pages() man page does not have an -EEXIST choice, so 1852 * use -EFAULT instead. 1853 */ 1854 if (err == -EEXIST) 1855 err = -EFAULT; 1856 1857 /* 1858 * If the page is already on the target node (!err), store the 1859 * node, otherwise, store the err. 1860 */ 1861 err = store_status(status, i, err ? : current_node, 1); 1862 if (err) 1863 goto out_flush; 1864 1865 err = move_pages_and_store_status(mm, current_node, &pagelist, 1866 status, start, i, nr_pages); 1867 if (err) { 1868 /* We have accounted for page i */ 1869 if (err > 0) 1870 err--; 1871 goto out; 1872 } 1873 current_node = NUMA_NO_NODE; 1874 } 1875 out_flush: 1876 /* Make sure we do not overwrite the existing error */ 1877 err1 = move_pages_and_store_status(mm, current_node, &pagelist, 1878 status, start, i, nr_pages); 1879 if (err >= 0) 1880 err = err1; 1881 out: 1882 lru_cache_enable(); 1883 return err; 1884 } 1885 1886 /* 1887 * Determine the nodes of an array of pages and store it in an array of status. 1888 */ 1889 static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages, 1890 const void __user **pages, int *status) 1891 { 1892 unsigned long i; 1893 1894 mmap_read_lock(mm); 1895 1896 for (i = 0; i < nr_pages; i++) { 1897 unsigned long addr = (unsigned long)(*pages); 1898 unsigned int foll_flags = FOLL_DUMP; 1899 struct vm_area_struct *vma; 1900 struct page *page; 1901 int err = -EFAULT; 1902 1903 vma = vma_lookup(mm, addr); 1904 if (!vma) 1905 goto set_status; 1906 1907 /* Not all huge page follow APIs support 'FOLL_GET' */ 1908 if (!is_vm_hugetlb_page(vma)) 1909 foll_flags |= FOLL_GET; 1910 1911 /* FOLL_DUMP to ignore special (like zero) pages */ 1912 page = follow_page(vma, addr, foll_flags); 1913 1914 err = PTR_ERR(page); 1915 if (IS_ERR(page)) 1916 goto set_status; 1917 1918 err = -ENOENT; 1919 if (!page) 1920 goto set_status; 1921 1922 if (!is_zone_device_page(page)) 1923 err = page_to_nid(page); 1924 1925 if (foll_flags & FOLL_GET) 1926 put_page(page); 1927 set_status: 1928 *status = err; 1929 1930 pages++; 1931 status++; 1932 } 1933 1934 mmap_read_unlock(mm); 1935 } 1936 1937 static int get_compat_pages_array(const void __user *chunk_pages[], 1938 const void __user * __user *pages, 1939 unsigned long chunk_nr) 1940 { 1941 compat_uptr_t __user *pages32 = (compat_uptr_t __user *)pages; 1942 compat_uptr_t p; 1943 int i; 1944 1945 for (i = 0; i < chunk_nr; i++) { 1946 if (get_user(p, pages32 + i)) 1947 return -EFAULT; 1948 chunk_pages[i] = compat_ptr(p); 1949 } 1950 1951 return 0; 1952 } 1953 1954 /* 1955 * Determine the nodes of a user array of pages and store it in 1956 * a user array of status. 1957 */ 1958 static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages, 1959 const void __user * __user *pages, 1960 int __user *status) 1961 { 1962 #define DO_PAGES_STAT_CHUNK_NR 16UL 1963 const void __user *chunk_pages[DO_PAGES_STAT_CHUNK_NR]; 1964 int chunk_status[DO_PAGES_STAT_CHUNK_NR]; 1965 1966 while (nr_pages) { 1967 unsigned long chunk_nr = min(nr_pages, DO_PAGES_STAT_CHUNK_NR); 1968 1969 if (in_compat_syscall()) { 1970 if (get_compat_pages_array(chunk_pages, pages, 1971 chunk_nr)) 1972 break; 1973 } else { 1974 if (copy_from_user(chunk_pages, pages, 1975 chunk_nr * sizeof(*chunk_pages))) 1976 break; 1977 } 1978 1979 do_pages_stat_array(mm, chunk_nr, chunk_pages, chunk_status); 1980 1981 if (copy_to_user(status, chunk_status, chunk_nr * sizeof(*status))) 1982 break; 1983 1984 pages += chunk_nr; 1985 status += chunk_nr; 1986 nr_pages -= chunk_nr; 1987 } 1988 return nr_pages ? -EFAULT : 0; 1989 } 1990 1991 static struct mm_struct *find_mm_struct(pid_t pid, nodemask_t *mem_nodes) 1992 { 1993 struct task_struct *task; 1994 struct mm_struct *mm; 1995 1996 /* 1997 * There is no need to check if current process has the right to modify 1998 * the specified process when they are same. 1999 */ 2000 if (!pid) { 2001 mmget(current->mm); 2002 *mem_nodes = cpuset_mems_allowed(current); 2003 return current->mm; 2004 } 2005 2006 /* Find the mm_struct */ 2007 rcu_read_lock(); 2008 task = find_task_by_vpid(pid); 2009 if (!task) { 2010 rcu_read_unlock(); 2011 return ERR_PTR(-ESRCH); 2012 } 2013 get_task_struct(task); 2014 2015 /* 2016 * Check if this process has the right to modify the specified 2017 * process. Use the regular "ptrace_may_access()" checks. 2018 */ 2019 if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) { 2020 rcu_read_unlock(); 2021 mm = ERR_PTR(-EPERM); 2022 goto out; 2023 } 2024 rcu_read_unlock(); 2025 2026 mm = ERR_PTR(security_task_movememory(task)); 2027 if (IS_ERR(mm)) 2028 goto out; 2029 *mem_nodes = cpuset_mems_allowed(task); 2030 mm = get_task_mm(task); 2031 out: 2032 put_task_struct(task); 2033 if (!mm) 2034 mm = ERR_PTR(-EINVAL); 2035 return mm; 2036 } 2037 2038 /* 2039 * Move a list of pages in the address space of the currently executing 2040 * process. 2041 */ 2042 static int kernel_move_pages(pid_t pid, unsigned long nr_pages, 2043 const void __user * __user *pages, 2044 const int __user *nodes, 2045 int __user *status, int flags) 2046 { 2047 struct mm_struct *mm; 2048 int err; 2049 nodemask_t task_nodes; 2050 2051 /* Check flags */ 2052 if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL)) 2053 return -EINVAL; 2054 2055 if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE)) 2056 return -EPERM; 2057 2058 mm = find_mm_struct(pid, &task_nodes); 2059 if (IS_ERR(mm)) 2060 return PTR_ERR(mm); 2061 2062 if (nodes) 2063 err = do_pages_move(mm, task_nodes, nr_pages, pages, 2064 nodes, status, flags); 2065 else 2066 err = do_pages_stat(mm, nr_pages, pages, status); 2067 2068 mmput(mm); 2069 return err; 2070 } 2071 2072 SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages, 2073 const void __user * __user *, pages, 2074 const int __user *, nodes, 2075 int __user *, status, int, flags) 2076 { 2077 return kernel_move_pages(pid, nr_pages, pages, nodes, status, flags); 2078 } 2079 2080 #ifdef CONFIG_NUMA_BALANCING 2081 /* 2082 * Returns true if this is a safe migration target node for misplaced NUMA 2083 * pages. Currently it only checks the watermarks which is crude. 2084 */ 2085 static bool migrate_balanced_pgdat(struct pglist_data *pgdat, 2086 unsigned long nr_migrate_pages) 2087 { 2088 int z; 2089 2090 for (z = pgdat->nr_zones - 1; z >= 0; z--) { 2091 struct zone *zone = pgdat->node_zones + z; 2092 2093 if (!managed_zone(zone)) 2094 continue; 2095 2096 /* Avoid waking kswapd by allocating pages_to_migrate pages. */ 2097 if (!zone_watermark_ok(zone, 0, 2098 high_wmark_pages(zone) + 2099 nr_migrate_pages, 2100 ZONE_MOVABLE, 0)) 2101 continue; 2102 return true; 2103 } 2104 return false; 2105 } 2106 2107 static struct page *alloc_misplaced_dst_page(struct page *page, 2108 unsigned long data) 2109 { 2110 int nid = (int) data; 2111 int order = compound_order(page); 2112 gfp_t gfp = __GFP_THISNODE; 2113 struct folio *new; 2114 2115 if (order > 0) 2116 gfp |= GFP_TRANSHUGE_LIGHT; 2117 else { 2118 gfp |= GFP_HIGHUSER_MOVABLE | __GFP_NOMEMALLOC | __GFP_NORETRY | 2119 __GFP_NOWARN; 2120 gfp &= ~__GFP_RECLAIM; 2121 } 2122 new = __folio_alloc_node(gfp, order, nid); 2123 2124 return &new->page; 2125 } 2126 2127 static int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page) 2128 { 2129 int nr_pages = thp_nr_pages(page); 2130 int order = compound_order(page); 2131 2132 VM_BUG_ON_PAGE(order && !PageTransHuge(page), page); 2133 2134 /* Do not migrate THP mapped by multiple processes */ 2135 if (PageTransHuge(page) && total_mapcount(page) > 1) 2136 return 0; 2137 2138 /* Avoid migrating to a node that is nearly full */ 2139 if (!migrate_balanced_pgdat(pgdat, nr_pages)) { 2140 int z; 2141 2142 if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING)) 2143 return 0; 2144 for (z = pgdat->nr_zones - 1; z >= 0; z--) { 2145 if (managed_zone(pgdat->node_zones + z)) 2146 break; 2147 } 2148 wakeup_kswapd(pgdat->node_zones + z, 0, order, ZONE_MOVABLE); 2149 return 0; 2150 } 2151 2152 if (isolate_lru_page(page)) 2153 return 0; 2154 2155 mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + page_is_file_lru(page), 2156 nr_pages); 2157 2158 /* 2159 * Isolating the page has taken another reference, so the 2160 * caller's reference can be safely dropped without the page 2161 * disappearing underneath us during migration. 2162 */ 2163 put_page(page); 2164 return 1; 2165 } 2166 2167 /* 2168 * Attempt to migrate a misplaced page to the specified destination 2169 * node. Caller is expected to have an elevated reference count on 2170 * the page that will be dropped by this function before returning. 2171 */ 2172 int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma, 2173 int node) 2174 { 2175 pg_data_t *pgdat = NODE_DATA(node); 2176 int isolated; 2177 int nr_remaining; 2178 unsigned int nr_succeeded; 2179 LIST_HEAD(migratepages); 2180 int nr_pages = thp_nr_pages(page); 2181 2182 /* 2183 * Don't migrate file pages that are mapped in multiple processes 2184 * with execute permissions as they are probably shared libraries. 2185 */ 2186 if (page_mapcount(page) != 1 && page_is_file_lru(page) && 2187 (vma->vm_flags & VM_EXEC)) 2188 goto out; 2189 2190 /* 2191 * Also do not migrate dirty pages as not all filesystems can move 2192 * dirty pages in MIGRATE_ASYNC mode which is a waste of cycles. 2193 */ 2194 if (page_is_file_lru(page) && PageDirty(page)) 2195 goto out; 2196 2197 isolated = numamigrate_isolate_page(pgdat, page); 2198 if (!isolated) 2199 goto out; 2200 2201 list_add(&page->lru, &migratepages); 2202 nr_remaining = migrate_pages(&migratepages, alloc_misplaced_dst_page, 2203 NULL, node, MIGRATE_ASYNC, 2204 MR_NUMA_MISPLACED, &nr_succeeded); 2205 if (nr_remaining) { 2206 if (!list_empty(&migratepages)) { 2207 list_del(&page->lru); 2208 mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + 2209 page_is_file_lru(page), -nr_pages); 2210 putback_lru_page(page); 2211 } 2212 isolated = 0; 2213 } 2214 if (nr_succeeded) { 2215 count_vm_numa_events(NUMA_PAGE_MIGRATE, nr_succeeded); 2216 if (!node_is_toptier(page_to_nid(page)) && node_is_toptier(node)) 2217 mod_node_page_state(pgdat, PGPROMOTE_SUCCESS, 2218 nr_succeeded); 2219 } 2220 BUG_ON(!list_empty(&migratepages)); 2221 return isolated; 2222 2223 out: 2224 put_page(page); 2225 return 0; 2226 } 2227 #endif /* CONFIG_NUMA_BALANCING */ 2228 #endif /* CONFIG_NUMA */ 2229