1 /* 2 * Memory Migration functionality - linux/mm/migration.c 3 * 4 * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter 5 * 6 * Page migration was first developed in the context of the memory hotplug 7 * project. The main authors of the migration code are: 8 * 9 * IWAMOTO Toshihiro <iwamoto@valinux.co.jp> 10 * Hirokazu Takahashi <taka@valinux.co.jp> 11 * Dave Hansen <haveblue@us.ibm.com> 12 * Christoph Lameter 13 */ 14 15 #include <linux/migrate.h> 16 #include <linux/export.h> 17 #include <linux/swap.h> 18 #include <linux/swapops.h> 19 #include <linux/pagemap.h> 20 #include <linux/buffer_head.h> 21 #include <linux/mm_inline.h> 22 #include <linux/nsproxy.h> 23 #include <linux/pagevec.h> 24 #include <linux/ksm.h> 25 #include <linux/rmap.h> 26 #include <linux/topology.h> 27 #include <linux/cpu.h> 28 #include <linux/cpuset.h> 29 #include <linux/writeback.h> 30 #include <linux/mempolicy.h> 31 #include <linux/vmalloc.h> 32 #include <linux/security.h> 33 #include <linux/memcontrol.h> 34 #include <linux/syscalls.h> 35 #include <linux/hugetlb.h> 36 #include <linux/hugetlb_cgroup.h> 37 #include <linux/gfp.h> 38 #include <linux/balloon_compaction.h> 39 #include <linux/mmu_notifier.h> 40 41 #include <asm/tlbflush.h> 42 43 #define CREATE_TRACE_POINTS 44 #include <trace/events/migrate.h> 45 46 #include "internal.h" 47 48 /* 49 * migrate_prep() needs to be called before we start compiling a list of pages 50 * to be migrated using isolate_lru_page(). If scheduling work on other CPUs is 51 * undesirable, use migrate_prep_local() 52 */ 53 int migrate_prep(void) 54 { 55 /* 56 * Clear the LRU lists so pages can be isolated. 57 * Note that pages may be moved off the LRU after we have 58 * drained them. Those pages will fail to migrate like other 59 * pages that may be busy. 60 */ 61 lru_add_drain_all(); 62 63 return 0; 64 } 65 66 /* Do the necessary work of migrate_prep but not if it involves other CPUs */ 67 int migrate_prep_local(void) 68 { 69 lru_add_drain(); 70 71 return 0; 72 } 73 74 /* 75 * Put previously isolated pages back onto the appropriate lists 76 * from where they were once taken off for compaction/migration. 77 * 78 * This function shall be used whenever the isolated pageset has been 79 * built from lru, balloon, hugetlbfs page. See isolate_migratepages_range() 80 * and isolate_huge_page(). 81 */ 82 void putback_movable_pages(struct list_head *l) 83 { 84 struct page *page; 85 struct page *page2; 86 87 list_for_each_entry_safe(page, page2, l, lru) { 88 if (unlikely(PageHuge(page))) { 89 putback_active_hugepage(page); 90 continue; 91 } 92 list_del(&page->lru); 93 dec_zone_page_state(page, NR_ISOLATED_ANON + 94 page_is_file_cache(page)); 95 if (unlikely(isolated_balloon_page(page))) 96 balloon_page_putback(page); 97 else 98 putback_lru_page(page); 99 } 100 } 101 102 /* 103 * Restore a potential migration pte to a working pte entry 104 */ 105 static int remove_migration_pte(struct page *new, struct vm_area_struct *vma, 106 unsigned long addr, void *old) 107 { 108 struct mm_struct *mm = vma->vm_mm; 109 swp_entry_t entry; 110 pmd_t *pmd; 111 pte_t *ptep, pte; 112 spinlock_t *ptl; 113 114 if (unlikely(PageHuge(new))) { 115 ptep = huge_pte_offset(mm, addr); 116 if (!ptep) 117 goto out; 118 ptl = huge_pte_lockptr(hstate_vma(vma), mm, ptep); 119 } else { 120 pmd = mm_find_pmd(mm, addr); 121 if (!pmd) 122 goto out; 123 124 ptep = pte_offset_map(pmd, addr); 125 126 /* 127 * Peek to check is_swap_pte() before taking ptlock? No, we 128 * can race mremap's move_ptes(), which skips anon_vma lock. 129 */ 130 131 ptl = pte_lockptr(mm, pmd); 132 } 133 134 spin_lock(ptl); 135 pte = *ptep; 136 if (!is_swap_pte(pte)) 137 goto unlock; 138 139 entry = pte_to_swp_entry(pte); 140 141 if (!is_migration_entry(entry) || 142 migration_entry_to_page(entry) != old) 143 goto unlock; 144 145 get_page(new); 146 pte = pte_mkold(mk_pte(new, vma->vm_page_prot)); 147 if (pte_swp_soft_dirty(*ptep)) 148 pte = pte_mksoft_dirty(pte); 149 if (is_write_migration_entry(entry)) 150 pte = pte_mkwrite(pte); 151 #ifdef CONFIG_HUGETLB_PAGE 152 if (PageHuge(new)) { 153 pte = pte_mkhuge(pte); 154 pte = arch_make_huge_pte(pte, vma, new, 0); 155 } 156 #endif 157 flush_dcache_page(new); 158 set_pte_at(mm, addr, ptep, pte); 159 160 if (PageHuge(new)) { 161 if (PageAnon(new)) 162 hugepage_add_anon_rmap(new, vma, addr); 163 else 164 page_dup_rmap(new); 165 } else if (PageAnon(new)) 166 page_add_anon_rmap(new, vma, addr); 167 else 168 page_add_file_rmap(new); 169 170 /* No need to invalidate - it was non-present before */ 171 update_mmu_cache(vma, addr, ptep); 172 unlock: 173 pte_unmap_unlock(ptep, ptl); 174 out: 175 return SWAP_AGAIN; 176 } 177 178 /* 179 * Congratulations to trinity for discovering this bug. 180 * mm/fremap.c's remap_file_pages() accepts any range within a single vma to 181 * convert that vma to VM_NONLINEAR; and generic_file_remap_pages() will then 182 * replace the specified range by file ptes throughout (maybe populated after). 183 * If page migration finds a page within that range, while it's still located 184 * by vma_interval_tree rather than lost to i_mmap_nonlinear list, no problem: 185 * zap_pte() clears the temporary migration entry before mmap_sem is dropped. 186 * But if the migrating page is in a part of the vma outside the range to be 187 * remapped, then it will not be cleared, and remove_migration_ptes() needs to 188 * deal with it. Fortunately, this part of the vma is of course still linear, 189 * so we just need to use linear location on the nonlinear list. 190 */ 191 static int remove_linear_migration_ptes_from_nonlinear(struct page *page, 192 struct address_space *mapping, void *arg) 193 { 194 struct vm_area_struct *vma; 195 /* hugetlbfs does not support remap_pages, so no huge pgoff worries */ 196 pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT); 197 unsigned long addr; 198 199 list_for_each_entry(vma, 200 &mapping->i_mmap_nonlinear, shared.nonlinear) { 201 202 addr = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); 203 if (addr >= vma->vm_start && addr < vma->vm_end) 204 remove_migration_pte(page, vma, addr, arg); 205 } 206 return SWAP_AGAIN; 207 } 208 209 /* 210 * Get rid of all migration entries and replace them by 211 * references to the indicated page. 212 */ 213 static void remove_migration_ptes(struct page *old, struct page *new) 214 { 215 struct rmap_walk_control rwc = { 216 .rmap_one = remove_migration_pte, 217 .arg = old, 218 .file_nonlinear = remove_linear_migration_ptes_from_nonlinear, 219 }; 220 221 rmap_walk(new, &rwc); 222 } 223 224 /* 225 * Something used the pte of a page under migration. We need to 226 * get to the page and wait until migration is finished. 227 * When we return from this function the fault will be retried. 228 */ 229 static void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep, 230 spinlock_t *ptl) 231 { 232 pte_t pte; 233 swp_entry_t entry; 234 struct page *page; 235 236 spin_lock(ptl); 237 pte = *ptep; 238 if (!is_swap_pte(pte)) 239 goto out; 240 241 entry = pte_to_swp_entry(pte); 242 if (!is_migration_entry(entry)) 243 goto out; 244 245 page = migration_entry_to_page(entry); 246 247 /* 248 * Once radix-tree replacement of page migration started, page_count 249 * *must* be zero. And, we don't want to call wait_on_page_locked() 250 * against a page without get_page(). 251 * So, we use get_page_unless_zero(), here. Even failed, page fault 252 * will occur again. 253 */ 254 if (!get_page_unless_zero(page)) 255 goto out; 256 pte_unmap_unlock(ptep, ptl); 257 wait_on_page_locked(page); 258 put_page(page); 259 return; 260 out: 261 pte_unmap_unlock(ptep, ptl); 262 } 263 264 void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, 265 unsigned long address) 266 { 267 spinlock_t *ptl = pte_lockptr(mm, pmd); 268 pte_t *ptep = pte_offset_map(pmd, address); 269 __migration_entry_wait(mm, ptep, ptl); 270 } 271 272 void migration_entry_wait_huge(struct vm_area_struct *vma, 273 struct mm_struct *mm, pte_t *pte) 274 { 275 spinlock_t *ptl = huge_pte_lockptr(hstate_vma(vma), mm, pte); 276 __migration_entry_wait(mm, pte, ptl); 277 } 278 279 #ifdef CONFIG_BLOCK 280 /* Returns true if all buffers are successfully locked */ 281 static bool buffer_migrate_lock_buffers(struct buffer_head *head, 282 enum migrate_mode mode) 283 { 284 struct buffer_head *bh = head; 285 286 /* Simple case, sync compaction */ 287 if (mode != MIGRATE_ASYNC) { 288 do { 289 get_bh(bh); 290 lock_buffer(bh); 291 bh = bh->b_this_page; 292 293 } while (bh != head); 294 295 return true; 296 } 297 298 /* async case, we cannot block on lock_buffer so use trylock_buffer */ 299 do { 300 get_bh(bh); 301 if (!trylock_buffer(bh)) { 302 /* 303 * We failed to lock the buffer and cannot stall in 304 * async migration. Release the taken locks 305 */ 306 struct buffer_head *failed_bh = bh; 307 put_bh(failed_bh); 308 bh = head; 309 while (bh != failed_bh) { 310 unlock_buffer(bh); 311 put_bh(bh); 312 bh = bh->b_this_page; 313 } 314 return false; 315 } 316 317 bh = bh->b_this_page; 318 } while (bh != head); 319 return true; 320 } 321 #else 322 static inline bool buffer_migrate_lock_buffers(struct buffer_head *head, 323 enum migrate_mode mode) 324 { 325 return true; 326 } 327 #endif /* CONFIG_BLOCK */ 328 329 /* 330 * Replace the page in the mapping. 331 * 332 * The number of remaining references must be: 333 * 1 for anonymous pages without a mapping 334 * 2 for pages with a mapping 335 * 3 for pages with a mapping and PagePrivate/PagePrivate2 set. 336 */ 337 int migrate_page_move_mapping(struct address_space *mapping, 338 struct page *newpage, struct page *page, 339 struct buffer_head *head, enum migrate_mode mode, 340 int extra_count) 341 { 342 int expected_count = 1 + extra_count; 343 void **pslot; 344 345 if (!mapping) { 346 /* Anonymous page without mapping */ 347 if (page_count(page) != expected_count) 348 return -EAGAIN; 349 return MIGRATEPAGE_SUCCESS; 350 } 351 352 spin_lock_irq(&mapping->tree_lock); 353 354 pslot = radix_tree_lookup_slot(&mapping->page_tree, 355 page_index(page)); 356 357 expected_count += 1 + page_has_private(page); 358 if (page_count(page) != expected_count || 359 radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) { 360 spin_unlock_irq(&mapping->tree_lock); 361 return -EAGAIN; 362 } 363 364 if (!page_freeze_refs(page, expected_count)) { 365 spin_unlock_irq(&mapping->tree_lock); 366 return -EAGAIN; 367 } 368 369 /* 370 * In the async migration case of moving a page with buffers, lock the 371 * buffers using trylock before the mapping is moved. If the mapping 372 * was moved, we later failed to lock the buffers and could not move 373 * the mapping back due to an elevated page count, we would have to 374 * block waiting on other references to be dropped. 375 */ 376 if (mode == MIGRATE_ASYNC && head && 377 !buffer_migrate_lock_buffers(head, mode)) { 378 page_unfreeze_refs(page, expected_count); 379 spin_unlock_irq(&mapping->tree_lock); 380 return -EAGAIN; 381 } 382 383 /* 384 * Now we know that no one else is looking at the page. 385 */ 386 get_page(newpage); /* add cache reference */ 387 if (PageSwapCache(page)) { 388 SetPageSwapCache(newpage); 389 set_page_private(newpage, page_private(page)); 390 } 391 392 radix_tree_replace_slot(pslot, newpage); 393 394 /* 395 * Drop cache reference from old page by unfreezing 396 * to one less reference. 397 * We know this isn't the last reference. 398 */ 399 page_unfreeze_refs(page, expected_count - 1); 400 401 /* 402 * If moved to a different zone then also account 403 * the page for that zone. Other VM counters will be 404 * taken care of when we establish references to the 405 * new page and drop references to the old page. 406 * 407 * Note that anonymous pages are accounted for 408 * via NR_FILE_PAGES and NR_ANON_PAGES if they 409 * are mapped to swap space. 410 */ 411 __dec_zone_page_state(page, NR_FILE_PAGES); 412 __inc_zone_page_state(newpage, NR_FILE_PAGES); 413 if (!PageSwapCache(page) && PageSwapBacked(page)) { 414 __dec_zone_page_state(page, NR_SHMEM); 415 __inc_zone_page_state(newpage, NR_SHMEM); 416 } 417 spin_unlock_irq(&mapping->tree_lock); 418 419 return MIGRATEPAGE_SUCCESS; 420 } 421 422 /* 423 * The expected number of remaining references is the same as that 424 * of migrate_page_move_mapping(). 425 */ 426 int migrate_huge_page_move_mapping(struct address_space *mapping, 427 struct page *newpage, struct page *page) 428 { 429 int expected_count; 430 void **pslot; 431 432 if (!mapping) { 433 if (page_count(page) != 1) 434 return -EAGAIN; 435 return MIGRATEPAGE_SUCCESS; 436 } 437 438 spin_lock_irq(&mapping->tree_lock); 439 440 pslot = radix_tree_lookup_slot(&mapping->page_tree, 441 page_index(page)); 442 443 expected_count = 2 + page_has_private(page); 444 if (page_count(page) != expected_count || 445 radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) { 446 spin_unlock_irq(&mapping->tree_lock); 447 return -EAGAIN; 448 } 449 450 if (!page_freeze_refs(page, expected_count)) { 451 spin_unlock_irq(&mapping->tree_lock); 452 return -EAGAIN; 453 } 454 455 get_page(newpage); 456 457 radix_tree_replace_slot(pslot, newpage); 458 459 page_unfreeze_refs(page, expected_count - 1); 460 461 spin_unlock_irq(&mapping->tree_lock); 462 return MIGRATEPAGE_SUCCESS; 463 } 464 465 /* 466 * Gigantic pages are so large that we do not guarantee that page++ pointer 467 * arithmetic will work across the entire page. We need something more 468 * specialized. 469 */ 470 static void __copy_gigantic_page(struct page *dst, struct page *src, 471 int nr_pages) 472 { 473 int i; 474 struct page *dst_base = dst; 475 struct page *src_base = src; 476 477 for (i = 0; i < nr_pages; ) { 478 cond_resched(); 479 copy_highpage(dst, src); 480 481 i++; 482 dst = mem_map_next(dst, dst_base, i); 483 src = mem_map_next(src, src_base, i); 484 } 485 } 486 487 static void copy_huge_page(struct page *dst, struct page *src) 488 { 489 int i; 490 int nr_pages; 491 492 if (PageHuge(src)) { 493 /* hugetlbfs page */ 494 struct hstate *h = page_hstate(src); 495 nr_pages = pages_per_huge_page(h); 496 497 if (unlikely(nr_pages > MAX_ORDER_NR_PAGES)) { 498 __copy_gigantic_page(dst, src, nr_pages); 499 return; 500 } 501 } else { 502 /* thp page */ 503 BUG_ON(!PageTransHuge(src)); 504 nr_pages = hpage_nr_pages(src); 505 } 506 507 for (i = 0; i < nr_pages; i++) { 508 cond_resched(); 509 copy_highpage(dst + i, src + i); 510 } 511 } 512 513 /* 514 * Copy the page to its new location 515 */ 516 void migrate_page_copy(struct page *newpage, struct page *page) 517 { 518 int cpupid; 519 520 if (PageHuge(page) || PageTransHuge(page)) 521 copy_huge_page(newpage, page); 522 else 523 copy_highpage(newpage, page); 524 525 if (PageError(page)) 526 SetPageError(newpage); 527 if (PageReferenced(page)) 528 SetPageReferenced(newpage); 529 if (PageUptodate(page)) 530 SetPageUptodate(newpage); 531 if (TestClearPageActive(page)) { 532 VM_BUG_ON_PAGE(PageUnevictable(page), page); 533 SetPageActive(newpage); 534 } else if (TestClearPageUnevictable(page)) 535 SetPageUnevictable(newpage); 536 if (PageChecked(page)) 537 SetPageChecked(newpage); 538 if (PageMappedToDisk(page)) 539 SetPageMappedToDisk(newpage); 540 541 if (PageDirty(page)) { 542 clear_page_dirty_for_io(page); 543 /* 544 * Want to mark the page and the radix tree as dirty, and 545 * redo the accounting that clear_page_dirty_for_io undid, 546 * but we can't use set_page_dirty because that function 547 * is actually a signal that all of the page has become dirty. 548 * Whereas only part of our page may be dirty. 549 */ 550 if (PageSwapBacked(page)) 551 SetPageDirty(newpage); 552 else 553 __set_page_dirty_nobuffers(newpage); 554 } 555 556 /* 557 * Copy NUMA information to the new page, to prevent over-eager 558 * future migrations of this same page. 559 */ 560 cpupid = page_cpupid_xchg_last(page, -1); 561 page_cpupid_xchg_last(newpage, cpupid); 562 563 mlock_migrate_page(newpage, page); 564 ksm_migrate_page(newpage, page); 565 /* 566 * Please do not reorder this without considering how mm/ksm.c's 567 * get_ksm_page() depends upon ksm_migrate_page() and PageSwapCache(). 568 */ 569 ClearPageSwapCache(page); 570 ClearPagePrivate(page); 571 set_page_private(page, 0); 572 573 /* 574 * If any waiters have accumulated on the new page then 575 * wake them up. 576 */ 577 if (PageWriteback(newpage)) 578 end_page_writeback(newpage); 579 } 580 581 /************************************************************ 582 * Migration functions 583 ***********************************************************/ 584 585 /* 586 * Common logic to directly migrate a single page suitable for 587 * pages that do not use PagePrivate/PagePrivate2. 588 * 589 * Pages are locked upon entry and exit. 590 */ 591 int migrate_page(struct address_space *mapping, 592 struct page *newpage, struct page *page, 593 enum migrate_mode mode) 594 { 595 int rc; 596 597 BUG_ON(PageWriteback(page)); /* Writeback must be complete */ 598 599 rc = migrate_page_move_mapping(mapping, newpage, page, NULL, mode, 0); 600 601 if (rc != MIGRATEPAGE_SUCCESS) 602 return rc; 603 604 migrate_page_copy(newpage, page); 605 return MIGRATEPAGE_SUCCESS; 606 } 607 EXPORT_SYMBOL(migrate_page); 608 609 #ifdef CONFIG_BLOCK 610 /* 611 * Migration function for pages with buffers. This function can only be used 612 * if the underlying filesystem guarantees that no other references to "page" 613 * exist. 614 */ 615 int buffer_migrate_page(struct address_space *mapping, 616 struct page *newpage, struct page *page, enum migrate_mode mode) 617 { 618 struct buffer_head *bh, *head; 619 int rc; 620 621 if (!page_has_buffers(page)) 622 return migrate_page(mapping, newpage, page, mode); 623 624 head = page_buffers(page); 625 626 rc = migrate_page_move_mapping(mapping, newpage, page, head, mode, 0); 627 628 if (rc != MIGRATEPAGE_SUCCESS) 629 return rc; 630 631 /* 632 * In the async case, migrate_page_move_mapping locked the buffers 633 * with an IRQ-safe spinlock held. In the sync case, the buffers 634 * need to be locked now 635 */ 636 if (mode != MIGRATE_ASYNC) 637 BUG_ON(!buffer_migrate_lock_buffers(head, mode)); 638 639 ClearPagePrivate(page); 640 set_page_private(newpage, page_private(page)); 641 set_page_private(page, 0); 642 put_page(page); 643 get_page(newpage); 644 645 bh = head; 646 do { 647 set_bh_page(bh, newpage, bh_offset(bh)); 648 bh = bh->b_this_page; 649 650 } while (bh != head); 651 652 SetPagePrivate(newpage); 653 654 migrate_page_copy(newpage, page); 655 656 bh = head; 657 do { 658 unlock_buffer(bh); 659 put_bh(bh); 660 bh = bh->b_this_page; 661 662 } while (bh != head); 663 664 return MIGRATEPAGE_SUCCESS; 665 } 666 EXPORT_SYMBOL(buffer_migrate_page); 667 #endif 668 669 /* 670 * Writeback a page to clean the dirty state 671 */ 672 static int writeout(struct address_space *mapping, struct page *page) 673 { 674 struct writeback_control wbc = { 675 .sync_mode = WB_SYNC_NONE, 676 .nr_to_write = 1, 677 .range_start = 0, 678 .range_end = LLONG_MAX, 679 .for_reclaim = 1 680 }; 681 int rc; 682 683 if (!mapping->a_ops->writepage) 684 /* No write method for the address space */ 685 return -EINVAL; 686 687 if (!clear_page_dirty_for_io(page)) 688 /* Someone else already triggered a write */ 689 return -EAGAIN; 690 691 /* 692 * A dirty page may imply that the underlying filesystem has 693 * the page on some queue. So the page must be clean for 694 * migration. Writeout may mean we loose the lock and the 695 * page state is no longer what we checked for earlier. 696 * At this point we know that the migration attempt cannot 697 * be successful. 698 */ 699 remove_migration_ptes(page, page); 700 701 rc = mapping->a_ops->writepage(page, &wbc); 702 703 if (rc != AOP_WRITEPAGE_ACTIVATE) 704 /* unlocked. Relock */ 705 lock_page(page); 706 707 return (rc < 0) ? -EIO : -EAGAIN; 708 } 709 710 /* 711 * Default handling if a filesystem does not provide a migration function. 712 */ 713 static int fallback_migrate_page(struct address_space *mapping, 714 struct page *newpage, struct page *page, enum migrate_mode mode) 715 { 716 if (PageDirty(page)) { 717 /* Only writeback pages in full synchronous migration */ 718 if (mode != MIGRATE_SYNC) 719 return -EBUSY; 720 return writeout(mapping, page); 721 } 722 723 /* 724 * Buffers may be managed in a filesystem specific way. 725 * We must have no buffers or drop them. 726 */ 727 if (page_has_private(page) && 728 !try_to_release_page(page, GFP_KERNEL)) 729 return -EAGAIN; 730 731 return migrate_page(mapping, newpage, page, mode); 732 } 733 734 /* 735 * Move a page to a newly allocated page 736 * The page is locked and all ptes have been successfully removed. 737 * 738 * The new page will have replaced the old page if this function 739 * is successful. 740 * 741 * Return value: 742 * < 0 - error code 743 * MIGRATEPAGE_SUCCESS - success 744 */ 745 static int move_to_new_page(struct page *newpage, struct page *page, 746 int remap_swapcache, enum migrate_mode mode) 747 { 748 struct address_space *mapping; 749 int rc; 750 751 /* 752 * Block others from accessing the page when we get around to 753 * establishing additional references. We are the only one 754 * holding a reference to the new page at this point. 755 */ 756 if (!trylock_page(newpage)) 757 BUG(); 758 759 /* Prepare mapping for the new page.*/ 760 newpage->index = page->index; 761 newpage->mapping = page->mapping; 762 if (PageSwapBacked(page)) 763 SetPageSwapBacked(newpage); 764 765 mapping = page_mapping(page); 766 if (!mapping) 767 rc = migrate_page(mapping, newpage, page, mode); 768 else if (mapping->a_ops->migratepage) 769 /* 770 * Most pages have a mapping and most filesystems provide a 771 * migratepage callback. Anonymous pages are part of swap 772 * space which also has its own migratepage callback. This 773 * is the most common path for page migration. 774 */ 775 rc = mapping->a_ops->migratepage(mapping, 776 newpage, page, mode); 777 else 778 rc = fallback_migrate_page(mapping, newpage, page, mode); 779 780 if (rc != MIGRATEPAGE_SUCCESS) { 781 newpage->mapping = NULL; 782 } else { 783 if (remap_swapcache) 784 remove_migration_ptes(page, newpage); 785 page->mapping = NULL; 786 } 787 788 unlock_page(newpage); 789 790 return rc; 791 } 792 793 static int __unmap_and_move(struct page *page, struct page *newpage, 794 int force, enum migrate_mode mode) 795 { 796 int rc = -EAGAIN; 797 int remap_swapcache = 1; 798 struct mem_cgroup *mem; 799 struct anon_vma *anon_vma = NULL; 800 801 if (!trylock_page(page)) { 802 if (!force || mode == MIGRATE_ASYNC) 803 goto out; 804 805 /* 806 * It's not safe for direct compaction to call lock_page. 807 * For example, during page readahead pages are added locked 808 * to the LRU. Later, when the IO completes the pages are 809 * marked uptodate and unlocked. However, the queueing 810 * could be merging multiple pages for one bio (e.g. 811 * mpage_readpages). If an allocation happens for the 812 * second or third page, the process can end up locking 813 * the same page twice and deadlocking. Rather than 814 * trying to be clever about what pages can be locked, 815 * avoid the use of lock_page for direct compaction 816 * altogether. 817 */ 818 if (current->flags & PF_MEMALLOC) 819 goto out; 820 821 lock_page(page); 822 } 823 824 /* charge against new page */ 825 mem_cgroup_prepare_migration(page, newpage, &mem); 826 827 if (PageWriteback(page)) { 828 /* 829 * Only in the case of a full synchronous migration is it 830 * necessary to wait for PageWriteback. In the async case, 831 * the retry loop is too short and in the sync-light case, 832 * the overhead of stalling is too much 833 */ 834 if (mode != MIGRATE_SYNC) { 835 rc = -EBUSY; 836 goto uncharge; 837 } 838 if (!force) 839 goto uncharge; 840 wait_on_page_writeback(page); 841 } 842 /* 843 * By try_to_unmap(), page->mapcount goes down to 0 here. In this case, 844 * we cannot notice that anon_vma is freed while we migrates a page. 845 * This get_anon_vma() delays freeing anon_vma pointer until the end 846 * of migration. File cache pages are no problem because of page_lock() 847 * File Caches may use write_page() or lock_page() in migration, then, 848 * just care Anon page here. 849 */ 850 if (PageAnon(page) && !PageKsm(page)) { 851 /* 852 * Only page_lock_anon_vma_read() understands the subtleties of 853 * getting a hold on an anon_vma from outside one of its mms. 854 */ 855 anon_vma = page_get_anon_vma(page); 856 if (anon_vma) { 857 /* 858 * Anon page 859 */ 860 } else if (PageSwapCache(page)) { 861 /* 862 * We cannot be sure that the anon_vma of an unmapped 863 * swapcache page is safe to use because we don't 864 * know in advance if the VMA that this page belonged 865 * to still exists. If the VMA and others sharing the 866 * data have been freed, then the anon_vma could 867 * already be invalid. 868 * 869 * To avoid this possibility, swapcache pages get 870 * migrated but are not remapped when migration 871 * completes 872 */ 873 remap_swapcache = 0; 874 } else { 875 goto uncharge; 876 } 877 } 878 879 if (unlikely(balloon_page_movable(page))) { 880 /* 881 * A ballooned page does not need any special attention from 882 * physical to virtual reverse mapping procedures. 883 * Skip any attempt to unmap PTEs or to remap swap cache, 884 * in order to avoid burning cycles at rmap level, and perform 885 * the page migration right away (proteced by page lock). 886 */ 887 rc = balloon_page_migrate(newpage, page, mode); 888 goto uncharge; 889 } 890 891 /* 892 * Corner case handling: 893 * 1. When a new swap-cache page is read into, it is added to the LRU 894 * and treated as swapcache but it has no rmap yet. 895 * Calling try_to_unmap() against a page->mapping==NULL page will 896 * trigger a BUG. So handle it here. 897 * 2. An orphaned page (see truncate_complete_page) might have 898 * fs-private metadata. The page can be picked up due to memory 899 * offlining. Everywhere else except page reclaim, the page is 900 * invisible to the vm, so the page can not be migrated. So try to 901 * free the metadata, so the page can be freed. 902 */ 903 if (!page->mapping) { 904 VM_BUG_ON_PAGE(PageAnon(page), page); 905 if (page_has_private(page)) { 906 try_to_free_buffers(page); 907 goto uncharge; 908 } 909 goto skip_unmap; 910 } 911 912 /* Establish migration ptes or remove ptes */ 913 try_to_unmap(page, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS); 914 915 skip_unmap: 916 if (!page_mapped(page)) 917 rc = move_to_new_page(newpage, page, remap_swapcache, mode); 918 919 if (rc && remap_swapcache) 920 remove_migration_ptes(page, page); 921 922 /* Drop an anon_vma reference if we took one */ 923 if (anon_vma) 924 put_anon_vma(anon_vma); 925 926 uncharge: 927 mem_cgroup_end_migration(mem, page, newpage, 928 (rc == MIGRATEPAGE_SUCCESS || 929 rc == MIGRATEPAGE_BALLOON_SUCCESS)); 930 unlock_page(page); 931 out: 932 return rc; 933 } 934 935 /* 936 * Obtain the lock on page, remove all ptes and migrate the page 937 * to the newly allocated page in newpage. 938 */ 939 static int unmap_and_move(new_page_t get_new_page, free_page_t put_new_page, 940 unsigned long private, struct page *page, int force, 941 enum migrate_mode mode) 942 { 943 int rc = 0; 944 int *result = NULL; 945 struct page *newpage = get_new_page(page, private, &result); 946 947 if (!newpage) 948 return -ENOMEM; 949 950 if (page_count(page) == 1) { 951 /* page was freed from under us. So we are done. */ 952 goto out; 953 } 954 955 if (unlikely(PageTransHuge(page))) 956 if (unlikely(split_huge_page(page))) 957 goto out; 958 959 rc = __unmap_and_move(page, newpage, force, mode); 960 961 if (unlikely(rc == MIGRATEPAGE_BALLOON_SUCCESS)) { 962 /* 963 * A ballooned page has been migrated already. 964 * Now, it's the time to wrap-up counters, 965 * handle the page back to Buddy and return. 966 */ 967 dec_zone_page_state(page, NR_ISOLATED_ANON + 968 page_is_file_cache(page)); 969 balloon_page_free(page); 970 return MIGRATEPAGE_SUCCESS; 971 } 972 out: 973 if (rc != -EAGAIN) { 974 /* 975 * A page that has been migrated has all references 976 * removed and will be freed. A page that has not been 977 * migrated will have kepts its references and be 978 * restored. 979 */ 980 list_del(&page->lru); 981 dec_zone_page_state(page, NR_ISOLATED_ANON + 982 page_is_file_cache(page)); 983 putback_lru_page(page); 984 } 985 986 /* 987 * If migration was not successful and there's a freeing callback, use 988 * it. Otherwise, putback_lru_page() will drop the reference grabbed 989 * during isolation. 990 */ 991 if (rc != MIGRATEPAGE_SUCCESS && put_new_page) 992 put_new_page(newpage, private); 993 else 994 putback_lru_page(newpage); 995 996 if (result) { 997 if (rc) 998 *result = rc; 999 else 1000 *result = page_to_nid(newpage); 1001 } 1002 return rc; 1003 } 1004 1005 /* 1006 * Counterpart of unmap_and_move_page() for hugepage migration. 1007 * 1008 * This function doesn't wait the completion of hugepage I/O 1009 * because there is no race between I/O and migration for hugepage. 1010 * Note that currently hugepage I/O occurs only in direct I/O 1011 * where no lock is held and PG_writeback is irrelevant, 1012 * and writeback status of all subpages are counted in the reference 1013 * count of the head page (i.e. if all subpages of a 2MB hugepage are 1014 * under direct I/O, the reference of the head page is 512 and a bit more.) 1015 * This means that when we try to migrate hugepage whose subpages are 1016 * doing direct I/O, some references remain after try_to_unmap() and 1017 * hugepage migration fails without data corruption. 1018 * 1019 * There is also no race when direct I/O is issued on the page under migration, 1020 * because then pte is replaced with migration swap entry and direct I/O code 1021 * will wait in the page fault for migration to complete. 1022 */ 1023 static int unmap_and_move_huge_page(new_page_t get_new_page, 1024 free_page_t put_new_page, unsigned long private, 1025 struct page *hpage, int force, 1026 enum migrate_mode mode) 1027 { 1028 int rc = 0; 1029 int *result = NULL; 1030 struct page *new_hpage; 1031 struct anon_vma *anon_vma = NULL; 1032 1033 /* 1034 * Movability of hugepages depends on architectures and hugepage size. 1035 * This check is necessary because some callers of hugepage migration 1036 * like soft offline and memory hotremove don't walk through page 1037 * tables or check whether the hugepage is pmd-based or not before 1038 * kicking migration. 1039 */ 1040 if (!hugepage_migration_supported(page_hstate(hpage))) { 1041 putback_active_hugepage(hpage); 1042 return -ENOSYS; 1043 } 1044 1045 new_hpage = get_new_page(hpage, private, &result); 1046 if (!new_hpage) 1047 return -ENOMEM; 1048 1049 rc = -EAGAIN; 1050 1051 if (!trylock_page(hpage)) { 1052 if (!force || mode != MIGRATE_SYNC) 1053 goto out; 1054 lock_page(hpage); 1055 } 1056 1057 if (PageAnon(hpage)) 1058 anon_vma = page_get_anon_vma(hpage); 1059 1060 try_to_unmap(hpage, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS); 1061 1062 if (!page_mapped(hpage)) 1063 rc = move_to_new_page(new_hpage, hpage, 1, mode); 1064 1065 if (rc != MIGRATEPAGE_SUCCESS) 1066 remove_migration_ptes(hpage, hpage); 1067 1068 if (anon_vma) 1069 put_anon_vma(anon_vma); 1070 1071 if (rc == MIGRATEPAGE_SUCCESS) 1072 hugetlb_cgroup_migrate(hpage, new_hpage); 1073 1074 unlock_page(hpage); 1075 out: 1076 if (rc != -EAGAIN) 1077 putback_active_hugepage(hpage); 1078 1079 /* 1080 * If migration was not successful and there's a freeing callback, use 1081 * it. Otherwise, put_page() will drop the reference grabbed during 1082 * isolation. 1083 */ 1084 if (rc != MIGRATEPAGE_SUCCESS && put_new_page) 1085 put_new_page(new_hpage, private); 1086 else 1087 put_page(new_hpage); 1088 1089 if (result) { 1090 if (rc) 1091 *result = rc; 1092 else 1093 *result = page_to_nid(new_hpage); 1094 } 1095 return rc; 1096 } 1097 1098 /* 1099 * migrate_pages - migrate the pages specified in a list, to the free pages 1100 * supplied as the target for the page migration 1101 * 1102 * @from: The list of pages to be migrated. 1103 * @get_new_page: The function used to allocate free pages to be used 1104 * as the target of the page migration. 1105 * @put_new_page: The function used to free target pages if migration 1106 * fails, or NULL if no special handling is necessary. 1107 * @private: Private data to be passed on to get_new_page() 1108 * @mode: The migration mode that specifies the constraints for 1109 * page migration, if any. 1110 * @reason: The reason for page migration. 1111 * 1112 * The function returns after 10 attempts or if no pages are movable any more 1113 * because the list has become empty or no retryable pages exist any more. 1114 * The caller should call putback_lru_pages() to return pages to the LRU 1115 * or free list only if ret != 0. 1116 * 1117 * Returns the number of pages that were not migrated, or an error code. 1118 */ 1119 int migrate_pages(struct list_head *from, new_page_t get_new_page, 1120 free_page_t put_new_page, unsigned long private, 1121 enum migrate_mode mode, int reason) 1122 { 1123 int retry = 1; 1124 int nr_failed = 0; 1125 int nr_succeeded = 0; 1126 int pass = 0; 1127 struct page *page; 1128 struct page *page2; 1129 int swapwrite = current->flags & PF_SWAPWRITE; 1130 int rc; 1131 1132 if (!swapwrite) 1133 current->flags |= PF_SWAPWRITE; 1134 1135 for(pass = 0; pass < 10 && retry; pass++) { 1136 retry = 0; 1137 1138 list_for_each_entry_safe(page, page2, from, lru) { 1139 cond_resched(); 1140 1141 if (PageHuge(page)) 1142 rc = unmap_and_move_huge_page(get_new_page, 1143 put_new_page, private, page, 1144 pass > 2, mode); 1145 else 1146 rc = unmap_and_move(get_new_page, put_new_page, 1147 private, page, pass > 2, mode); 1148 1149 switch(rc) { 1150 case -ENOMEM: 1151 goto out; 1152 case -EAGAIN: 1153 retry++; 1154 break; 1155 case MIGRATEPAGE_SUCCESS: 1156 nr_succeeded++; 1157 break; 1158 default: 1159 /* 1160 * Permanent failure (-EBUSY, -ENOSYS, etc.): 1161 * unlike -EAGAIN case, the failed page is 1162 * removed from migration page list and not 1163 * retried in the next outer loop. 1164 */ 1165 nr_failed++; 1166 break; 1167 } 1168 } 1169 } 1170 rc = nr_failed + retry; 1171 out: 1172 if (nr_succeeded) 1173 count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded); 1174 if (nr_failed) 1175 count_vm_events(PGMIGRATE_FAIL, nr_failed); 1176 trace_mm_migrate_pages(nr_succeeded, nr_failed, mode, reason); 1177 1178 if (!swapwrite) 1179 current->flags &= ~PF_SWAPWRITE; 1180 1181 return rc; 1182 } 1183 1184 #ifdef CONFIG_NUMA 1185 /* 1186 * Move a list of individual pages 1187 */ 1188 struct page_to_node { 1189 unsigned long addr; 1190 struct page *page; 1191 int node; 1192 int status; 1193 }; 1194 1195 static struct page *new_page_node(struct page *p, unsigned long private, 1196 int **result) 1197 { 1198 struct page_to_node *pm = (struct page_to_node *)private; 1199 1200 while (pm->node != MAX_NUMNODES && pm->page != p) 1201 pm++; 1202 1203 if (pm->node == MAX_NUMNODES) 1204 return NULL; 1205 1206 *result = &pm->status; 1207 1208 if (PageHuge(p)) 1209 return alloc_huge_page_node(page_hstate(compound_head(p)), 1210 pm->node); 1211 else 1212 return alloc_pages_exact_node(pm->node, 1213 GFP_HIGHUSER_MOVABLE | __GFP_THISNODE, 0); 1214 } 1215 1216 /* 1217 * Move a set of pages as indicated in the pm array. The addr 1218 * field must be set to the virtual address of the page to be moved 1219 * and the node number must contain a valid target node. 1220 * The pm array ends with node = MAX_NUMNODES. 1221 */ 1222 static int do_move_page_to_node_array(struct mm_struct *mm, 1223 struct page_to_node *pm, 1224 int migrate_all) 1225 { 1226 int err; 1227 struct page_to_node *pp; 1228 LIST_HEAD(pagelist); 1229 1230 down_read(&mm->mmap_sem); 1231 1232 /* 1233 * Build a list of pages to migrate 1234 */ 1235 for (pp = pm; pp->node != MAX_NUMNODES; pp++) { 1236 struct vm_area_struct *vma; 1237 struct page *page; 1238 1239 err = -EFAULT; 1240 vma = find_vma(mm, pp->addr); 1241 if (!vma || pp->addr < vma->vm_start || !vma_migratable(vma)) 1242 goto set_status; 1243 1244 page = follow_page(vma, pp->addr, FOLL_GET|FOLL_SPLIT); 1245 1246 err = PTR_ERR(page); 1247 if (IS_ERR(page)) 1248 goto set_status; 1249 1250 err = -ENOENT; 1251 if (!page) 1252 goto set_status; 1253 1254 /* Use PageReserved to check for zero page */ 1255 if (PageReserved(page)) 1256 goto put_and_set; 1257 1258 pp->page = page; 1259 err = page_to_nid(page); 1260 1261 if (err == pp->node) 1262 /* 1263 * Node already in the right place 1264 */ 1265 goto put_and_set; 1266 1267 err = -EACCES; 1268 if (page_mapcount(page) > 1 && 1269 !migrate_all) 1270 goto put_and_set; 1271 1272 if (PageHuge(page)) { 1273 isolate_huge_page(page, &pagelist); 1274 goto put_and_set; 1275 } 1276 1277 err = isolate_lru_page(page); 1278 if (!err) { 1279 list_add_tail(&page->lru, &pagelist); 1280 inc_zone_page_state(page, NR_ISOLATED_ANON + 1281 page_is_file_cache(page)); 1282 } 1283 put_and_set: 1284 /* 1285 * Either remove the duplicate refcount from 1286 * isolate_lru_page() or drop the page ref if it was 1287 * not isolated. 1288 */ 1289 put_page(page); 1290 set_status: 1291 pp->status = err; 1292 } 1293 1294 err = 0; 1295 if (!list_empty(&pagelist)) { 1296 err = migrate_pages(&pagelist, new_page_node, NULL, 1297 (unsigned long)pm, MIGRATE_SYNC, MR_SYSCALL); 1298 if (err) 1299 putback_movable_pages(&pagelist); 1300 } 1301 1302 up_read(&mm->mmap_sem); 1303 return err; 1304 } 1305 1306 /* 1307 * Migrate an array of page address onto an array of nodes and fill 1308 * the corresponding array of status. 1309 */ 1310 static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes, 1311 unsigned long nr_pages, 1312 const void __user * __user *pages, 1313 const int __user *nodes, 1314 int __user *status, int flags) 1315 { 1316 struct page_to_node *pm; 1317 unsigned long chunk_nr_pages; 1318 unsigned long chunk_start; 1319 int err; 1320 1321 err = -ENOMEM; 1322 pm = (struct page_to_node *)__get_free_page(GFP_KERNEL); 1323 if (!pm) 1324 goto out; 1325 1326 migrate_prep(); 1327 1328 /* 1329 * Store a chunk of page_to_node array in a page, 1330 * but keep the last one as a marker 1331 */ 1332 chunk_nr_pages = (PAGE_SIZE / sizeof(struct page_to_node)) - 1; 1333 1334 for (chunk_start = 0; 1335 chunk_start < nr_pages; 1336 chunk_start += chunk_nr_pages) { 1337 int j; 1338 1339 if (chunk_start + chunk_nr_pages > nr_pages) 1340 chunk_nr_pages = nr_pages - chunk_start; 1341 1342 /* fill the chunk pm with addrs and nodes from user-space */ 1343 for (j = 0; j < chunk_nr_pages; j++) { 1344 const void __user *p; 1345 int node; 1346 1347 err = -EFAULT; 1348 if (get_user(p, pages + j + chunk_start)) 1349 goto out_pm; 1350 pm[j].addr = (unsigned long) p; 1351 1352 if (get_user(node, nodes + j + chunk_start)) 1353 goto out_pm; 1354 1355 err = -ENODEV; 1356 if (node < 0 || node >= MAX_NUMNODES) 1357 goto out_pm; 1358 1359 if (!node_state(node, N_MEMORY)) 1360 goto out_pm; 1361 1362 err = -EACCES; 1363 if (!node_isset(node, task_nodes)) 1364 goto out_pm; 1365 1366 pm[j].node = node; 1367 } 1368 1369 /* End marker for this chunk */ 1370 pm[chunk_nr_pages].node = MAX_NUMNODES; 1371 1372 /* Migrate this chunk */ 1373 err = do_move_page_to_node_array(mm, pm, 1374 flags & MPOL_MF_MOVE_ALL); 1375 if (err < 0) 1376 goto out_pm; 1377 1378 /* Return status information */ 1379 for (j = 0; j < chunk_nr_pages; j++) 1380 if (put_user(pm[j].status, status + j + chunk_start)) { 1381 err = -EFAULT; 1382 goto out_pm; 1383 } 1384 } 1385 err = 0; 1386 1387 out_pm: 1388 free_page((unsigned long)pm); 1389 out: 1390 return err; 1391 } 1392 1393 /* 1394 * Determine the nodes of an array of pages and store it in an array of status. 1395 */ 1396 static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages, 1397 const void __user **pages, int *status) 1398 { 1399 unsigned long i; 1400 1401 down_read(&mm->mmap_sem); 1402 1403 for (i = 0; i < nr_pages; i++) { 1404 unsigned long addr = (unsigned long)(*pages); 1405 struct vm_area_struct *vma; 1406 struct page *page; 1407 int err = -EFAULT; 1408 1409 vma = find_vma(mm, addr); 1410 if (!vma || addr < vma->vm_start) 1411 goto set_status; 1412 1413 page = follow_page(vma, addr, 0); 1414 1415 err = PTR_ERR(page); 1416 if (IS_ERR(page)) 1417 goto set_status; 1418 1419 err = -ENOENT; 1420 /* Use PageReserved to check for zero page */ 1421 if (!page || PageReserved(page)) 1422 goto set_status; 1423 1424 err = page_to_nid(page); 1425 set_status: 1426 *status = err; 1427 1428 pages++; 1429 status++; 1430 } 1431 1432 up_read(&mm->mmap_sem); 1433 } 1434 1435 /* 1436 * Determine the nodes of a user array of pages and store it in 1437 * a user array of status. 1438 */ 1439 static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages, 1440 const void __user * __user *pages, 1441 int __user *status) 1442 { 1443 #define DO_PAGES_STAT_CHUNK_NR 16 1444 const void __user *chunk_pages[DO_PAGES_STAT_CHUNK_NR]; 1445 int chunk_status[DO_PAGES_STAT_CHUNK_NR]; 1446 1447 while (nr_pages) { 1448 unsigned long chunk_nr; 1449 1450 chunk_nr = nr_pages; 1451 if (chunk_nr > DO_PAGES_STAT_CHUNK_NR) 1452 chunk_nr = DO_PAGES_STAT_CHUNK_NR; 1453 1454 if (copy_from_user(chunk_pages, pages, chunk_nr * sizeof(*chunk_pages))) 1455 break; 1456 1457 do_pages_stat_array(mm, chunk_nr, chunk_pages, chunk_status); 1458 1459 if (copy_to_user(status, chunk_status, chunk_nr * sizeof(*status))) 1460 break; 1461 1462 pages += chunk_nr; 1463 status += chunk_nr; 1464 nr_pages -= chunk_nr; 1465 } 1466 return nr_pages ? -EFAULT : 0; 1467 } 1468 1469 /* 1470 * Move a list of pages in the address space of the currently executing 1471 * process. 1472 */ 1473 SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages, 1474 const void __user * __user *, pages, 1475 const int __user *, nodes, 1476 int __user *, status, int, flags) 1477 { 1478 const struct cred *cred = current_cred(), *tcred; 1479 struct task_struct *task; 1480 struct mm_struct *mm; 1481 int err; 1482 nodemask_t task_nodes; 1483 1484 /* Check flags */ 1485 if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL)) 1486 return -EINVAL; 1487 1488 if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE)) 1489 return -EPERM; 1490 1491 /* Find the mm_struct */ 1492 rcu_read_lock(); 1493 task = pid ? find_task_by_vpid(pid) : current; 1494 if (!task) { 1495 rcu_read_unlock(); 1496 return -ESRCH; 1497 } 1498 get_task_struct(task); 1499 1500 /* 1501 * Check if this process has the right to modify the specified 1502 * process. The right exists if the process has administrative 1503 * capabilities, superuser privileges or the same 1504 * userid as the target process. 1505 */ 1506 tcred = __task_cred(task); 1507 if (!uid_eq(cred->euid, tcred->suid) && !uid_eq(cred->euid, tcred->uid) && 1508 !uid_eq(cred->uid, tcred->suid) && !uid_eq(cred->uid, tcred->uid) && 1509 !capable(CAP_SYS_NICE)) { 1510 rcu_read_unlock(); 1511 err = -EPERM; 1512 goto out; 1513 } 1514 rcu_read_unlock(); 1515 1516 err = security_task_movememory(task); 1517 if (err) 1518 goto out; 1519 1520 task_nodes = cpuset_mems_allowed(task); 1521 mm = get_task_mm(task); 1522 put_task_struct(task); 1523 1524 if (!mm) 1525 return -EINVAL; 1526 1527 if (nodes) 1528 err = do_pages_move(mm, task_nodes, nr_pages, pages, 1529 nodes, status, flags); 1530 else 1531 err = do_pages_stat(mm, nr_pages, pages, status); 1532 1533 mmput(mm); 1534 return err; 1535 1536 out: 1537 put_task_struct(task); 1538 return err; 1539 } 1540 1541 /* 1542 * Call migration functions in the vma_ops that may prepare 1543 * memory in a vm for migration. migration functions may perform 1544 * the migration for vmas that do not have an underlying page struct. 1545 */ 1546 int migrate_vmas(struct mm_struct *mm, const nodemask_t *to, 1547 const nodemask_t *from, unsigned long flags) 1548 { 1549 struct vm_area_struct *vma; 1550 int err = 0; 1551 1552 for (vma = mm->mmap; vma && !err; vma = vma->vm_next) { 1553 if (vma->vm_ops && vma->vm_ops->migrate) { 1554 err = vma->vm_ops->migrate(vma, to, from, flags); 1555 if (err) 1556 break; 1557 } 1558 } 1559 return err; 1560 } 1561 1562 #ifdef CONFIG_NUMA_BALANCING 1563 /* 1564 * Returns true if this is a safe migration target node for misplaced NUMA 1565 * pages. Currently it only checks the watermarks which crude 1566 */ 1567 static bool migrate_balanced_pgdat(struct pglist_data *pgdat, 1568 unsigned long nr_migrate_pages) 1569 { 1570 int z; 1571 for (z = pgdat->nr_zones - 1; z >= 0; z--) { 1572 struct zone *zone = pgdat->node_zones + z; 1573 1574 if (!populated_zone(zone)) 1575 continue; 1576 1577 if (!zone_reclaimable(zone)) 1578 continue; 1579 1580 /* Avoid waking kswapd by allocating pages_to_migrate pages. */ 1581 if (!zone_watermark_ok(zone, 0, 1582 high_wmark_pages(zone) + 1583 nr_migrate_pages, 1584 0, 0)) 1585 continue; 1586 return true; 1587 } 1588 return false; 1589 } 1590 1591 static struct page *alloc_misplaced_dst_page(struct page *page, 1592 unsigned long data, 1593 int **result) 1594 { 1595 int nid = (int) data; 1596 struct page *newpage; 1597 1598 newpage = alloc_pages_exact_node(nid, 1599 (GFP_HIGHUSER_MOVABLE | 1600 __GFP_THISNODE | __GFP_NOMEMALLOC | 1601 __GFP_NORETRY | __GFP_NOWARN) & 1602 ~GFP_IOFS, 0); 1603 1604 return newpage; 1605 } 1606 1607 /* 1608 * page migration rate limiting control. 1609 * Do not migrate more than @pages_to_migrate in a @migrate_interval_millisecs 1610 * window of time. Default here says do not migrate more than 1280M per second. 1611 * If a node is rate-limited then PTE NUMA updates are also rate-limited. However 1612 * as it is faults that reset the window, pte updates will happen unconditionally 1613 * if there has not been a fault since @pteupdate_interval_millisecs after the 1614 * throttle window closed. 1615 */ 1616 static unsigned int migrate_interval_millisecs __read_mostly = 100; 1617 static unsigned int pteupdate_interval_millisecs __read_mostly = 1000; 1618 static unsigned int ratelimit_pages __read_mostly = 128 << (20 - PAGE_SHIFT); 1619 1620 /* Returns true if NUMA migration is currently rate limited */ 1621 bool migrate_ratelimited(int node) 1622 { 1623 pg_data_t *pgdat = NODE_DATA(node); 1624 1625 if (time_after(jiffies, pgdat->numabalancing_migrate_next_window + 1626 msecs_to_jiffies(pteupdate_interval_millisecs))) 1627 return false; 1628 1629 if (pgdat->numabalancing_migrate_nr_pages < ratelimit_pages) 1630 return false; 1631 1632 return true; 1633 } 1634 1635 /* Returns true if the node is migrate rate-limited after the update */ 1636 static bool numamigrate_update_ratelimit(pg_data_t *pgdat, 1637 unsigned long nr_pages) 1638 { 1639 /* 1640 * Rate-limit the amount of data that is being migrated to a node. 1641 * Optimal placement is no good if the memory bus is saturated and 1642 * all the time is being spent migrating! 1643 */ 1644 if (time_after(jiffies, pgdat->numabalancing_migrate_next_window)) { 1645 spin_lock(&pgdat->numabalancing_migrate_lock); 1646 pgdat->numabalancing_migrate_nr_pages = 0; 1647 pgdat->numabalancing_migrate_next_window = jiffies + 1648 msecs_to_jiffies(migrate_interval_millisecs); 1649 spin_unlock(&pgdat->numabalancing_migrate_lock); 1650 } 1651 if (pgdat->numabalancing_migrate_nr_pages > ratelimit_pages) { 1652 trace_mm_numa_migrate_ratelimit(current, pgdat->node_id, 1653 nr_pages); 1654 return true; 1655 } 1656 1657 /* 1658 * This is an unlocked non-atomic update so errors are possible. 1659 * The consequences are failing to migrate when we potentiall should 1660 * have which is not severe enough to warrant locking. If it is ever 1661 * a problem, it can be converted to a per-cpu counter. 1662 */ 1663 pgdat->numabalancing_migrate_nr_pages += nr_pages; 1664 return false; 1665 } 1666 1667 static int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page) 1668 { 1669 int page_lru; 1670 1671 VM_BUG_ON_PAGE(compound_order(page) && !PageTransHuge(page), page); 1672 1673 /* Avoid migrating to a node that is nearly full */ 1674 if (!migrate_balanced_pgdat(pgdat, 1UL << compound_order(page))) 1675 return 0; 1676 1677 if (isolate_lru_page(page)) 1678 return 0; 1679 1680 /* 1681 * migrate_misplaced_transhuge_page() skips page migration's usual 1682 * check on page_count(), so we must do it here, now that the page 1683 * has been isolated: a GUP pin, or any other pin, prevents migration. 1684 * The expected page count is 3: 1 for page's mapcount and 1 for the 1685 * caller's pin and 1 for the reference taken by isolate_lru_page(). 1686 */ 1687 if (PageTransHuge(page) && page_count(page) != 3) { 1688 putback_lru_page(page); 1689 return 0; 1690 } 1691 1692 page_lru = page_is_file_cache(page); 1693 mod_zone_page_state(page_zone(page), NR_ISOLATED_ANON + page_lru, 1694 hpage_nr_pages(page)); 1695 1696 /* 1697 * Isolating the page has taken another reference, so the 1698 * caller's reference can be safely dropped without the page 1699 * disappearing underneath us during migration. 1700 */ 1701 put_page(page); 1702 return 1; 1703 } 1704 1705 bool pmd_trans_migrating(pmd_t pmd) 1706 { 1707 struct page *page = pmd_page(pmd); 1708 return PageLocked(page); 1709 } 1710 1711 void wait_migrate_huge_page(struct anon_vma *anon_vma, pmd_t *pmd) 1712 { 1713 struct page *page = pmd_page(*pmd); 1714 wait_on_page_locked(page); 1715 } 1716 1717 /* 1718 * Attempt to migrate a misplaced page to the specified destination 1719 * node. Caller is expected to have an elevated reference count on 1720 * the page that will be dropped by this function before returning. 1721 */ 1722 int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma, 1723 int node) 1724 { 1725 pg_data_t *pgdat = NODE_DATA(node); 1726 int isolated; 1727 int nr_remaining; 1728 LIST_HEAD(migratepages); 1729 1730 /* 1731 * Don't migrate file pages that are mapped in multiple processes 1732 * with execute permissions as they are probably shared libraries. 1733 */ 1734 if (page_mapcount(page) != 1 && page_is_file_cache(page) && 1735 (vma->vm_flags & VM_EXEC)) 1736 goto out; 1737 1738 /* 1739 * Rate-limit the amount of data that is being migrated to a node. 1740 * Optimal placement is no good if the memory bus is saturated and 1741 * all the time is being spent migrating! 1742 */ 1743 if (numamigrate_update_ratelimit(pgdat, 1)) 1744 goto out; 1745 1746 isolated = numamigrate_isolate_page(pgdat, page); 1747 if (!isolated) 1748 goto out; 1749 1750 list_add(&page->lru, &migratepages); 1751 nr_remaining = migrate_pages(&migratepages, alloc_misplaced_dst_page, 1752 NULL, node, MIGRATE_ASYNC, 1753 MR_NUMA_MISPLACED); 1754 if (nr_remaining) { 1755 if (!list_empty(&migratepages)) { 1756 list_del(&page->lru); 1757 dec_zone_page_state(page, NR_ISOLATED_ANON + 1758 page_is_file_cache(page)); 1759 putback_lru_page(page); 1760 } 1761 isolated = 0; 1762 } else 1763 count_vm_numa_event(NUMA_PAGE_MIGRATE); 1764 BUG_ON(!list_empty(&migratepages)); 1765 return isolated; 1766 1767 out: 1768 put_page(page); 1769 return 0; 1770 } 1771 #endif /* CONFIG_NUMA_BALANCING */ 1772 1773 #if defined(CONFIG_NUMA_BALANCING) && defined(CONFIG_TRANSPARENT_HUGEPAGE) 1774 /* 1775 * Migrates a THP to a given target node. page must be locked and is unlocked 1776 * before returning. 1777 */ 1778 int migrate_misplaced_transhuge_page(struct mm_struct *mm, 1779 struct vm_area_struct *vma, 1780 pmd_t *pmd, pmd_t entry, 1781 unsigned long address, 1782 struct page *page, int node) 1783 { 1784 spinlock_t *ptl; 1785 pg_data_t *pgdat = NODE_DATA(node); 1786 int isolated = 0; 1787 struct page *new_page = NULL; 1788 struct mem_cgroup *memcg = NULL; 1789 int page_lru = page_is_file_cache(page); 1790 unsigned long mmun_start = address & HPAGE_PMD_MASK; 1791 unsigned long mmun_end = mmun_start + HPAGE_PMD_SIZE; 1792 pmd_t orig_entry; 1793 1794 /* 1795 * Rate-limit the amount of data that is being migrated to a node. 1796 * Optimal placement is no good if the memory bus is saturated and 1797 * all the time is being spent migrating! 1798 */ 1799 if (numamigrate_update_ratelimit(pgdat, HPAGE_PMD_NR)) 1800 goto out_dropref; 1801 1802 new_page = alloc_pages_node(node, 1803 (GFP_TRANSHUGE | __GFP_THISNODE) & ~__GFP_WAIT, 1804 HPAGE_PMD_ORDER); 1805 if (!new_page) 1806 goto out_fail; 1807 1808 isolated = numamigrate_isolate_page(pgdat, page); 1809 if (!isolated) { 1810 put_page(new_page); 1811 goto out_fail; 1812 } 1813 1814 if (mm_tlb_flush_pending(mm)) 1815 flush_tlb_range(vma, mmun_start, mmun_end); 1816 1817 /* Prepare a page as a migration target */ 1818 __set_page_locked(new_page); 1819 SetPageSwapBacked(new_page); 1820 1821 /* anon mapping, we can simply copy page->mapping to the new page: */ 1822 new_page->mapping = page->mapping; 1823 new_page->index = page->index; 1824 migrate_page_copy(new_page, page); 1825 WARN_ON(PageLRU(new_page)); 1826 1827 /* Recheck the target PMD */ 1828 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 1829 ptl = pmd_lock(mm, pmd); 1830 if (unlikely(!pmd_same(*pmd, entry) || page_count(page) != 2)) { 1831 fail_putback: 1832 spin_unlock(ptl); 1833 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 1834 1835 /* Reverse changes made by migrate_page_copy() */ 1836 if (TestClearPageActive(new_page)) 1837 SetPageActive(page); 1838 if (TestClearPageUnevictable(new_page)) 1839 SetPageUnevictable(page); 1840 mlock_migrate_page(page, new_page); 1841 1842 unlock_page(new_page); 1843 put_page(new_page); /* Free it */ 1844 1845 /* Retake the callers reference and putback on LRU */ 1846 get_page(page); 1847 putback_lru_page(page); 1848 mod_zone_page_state(page_zone(page), 1849 NR_ISOLATED_ANON + page_lru, -HPAGE_PMD_NR); 1850 1851 goto out_unlock; 1852 } 1853 1854 /* 1855 * Traditional migration needs to prepare the memcg charge 1856 * transaction early to prevent the old page from being 1857 * uncharged when installing migration entries. Here we can 1858 * save the potential rollback and start the charge transfer 1859 * only when migration is already known to end successfully. 1860 */ 1861 mem_cgroup_prepare_migration(page, new_page, &memcg); 1862 1863 orig_entry = *pmd; 1864 entry = mk_pmd(new_page, vma->vm_page_prot); 1865 entry = pmd_mkhuge(entry); 1866 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1867 1868 /* 1869 * Clear the old entry under pagetable lock and establish the new PTE. 1870 * Any parallel GUP will either observe the old page blocking on the 1871 * page lock, block on the page table lock or observe the new page. 1872 * The SetPageUptodate on the new page and page_add_new_anon_rmap 1873 * guarantee the copy is visible before the pagetable update. 1874 */ 1875 flush_cache_range(vma, mmun_start, mmun_end); 1876 page_add_anon_rmap(new_page, vma, mmun_start); 1877 pmdp_clear_flush(vma, mmun_start, pmd); 1878 set_pmd_at(mm, mmun_start, pmd, entry); 1879 flush_tlb_range(vma, mmun_start, mmun_end); 1880 update_mmu_cache_pmd(vma, address, &entry); 1881 1882 if (page_count(page) != 2) { 1883 set_pmd_at(mm, mmun_start, pmd, orig_entry); 1884 flush_tlb_range(vma, mmun_start, mmun_end); 1885 update_mmu_cache_pmd(vma, address, &entry); 1886 page_remove_rmap(new_page); 1887 goto fail_putback; 1888 } 1889 1890 page_remove_rmap(page); 1891 1892 /* 1893 * Finish the charge transaction under the page table lock to 1894 * prevent split_huge_page() from dividing up the charge 1895 * before it's fully transferred to the new page. 1896 */ 1897 mem_cgroup_end_migration(memcg, page, new_page, true); 1898 spin_unlock(ptl); 1899 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 1900 1901 /* Take an "isolate" reference and put new page on the LRU. */ 1902 get_page(new_page); 1903 putback_lru_page(new_page); 1904 1905 unlock_page(new_page); 1906 unlock_page(page); 1907 put_page(page); /* Drop the rmap reference */ 1908 put_page(page); /* Drop the LRU isolation reference */ 1909 1910 count_vm_events(PGMIGRATE_SUCCESS, HPAGE_PMD_NR); 1911 count_vm_numa_events(NUMA_PAGE_MIGRATE, HPAGE_PMD_NR); 1912 1913 mod_zone_page_state(page_zone(page), 1914 NR_ISOLATED_ANON + page_lru, 1915 -HPAGE_PMD_NR); 1916 return isolated; 1917 1918 out_fail: 1919 count_vm_events(PGMIGRATE_FAIL, HPAGE_PMD_NR); 1920 out_dropref: 1921 ptl = pmd_lock(mm, pmd); 1922 if (pmd_same(*pmd, entry)) { 1923 entry = pmd_mknonnuma(entry); 1924 set_pmd_at(mm, mmun_start, pmd, entry); 1925 update_mmu_cache_pmd(vma, address, &entry); 1926 } 1927 spin_unlock(ptl); 1928 1929 out_unlock: 1930 unlock_page(page); 1931 put_page(page); 1932 return 0; 1933 } 1934 #endif /* CONFIG_NUMA_BALANCING */ 1935 1936 #endif /* CONFIG_NUMA */ 1937