1 /* 2 * Memory Migration functionality - linux/mm/migration.c 3 * 4 * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter 5 * 6 * Page migration was first developed in the context of the memory hotplug 7 * project. The main authors of the migration code are: 8 * 9 * IWAMOTO Toshihiro <iwamoto@valinux.co.jp> 10 * Hirokazu Takahashi <taka@valinux.co.jp> 11 * Dave Hansen <haveblue@us.ibm.com> 12 * Christoph Lameter <clameter@sgi.com> 13 */ 14 15 #include <linux/migrate.h> 16 #include <linux/module.h> 17 #include <linux/swap.h> 18 #include <linux/swapops.h> 19 #include <linux/pagemap.h> 20 #include <linux/buffer_head.h> 21 #include <linux/mm_inline.h> 22 #include <linux/pagevec.h> 23 #include <linux/rmap.h> 24 #include <linux/topology.h> 25 #include <linux/cpu.h> 26 #include <linux/cpuset.h> 27 #include <linux/writeback.h> 28 #include <linux/mempolicy.h> 29 #include <linux/vmalloc.h> 30 #include <linux/security.h> 31 32 #include "internal.h" 33 34 #define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru)) 35 36 /* 37 * Isolate one page from the LRU lists. If successful put it onto 38 * the indicated list with elevated page count. 39 * 40 * Result: 41 * -EBUSY: page not on LRU list 42 * 0: page removed from LRU list and added to the specified list. 43 */ 44 int isolate_lru_page(struct page *page, struct list_head *pagelist) 45 { 46 int ret = -EBUSY; 47 48 if (PageLRU(page)) { 49 struct zone *zone = page_zone(page); 50 51 spin_lock_irq(&zone->lru_lock); 52 if (PageLRU(page)) { 53 ret = 0; 54 get_page(page); 55 ClearPageLRU(page); 56 if (PageActive(page)) 57 del_page_from_active_list(zone, page); 58 else 59 del_page_from_inactive_list(zone, page); 60 list_add_tail(&page->lru, pagelist); 61 } 62 spin_unlock_irq(&zone->lru_lock); 63 } 64 return ret; 65 } 66 67 /* 68 * migrate_prep() needs to be called before we start compiling a list of pages 69 * to be migrated using isolate_lru_page(). 70 */ 71 int migrate_prep(void) 72 { 73 /* 74 * Clear the LRU lists so pages can be isolated. 75 * Note that pages may be moved off the LRU after we have 76 * drained them. Those pages will fail to migrate like other 77 * pages that may be busy. 78 */ 79 lru_add_drain_all(); 80 81 return 0; 82 } 83 84 static inline void move_to_lru(struct page *page) 85 { 86 if (PageActive(page)) { 87 /* 88 * lru_cache_add_active checks that 89 * the PG_active bit is off. 90 */ 91 ClearPageActive(page); 92 lru_cache_add_active(page); 93 } else { 94 lru_cache_add(page); 95 } 96 put_page(page); 97 } 98 99 /* 100 * Add isolated pages on the list back to the LRU. 101 * 102 * returns the number of pages put back. 103 */ 104 int putback_lru_pages(struct list_head *l) 105 { 106 struct page *page; 107 struct page *page2; 108 int count = 0; 109 110 list_for_each_entry_safe(page, page2, l, lru) { 111 list_del(&page->lru); 112 move_to_lru(page); 113 count++; 114 } 115 return count; 116 } 117 118 static inline int is_swap_pte(pte_t pte) 119 { 120 return !pte_none(pte) && !pte_present(pte) && !pte_file(pte); 121 } 122 123 /* 124 * Restore a potential migration pte to a working pte entry 125 */ 126 static void remove_migration_pte(struct vm_area_struct *vma, 127 struct page *old, struct page *new) 128 { 129 struct mm_struct *mm = vma->vm_mm; 130 swp_entry_t entry; 131 pgd_t *pgd; 132 pud_t *pud; 133 pmd_t *pmd; 134 pte_t *ptep, pte; 135 spinlock_t *ptl; 136 unsigned long addr = page_address_in_vma(new, vma); 137 138 if (addr == -EFAULT) 139 return; 140 141 pgd = pgd_offset(mm, addr); 142 if (!pgd_present(*pgd)) 143 return; 144 145 pud = pud_offset(pgd, addr); 146 if (!pud_present(*pud)) 147 return; 148 149 pmd = pmd_offset(pud, addr); 150 if (!pmd_present(*pmd)) 151 return; 152 153 ptep = pte_offset_map(pmd, addr); 154 155 if (!is_swap_pte(*ptep)) { 156 pte_unmap(ptep); 157 return; 158 } 159 160 ptl = pte_lockptr(mm, pmd); 161 spin_lock(ptl); 162 pte = *ptep; 163 if (!is_swap_pte(pte)) 164 goto out; 165 166 entry = pte_to_swp_entry(pte); 167 168 if (!is_migration_entry(entry) || migration_entry_to_page(entry) != old) 169 goto out; 170 171 get_page(new); 172 pte = pte_mkold(mk_pte(new, vma->vm_page_prot)); 173 if (is_write_migration_entry(entry)) 174 pte = pte_mkwrite(pte); 175 set_pte_at(mm, addr, ptep, pte); 176 177 if (PageAnon(new)) 178 page_add_anon_rmap(new, vma, addr); 179 else 180 page_add_file_rmap(new); 181 182 /* No need to invalidate - it was non-present before */ 183 update_mmu_cache(vma, addr, pte); 184 lazy_mmu_prot_update(pte); 185 186 out: 187 pte_unmap_unlock(ptep, ptl); 188 } 189 190 /* 191 * Note that remove_file_migration_ptes will only work on regular mappings, 192 * Nonlinear mappings do not use migration entries. 193 */ 194 static void remove_file_migration_ptes(struct page *old, struct page *new) 195 { 196 struct vm_area_struct *vma; 197 struct address_space *mapping = page_mapping(new); 198 struct prio_tree_iter iter; 199 pgoff_t pgoff = new->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT); 200 201 if (!mapping) 202 return; 203 204 spin_lock(&mapping->i_mmap_lock); 205 206 vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff) 207 remove_migration_pte(vma, old, new); 208 209 spin_unlock(&mapping->i_mmap_lock); 210 } 211 212 /* 213 * Must hold mmap_sem lock on at least one of the vmas containing 214 * the page so that the anon_vma cannot vanish. 215 */ 216 static void remove_anon_migration_ptes(struct page *old, struct page *new) 217 { 218 struct anon_vma *anon_vma; 219 struct vm_area_struct *vma; 220 unsigned long mapping; 221 222 mapping = (unsigned long)new->mapping; 223 224 if (!mapping || (mapping & PAGE_MAPPING_ANON) == 0) 225 return; 226 227 /* 228 * We hold the mmap_sem lock. So no need to call page_lock_anon_vma. 229 */ 230 anon_vma = (struct anon_vma *) (mapping - PAGE_MAPPING_ANON); 231 spin_lock(&anon_vma->lock); 232 233 list_for_each_entry(vma, &anon_vma->head, anon_vma_node) 234 remove_migration_pte(vma, old, new); 235 236 spin_unlock(&anon_vma->lock); 237 } 238 239 /* 240 * Get rid of all migration entries and replace them by 241 * references to the indicated page. 242 */ 243 static void remove_migration_ptes(struct page *old, struct page *new) 244 { 245 if (PageAnon(new)) 246 remove_anon_migration_ptes(old, new); 247 else 248 remove_file_migration_ptes(old, new); 249 } 250 251 /* 252 * Something used the pte of a page under migration. We need to 253 * get to the page and wait until migration is finished. 254 * When we return from this function the fault will be retried. 255 * 256 * This function is called from do_swap_page(). 257 */ 258 void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, 259 unsigned long address) 260 { 261 pte_t *ptep, pte; 262 spinlock_t *ptl; 263 swp_entry_t entry; 264 struct page *page; 265 266 ptep = pte_offset_map_lock(mm, pmd, address, &ptl); 267 pte = *ptep; 268 if (!is_swap_pte(pte)) 269 goto out; 270 271 entry = pte_to_swp_entry(pte); 272 if (!is_migration_entry(entry)) 273 goto out; 274 275 page = migration_entry_to_page(entry); 276 277 get_page(page); 278 pte_unmap_unlock(ptep, ptl); 279 wait_on_page_locked(page); 280 put_page(page); 281 return; 282 out: 283 pte_unmap_unlock(ptep, ptl); 284 } 285 286 /* 287 * Replace the page in the mapping. 288 * 289 * The number of remaining references must be: 290 * 1 for anonymous pages without a mapping 291 * 2 for pages with a mapping 292 * 3 for pages with a mapping and PagePrivate set. 293 */ 294 static int migrate_page_move_mapping(struct address_space *mapping, 295 struct page *newpage, struct page *page) 296 { 297 void **pslot; 298 299 if (!mapping) { 300 /* Anonymous page */ 301 if (page_count(page) != 1) 302 return -EAGAIN; 303 return 0; 304 } 305 306 write_lock_irq(&mapping->tree_lock); 307 308 pslot = radix_tree_lookup_slot(&mapping->page_tree, 309 page_index(page)); 310 311 if (page_count(page) != 2 + !!PagePrivate(page) || 312 (struct page *)radix_tree_deref_slot(pslot) != page) { 313 write_unlock_irq(&mapping->tree_lock); 314 return -EAGAIN; 315 } 316 317 /* 318 * Now we know that no one else is looking at the page. 319 */ 320 get_page(newpage); /* add cache reference */ 321 #ifdef CONFIG_SWAP 322 if (PageSwapCache(page)) { 323 SetPageSwapCache(newpage); 324 set_page_private(newpage, page_private(page)); 325 } 326 #endif 327 328 radix_tree_replace_slot(pslot, newpage); 329 330 /* 331 * Drop cache reference from old page. 332 * We know this isn't the last reference. 333 */ 334 __put_page(page); 335 336 write_unlock_irq(&mapping->tree_lock); 337 338 return 0; 339 } 340 341 /* 342 * Copy the page to its new location 343 */ 344 static void migrate_page_copy(struct page *newpage, struct page *page) 345 { 346 copy_highpage(newpage, page); 347 348 if (PageError(page)) 349 SetPageError(newpage); 350 if (PageReferenced(page)) 351 SetPageReferenced(newpage); 352 if (PageUptodate(page)) 353 SetPageUptodate(newpage); 354 if (PageActive(page)) 355 SetPageActive(newpage); 356 if (PageChecked(page)) 357 SetPageChecked(newpage); 358 if (PageMappedToDisk(page)) 359 SetPageMappedToDisk(newpage); 360 361 if (PageDirty(page)) { 362 clear_page_dirty_for_io(page); 363 set_page_dirty(newpage); 364 } 365 366 #ifdef CONFIG_SWAP 367 ClearPageSwapCache(page); 368 #endif 369 ClearPageActive(page); 370 ClearPagePrivate(page); 371 set_page_private(page, 0); 372 page->mapping = NULL; 373 374 /* 375 * If any waiters have accumulated on the new page then 376 * wake them up. 377 */ 378 if (PageWriteback(newpage)) 379 end_page_writeback(newpage); 380 } 381 382 /************************************************************ 383 * Migration functions 384 ***********************************************************/ 385 386 /* Always fail migration. Used for mappings that are not movable */ 387 int fail_migrate_page(struct address_space *mapping, 388 struct page *newpage, struct page *page) 389 { 390 return -EIO; 391 } 392 EXPORT_SYMBOL(fail_migrate_page); 393 394 /* 395 * Common logic to directly migrate a single page suitable for 396 * pages that do not use PagePrivate. 397 * 398 * Pages are locked upon entry and exit. 399 */ 400 int migrate_page(struct address_space *mapping, 401 struct page *newpage, struct page *page) 402 { 403 int rc; 404 405 BUG_ON(PageWriteback(page)); /* Writeback must be complete */ 406 407 rc = migrate_page_move_mapping(mapping, newpage, page); 408 409 if (rc) 410 return rc; 411 412 migrate_page_copy(newpage, page); 413 return 0; 414 } 415 EXPORT_SYMBOL(migrate_page); 416 417 #ifdef CONFIG_BLOCK 418 /* 419 * Migration function for pages with buffers. This function can only be used 420 * if the underlying filesystem guarantees that no other references to "page" 421 * exist. 422 */ 423 int buffer_migrate_page(struct address_space *mapping, 424 struct page *newpage, struct page *page) 425 { 426 struct buffer_head *bh, *head; 427 int rc; 428 429 if (!page_has_buffers(page)) 430 return migrate_page(mapping, newpage, page); 431 432 head = page_buffers(page); 433 434 rc = migrate_page_move_mapping(mapping, newpage, page); 435 436 if (rc) 437 return rc; 438 439 bh = head; 440 do { 441 get_bh(bh); 442 lock_buffer(bh); 443 bh = bh->b_this_page; 444 445 } while (bh != head); 446 447 ClearPagePrivate(page); 448 set_page_private(newpage, page_private(page)); 449 set_page_private(page, 0); 450 put_page(page); 451 get_page(newpage); 452 453 bh = head; 454 do { 455 set_bh_page(bh, newpage, bh_offset(bh)); 456 bh = bh->b_this_page; 457 458 } while (bh != head); 459 460 SetPagePrivate(newpage); 461 462 migrate_page_copy(newpage, page); 463 464 bh = head; 465 do { 466 unlock_buffer(bh); 467 put_bh(bh); 468 bh = bh->b_this_page; 469 470 } while (bh != head); 471 472 return 0; 473 } 474 EXPORT_SYMBOL(buffer_migrate_page); 475 #endif 476 477 /* 478 * Writeback a page to clean the dirty state 479 */ 480 static int writeout(struct address_space *mapping, struct page *page) 481 { 482 struct writeback_control wbc = { 483 .sync_mode = WB_SYNC_NONE, 484 .nr_to_write = 1, 485 .range_start = 0, 486 .range_end = LLONG_MAX, 487 .nonblocking = 1, 488 .for_reclaim = 1 489 }; 490 int rc; 491 492 if (!mapping->a_ops->writepage) 493 /* No write method for the address space */ 494 return -EINVAL; 495 496 if (!clear_page_dirty_for_io(page)) 497 /* Someone else already triggered a write */ 498 return -EAGAIN; 499 500 /* 501 * A dirty page may imply that the underlying filesystem has 502 * the page on some queue. So the page must be clean for 503 * migration. Writeout may mean we loose the lock and the 504 * page state is no longer what we checked for earlier. 505 * At this point we know that the migration attempt cannot 506 * be successful. 507 */ 508 remove_migration_ptes(page, page); 509 510 rc = mapping->a_ops->writepage(page, &wbc); 511 if (rc < 0) 512 /* I/O Error writing */ 513 return -EIO; 514 515 if (rc != AOP_WRITEPAGE_ACTIVATE) 516 /* unlocked. Relock */ 517 lock_page(page); 518 519 return -EAGAIN; 520 } 521 522 /* 523 * Default handling if a filesystem does not provide a migration function. 524 */ 525 static int fallback_migrate_page(struct address_space *mapping, 526 struct page *newpage, struct page *page) 527 { 528 if (PageDirty(page)) 529 return writeout(mapping, page); 530 531 /* 532 * Buffers may be managed in a filesystem specific way. 533 * We must have no buffers or drop them. 534 */ 535 if (PagePrivate(page) && 536 !try_to_release_page(page, GFP_KERNEL)) 537 return -EAGAIN; 538 539 return migrate_page(mapping, newpage, page); 540 } 541 542 /* 543 * Move a page to a newly allocated page 544 * The page is locked and all ptes have been successfully removed. 545 * 546 * The new page will have replaced the old page if this function 547 * is successful. 548 */ 549 static int move_to_new_page(struct page *newpage, struct page *page) 550 { 551 struct address_space *mapping; 552 int rc; 553 554 /* 555 * Block others from accessing the page when we get around to 556 * establishing additional references. We are the only one 557 * holding a reference to the new page at this point. 558 */ 559 if (TestSetPageLocked(newpage)) 560 BUG(); 561 562 /* Prepare mapping for the new page.*/ 563 newpage->index = page->index; 564 newpage->mapping = page->mapping; 565 566 mapping = page_mapping(page); 567 if (!mapping) 568 rc = migrate_page(mapping, newpage, page); 569 else if (mapping->a_ops->migratepage) 570 /* 571 * Most pages have a mapping and most filesystems 572 * should provide a migration function. Anonymous 573 * pages are part of swap space which also has its 574 * own migration function. This is the most common 575 * path for page migration. 576 */ 577 rc = mapping->a_ops->migratepage(mapping, 578 newpage, page); 579 else 580 rc = fallback_migrate_page(mapping, newpage, page); 581 582 if (!rc) 583 remove_migration_ptes(page, newpage); 584 else 585 newpage->mapping = NULL; 586 587 unlock_page(newpage); 588 589 return rc; 590 } 591 592 /* 593 * Obtain the lock on page, remove all ptes and migrate the page 594 * to the newly allocated page in newpage. 595 */ 596 static int unmap_and_move(new_page_t get_new_page, unsigned long private, 597 struct page *page, int force) 598 { 599 int rc = 0; 600 int *result = NULL; 601 struct page *newpage = get_new_page(page, private, &result); 602 603 if (!newpage) 604 return -ENOMEM; 605 606 if (page_count(page) == 1) 607 /* page was freed from under us. So we are done. */ 608 goto move_newpage; 609 610 rc = -EAGAIN; 611 if (TestSetPageLocked(page)) { 612 if (!force) 613 goto move_newpage; 614 lock_page(page); 615 } 616 617 if (PageWriteback(page)) { 618 if (!force) 619 goto unlock; 620 wait_on_page_writeback(page); 621 } 622 623 /* 624 * Establish migration ptes or remove ptes 625 */ 626 try_to_unmap(page, 1); 627 if (!page_mapped(page)) 628 rc = move_to_new_page(newpage, page); 629 630 if (rc) 631 remove_migration_ptes(page, page); 632 633 unlock: 634 unlock_page(page); 635 636 if (rc != -EAGAIN) { 637 /* 638 * A page that has been migrated has all references 639 * removed and will be freed. A page that has not been 640 * migrated will have kepts its references and be 641 * restored. 642 */ 643 list_del(&page->lru); 644 move_to_lru(page); 645 } 646 647 move_newpage: 648 /* 649 * Move the new page to the LRU. If migration was not successful 650 * then this will free the page. 651 */ 652 move_to_lru(newpage); 653 if (result) { 654 if (rc) 655 *result = rc; 656 else 657 *result = page_to_nid(newpage); 658 } 659 return rc; 660 } 661 662 /* 663 * migrate_pages 664 * 665 * The function takes one list of pages to migrate and a function 666 * that determines from the page to be migrated and the private data 667 * the target of the move and allocates the page. 668 * 669 * The function returns after 10 attempts or if no pages 670 * are movable anymore because to has become empty 671 * or no retryable pages exist anymore. All pages will be 672 * retruned to the LRU or freed. 673 * 674 * Return: Number of pages not migrated or error code. 675 */ 676 int migrate_pages(struct list_head *from, 677 new_page_t get_new_page, unsigned long private) 678 { 679 int retry = 1; 680 int nr_failed = 0; 681 int pass = 0; 682 struct page *page; 683 struct page *page2; 684 int swapwrite = current->flags & PF_SWAPWRITE; 685 int rc; 686 687 if (!swapwrite) 688 current->flags |= PF_SWAPWRITE; 689 690 for(pass = 0; pass < 10 && retry; pass++) { 691 retry = 0; 692 693 list_for_each_entry_safe(page, page2, from, lru) { 694 cond_resched(); 695 696 rc = unmap_and_move(get_new_page, private, 697 page, pass > 2); 698 699 switch(rc) { 700 case -ENOMEM: 701 goto out; 702 case -EAGAIN: 703 retry++; 704 break; 705 case 0: 706 break; 707 default: 708 /* Permanent failure */ 709 nr_failed++; 710 break; 711 } 712 } 713 } 714 rc = 0; 715 out: 716 if (!swapwrite) 717 current->flags &= ~PF_SWAPWRITE; 718 719 putback_lru_pages(from); 720 721 if (rc) 722 return rc; 723 724 return nr_failed + retry; 725 } 726 727 #ifdef CONFIG_NUMA 728 /* 729 * Move a list of individual pages 730 */ 731 struct page_to_node { 732 unsigned long addr; 733 struct page *page; 734 int node; 735 int status; 736 }; 737 738 static struct page *new_page_node(struct page *p, unsigned long private, 739 int **result) 740 { 741 struct page_to_node *pm = (struct page_to_node *)private; 742 743 while (pm->node != MAX_NUMNODES && pm->page != p) 744 pm++; 745 746 if (pm->node == MAX_NUMNODES) 747 return NULL; 748 749 *result = &pm->status; 750 751 return alloc_pages_node(pm->node, GFP_HIGHUSER | GFP_THISNODE, 0); 752 } 753 754 /* 755 * Move a set of pages as indicated in the pm array. The addr 756 * field must be set to the virtual address of the page to be moved 757 * and the node number must contain a valid target node. 758 */ 759 static int do_move_pages(struct mm_struct *mm, struct page_to_node *pm, 760 int migrate_all) 761 { 762 int err; 763 struct page_to_node *pp; 764 LIST_HEAD(pagelist); 765 766 down_read(&mm->mmap_sem); 767 768 /* 769 * Build a list of pages to migrate 770 */ 771 migrate_prep(); 772 for (pp = pm; pp->node != MAX_NUMNODES; pp++) { 773 struct vm_area_struct *vma; 774 struct page *page; 775 776 /* 777 * A valid page pointer that will not match any of the 778 * pages that will be moved. 779 */ 780 pp->page = ZERO_PAGE(0); 781 782 err = -EFAULT; 783 vma = find_vma(mm, pp->addr); 784 if (!vma || !vma_migratable(vma)) 785 goto set_status; 786 787 page = follow_page(vma, pp->addr, FOLL_GET); 788 err = -ENOENT; 789 if (!page) 790 goto set_status; 791 792 if (PageReserved(page)) /* Check for zero page */ 793 goto put_and_set; 794 795 pp->page = page; 796 err = page_to_nid(page); 797 798 if (err == pp->node) 799 /* 800 * Node already in the right place 801 */ 802 goto put_and_set; 803 804 err = -EACCES; 805 if (page_mapcount(page) > 1 && 806 !migrate_all) 807 goto put_and_set; 808 809 err = isolate_lru_page(page, &pagelist); 810 put_and_set: 811 /* 812 * Either remove the duplicate refcount from 813 * isolate_lru_page() or drop the page ref if it was 814 * not isolated. 815 */ 816 put_page(page); 817 set_status: 818 pp->status = err; 819 } 820 821 if (!list_empty(&pagelist)) 822 err = migrate_pages(&pagelist, new_page_node, 823 (unsigned long)pm); 824 else 825 err = -ENOENT; 826 827 up_read(&mm->mmap_sem); 828 return err; 829 } 830 831 /* 832 * Determine the nodes of a list of pages. The addr in the pm array 833 * must have been set to the virtual address of which we want to determine 834 * the node number. 835 */ 836 static int do_pages_stat(struct mm_struct *mm, struct page_to_node *pm) 837 { 838 down_read(&mm->mmap_sem); 839 840 for ( ; pm->node != MAX_NUMNODES; pm++) { 841 struct vm_area_struct *vma; 842 struct page *page; 843 int err; 844 845 err = -EFAULT; 846 vma = find_vma(mm, pm->addr); 847 if (!vma) 848 goto set_status; 849 850 page = follow_page(vma, pm->addr, 0); 851 err = -ENOENT; 852 /* Use PageReserved to check for zero page */ 853 if (!page || PageReserved(page)) 854 goto set_status; 855 856 err = page_to_nid(page); 857 set_status: 858 pm->status = err; 859 } 860 861 up_read(&mm->mmap_sem); 862 return 0; 863 } 864 865 /* 866 * Move a list of pages in the address space of the currently executing 867 * process. 868 */ 869 asmlinkage long sys_move_pages(pid_t pid, unsigned long nr_pages, 870 const void __user * __user *pages, 871 const int __user *nodes, 872 int __user *status, int flags) 873 { 874 int err = 0; 875 int i; 876 struct task_struct *task; 877 nodemask_t task_nodes; 878 struct mm_struct *mm; 879 struct page_to_node *pm = NULL; 880 881 /* Check flags */ 882 if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL)) 883 return -EINVAL; 884 885 if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE)) 886 return -EPERM; 887 888 /* Find the mm_struct */ 889 read_lock(&tasklist_lock); 890 task = pid ? find_task_by_pid(pid) : current; 891 if (!task) { 892 read_unlock(&tasklist_lock); 893 return -ESRCH; 894 } 895 mm = get_task_mm(task); 896 read_unlock(&tasklist_lock); 897 898 if (!mm) 899 return -EINVAL; 900 901 /* 902 * Check if this process has the right to modify the specified 903 * process. The right exists if the process has administrative 904 * capabilities, superuser privileges or the same 905 * userid as the target process. 906 */ 907 if ((current->euid != task->suid) && (current->euid != task->uid) && 908 (current->uid != task->suid) && (current->uid != task->uid) && 909 !capable(CAP_SYS_NICE)) { 910 err = -EPERM; 911 goto out2; 912 } 913 914 err = security_task_movememory(task); 915 if (err) 916 goto out2; 917 918 919 task_nodes = cpuset_mems_allowed(task); 920 921 /* Limit nr_pages so that the multiplication may not overflow */ 922 if (nr_pages >= ULONG_MAX / sizeof(struct page_to_node) - 1) { 923 err = -E2BIG; 924 goto out2; 925 } 926 927 pm = vmalloc((nr_pages + 1) * sizeof(struct page_to_node)); 928 if (!pm) { 929 err = -ENOMEM; 930 goto out2; 931 } 932 933 /* 934 * Get parameters from user space and initialize the pm 935 * array. Return various errors if the user did something wrong. 936 */ 937 for (i = 0; i < nr_pages; i++) { 938 const void *p; 939 940 err = -EFAULT; 941 if (get_user(p, pages + i)) 942 goto out; 943 944 pm[i].addr = (unsigned long)p; 945 if (nodes) { 946 int node; 947 948 if (get_user(node, nodes + i)) 949 goto out; 950 951 err = -ENODEV; 952 if (!node_online(node)) 953 goto out; 954 955 err = -EACCES; 956 if (!node_isset(node, task_nodes)) 957 goto out; 958 959 pm[i].node = node; 960 } else 961 pm[i].node = 0; /* anything to not match MAX_NUMNODES */ 962 } 963 /* End marker */ 964 pm[nr_pages].node = MAX_NUMNODES; 965 966 if (nodes) 967 err = do_move_pages(mm, pm, flags & MPOL_MF_MOVE_ALL); 968 else 969 err = do_pages_stat(mm, pm); 970 971 if (err >= 0) 972 /* Return status information */ 973 for (i = 0; i < nr_pages; i++) 974 if (put_user(pm[i].status, status + i)) 975 err = -EFAULT; 976 977 out: 978 vfree(pm); 979 out2: 980 mmput(mm); 981 return err; 982 } 983 #endif 984 985 /* 986 * Call migration functions in the vma_ops that may prepare 987 * memory in a vm for migration. migration functions may perform 988 * the migration for vmas that do not have an underlying page struct. 989 */ 990 int migrate_vmas(struct mm_struct *mm, const nodemask_t *to, 991 const nodemask_t *from, unsigned long flags) 992 { 993 struct vm_area_struct *vma; 994 int err = 0; 995 996 for(vma = mm->mmap; vma->vm_next && !err; vma = vma->vm_next) { 997 if (vma->vm_ops && vma->vm_ops->migrate) { 998 err = vma->vm_ops->migrate(vma, to, from, flags); 999 if (err) 1000 break; 1001 } 1002 } 1003 return err; 1004 } 1005