xref: /openbmc/linux/mm/migrate.c (revision e868d61272caa648214046a096e5a6bfc068dc8c)
1 /*
2  * Memory Migration functionality - linux/mm/migration.c
3  *
4  * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter
5  *
6  * Page migration was first developed in the context of the memory hotplug
7  * project. The main authors of the migration code are:
8  *
9  * IWAMOTO Toshihiro <iwamoto@valinux.co.jp>
10  * Hirokazu Takahashi <taka@valinux.co.jp>
11  * Dave Hansen <haveblue@us.ibm.com>
12  * Christoph Lameter <clameter@sgi.com>
13  */
14 
15 #include <linux/migrate.h>
16 #include <linux/module.h>
17 #include <linux/swap.h>
18 #include <linux/swapops.h>
19 #include <linux/pagemap.h>
20 #include <linux/buffer_head.h>
21 #include <linux/mm_inline.h>
22 #include <linux/pagevec.h>
23 #include <linux/rmap.h>
24 #include <linux/topology.h>
25 #include <linux/cpu.h>
26 #include <linux/cpuset.h>
27 #include <linux/writeback.h>
28 #include <linux/mempolicy.h>
29 #include <linux/vmalloc.h>
30 #include <linux/security.h>
31 
32 #include "internal.h"
33 
34 #define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru))
35 
36 /*
37  * Isolate one page from the LRU lists. If successful put it onto
38  * the indicated list with elevated page count.
39  *
40  * Result:
41  *  -EBUSY: page not on LRU list
42  *  0: page removed from LRU list and added to the specified list.
43  */
44 int isolate_lru_page(struct page *page, struct list_head *pagelist)
45 {
46 	int ret = -EBUSY;
47 
48 	if (PageLRU(page)) {
49 		struct zone *zone = page_zone(page);
50 
51 		spin_lock_irq(&zone->lru_lock);
52 		if (PageLRU(page)) {
53 			ret = 0;
54 			get_page(page);
55 			ClearPageLRU(page);
56 			if (PageActive(page))
57 				del_page_from_active_list(zone, page);
58 			else
59 				del_page_from_inactive_list(zone, page);
60 			list_add_tail(&page->lru, pagelist);
61 		}
62 		spin_unlock_irq(&zone->lru_lock);
63 	}
64 	return ret;
65 }
66 
67 /*
68  * migrate_prep() needs to be called before we start compiling a list of pages
69  * to be migrated using isolate_lru_page().
70  */
71 int migrate_prep(void)
72 {
73 	/*
74 	 * Clear the LRU lists so pages can be isolated.
75 	 * Note that pages may be moved off the LRU after we have
76 	 * drained them. Those pages will fail to migrate like other
77 	 * pages that may be busy.
78 	 */
79 	lru_add_drain_all();
80 
81 	return 0;
82 }
83 
84 static inline void move_to_lru(struct page *page)
85 {
86 	if (PageActive(page)) {
87 		/*
88 		 * lru_cache_add_active checks that
89 		 * the PG_active bit is off.
90 		 */
91 		ClearPageActive(page);
92 		lru_cache_add_active(page);
93 	} else {
94 		lru_cache_add(page);
95 	}
96 	put_page(page);
97 }
98 
99 /*
100  * Add isolated pages on the list back to the LRU.
101  *
102  * returns the number of pages put back.
103  */
104 int putback_lru_pages(struct list_head *l)
105 {
106 	struct page *page;
107 	struct page *page2;
108 	int count = 0;
109 
110 	list_for_each_entry_safe(page, page2, l, lru) {
111 		list_del(&page->lru);
112 		move_to_lru(page);
113 		count++;
114 	}
115 	return count;
116 }
117 
118 static inline int is_swap_pte(pte_t pte)
119 {
120 	return !pte_none(pte) && !pte_present(pte) && !pte_file(pte);
121 }
122 
123 /*
124  * Restore a potential migration pte to a working pte entry
125  */
126 static void remove_migration_pte(struct vm_area_struct *vma,
127 		struct page *old, struct page *new)
128 {
129 	struct mm_struct *mm = vma->vm_mm;
130 	swp_entry_t entry;
131  	pgd_t *pgd;
132  	pud_t *pud;
133  	pmd_t *pmd;
134 	pte_t *ptep, pte;
135  	spinlock_t *ptl;
136 	unsigned long addr = page_address_in_vma(new, vma);
137 
138 	if (addr == -EFAULT)
139 		return;
140 
141  	pgd = pgd_offset(mm, addr);
142 	if (!pgd_present(*pgd))
143                 return;
144 
145 	pud = pud_offset(pgd, addr);
146 	if (!pud_present(*pud))
147                 return;
148 
149 	pmd = pmd_offset(pud, addr);
150 	if (!pmd_present(*pmd))
151 		return;
152 
153 	ptep = pte_offset_map(pmd, addr);
154 
155 	if (!is_swap_pte(*ptep)) {
156 		pte_unmap(ptep);
157  		return;
158  	}
159 
160  	ptl = pte_lockptr(mm, pmd);
161  	spin_lock(ptl);
162 	pte = *ptep;
163 	if (!is_swap_pte(pte))
164 		goto out;
165 
166 	entry = pte_to_swp_entry(pte);
167 
168 	if (!is_migration_entry(entry) || migration_entry_to_page(entry) != old)
169 		goto out;
170 
171 	get_page(new);
172 	pte = pte_mkold(mk_pte(new, vma->vm_page_prot));
173 	if (is_write_migration_entry(entry))
174 		pte = pte_mkwrite(pte);
175 	set_pte_at(mm, addr, ptep, pte);
176 
177 	if (PageAnon(new))
178 		page_add_anon_rmap(new, vma, addr);
179 	else
180 		page_add_file_rmap(new);
181 
182 	/* No need to invalidate - it was non-present before */
183 	update_mmu_cache(vma, addr, pte);
184 	lazy_mmu_prot_update(pte);
185 
186 out:
187 	pte_unmap_unlock(ptep, ptl);
188 }
189 
190 /*
191  * Note that remove_file_migration_ptes will only work on regular mappings,
192  * Nonlinear mappings do not use migration entries.
193  */
194 static void remove_file_migration_ptes(struct page *old, struct page *new)
195 {
196 	struct vm_area_struct *vma;
197 	struct address_space *mapping = page_mapping(new);
198 	struct prio_tree_iter iter;
199 	pgoff_t pgoff = new->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
200 
201 	if (!mapping)
202 		return;
203 
204 	spin_lock(&mapping->i_mmap_lock);
205 
206 	vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff)
207 		remove_migration_pte(vma, old, new);
208 
209 	spin_unlock(&mapping->i_mmap_lock);
210 }
211 
212 /*
213  * Must hold mmap_sem lock on at least one of the vmas containing
214  * the page so that the anon_vma cannot vanish.
215  */
216 static void remove_anon_migration_ptes(struct page *old, struct page *new)
217 {
218 	struct anon_vma *anon_vma;
219 	struct vm_area_struct *vma;
220 	unsigned long mapping;
221 
222 	mapping = (unsigned long)new->mapping;
223 
224 	if (!mapping || (mapping & PAGE_MAPPING_ANON) == 0)
225 		return;
226 
227 	/*
228 	 * We hold the mmap_sem lock. So no need to call page_lock_anon_vma.
229 	 */
230 	anon_vma = (struct anon_vma *) (mapping - PAGE_MAPPING_ANON);
231 	spin_lock(&anon_vma->lock);
232 
233 	list_for_each_entry(vma, &anon_vma->head, anon_vma_node)
234 		remove_migration_pte(vma, old, new);
235 
236 	spin_unlock(&anon_vma->lock);
237 }
238 
239 /*
240  * Get rid of all migration entries and replace them by
241  * references to the indicated page.
242  */
243 static void remove_migration_ptes(struct page *old, struct page *new)
244 {
245 	if (PageAnon(new))
246 		remove_anon_migration_ptes(old, new);
247 	else
248 		remove_file_migration_ptes(old, new);
249 }
250 
251 /*
252  * Something used the pte of a page under migration. We need to
253  * get to the page and wait until migration is finished.
254  * When we return from this function the fault will be retried.
255  *
256  * This function is called from do_swap_page().
257  */
258 void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
259 				unsigned long address)
260 {
261 	pte_t *ptep, pte;
262 	spinlock_t *ptl;
263 	swp_entry_t entry;
264 	struct page *page;
265 
266 	ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
267 	pte = *ptep;
268 	if (!is_swap_pte(pte))
269 		goto out;
270 
271 	entry = pte_to_swp_entry(pte);
272 	if (!is_migration_entry(entry))
273 		goto out;
274 
275 	page = migration_entry_to_page(entry);
276 
277 	get_page(page);
278 	pte_unmap_unlock(ptep, ptl);
279 	wait_on_page_locked(page);
280 	put_page(page);
281 	return;
282 out:
283 	pte_unmap_unlock(ptep, ptl);
284 }
285 
286 /*
287  * Replace the page in the mapping.
288  *
289  * The number of remaining references must be:
290  * 1 for anonymous pages without a mapping
291  * 2 for pages with a mapping
292  * 3 for pages with a mapping and PagePrivate set.
293  */
294 static int migrate_page_move_mapping(struct address_space *mapping,
295 		struct page *newpage, struct page *page)
296 {
297 	void **pslot;
298 
299 	if (!mapping) {
300 		/* Anonymous page without mapping */
301 		if (page_count(page) != 1)
302 			return -EAGAIN;
303 		return 0;
304 	}
305 
306 	write_lock_irq(&mapping->tree_lock);
307 
308 	pslot = radix_tree_lookup_slot(&mapping->page_tree,
309  					page_index(page));
310 
311 	if (page_count(page) != 2 + !!PagePrivate(page) ||
312 			(struct page *)radix_tree_deref_slot(pslot) != page) {
313 		write_unlock_irq(&mapping->tree_lock);
314 		return -EAGAIN;
315 	}
316 
317 	/*
318 	 * Now we know that no one else is looking at the page.
319 	 */
320 	get_page(newpage);	/* add cache reference */
321 #ifdef CONFIG_SWAP
322 	if (PageSwapCache(page)) {
323 		SetPageSwapCache(newpage);
324 		set_page_private(newpage, page_private(page));
325 	}
326 #endif
327 
328 	radix_tree_replace_slot(pslot, newpage);
329 
330 	/*
331 	 * Drop cache reference from old page.
332 	 * We know this isn't the last reference.
333 	 */
334 	__put_page(page);
335 
336 	/*
337 	 * If moved to a different zone then also account
338 	 * the page for that zone. Other VM counters will be
339 	 * taken care of when we establish references to the
340 	 * new page and drop references to the old page.
341 	 *
342 	 * Note that anonymous pages are accounted for
343 	 * via NR_FILE_PAGES and NR_ANON_PAGES if they
344 	 * are mapped to swap space.
345 	 */
346 	__dec_zone_page_state(page, NR_FILE_PAGES);
347 	__inc_zone_page_state(newpage, NR_FILE_PAGES);
348 
349 	write_unlock_irq(&mapping->tree_lock);
350 
351 	return 0;
352 }
353 
354 /*
355  * Copy the page to its new location
356  */
357 static void migrate_page_copy(struct page *newpage, struct page *page)
358 {
359 	copy_highpage(newpage, page);
360 
361 	if (PageError(page))
362 		SetPageError(newpage);
363 	if (PageReferenced(page))
364 		SetPageReferenced(newpage);
365 	if (PageUptodate(page))
366 		SetPageUptodate(newpage);
367 	if (PageActive(page))
368 		SetPageActive(newpage);
369 	if (PageChecked(page))
370 		SetPageChecked(newpage);
371 	if (PageMappedToDisk(page))
372 		SetPageMappedToDisk(newpage);
373 
374 	if (PageDirty(page)) {
375 		clear_page_dirty_for_io(page);
376 		set_page_dirty(newpage);
377  	}
378 
379 #ifdef CONFIG_SWAP
380 	ClearPageSwapCache(page);
381 #endif
382 	ClearPageActive(page);
383 	ClearPagePrivate(page);
384 	set_page_private(page, 0);
385 	page->mapping = NULL;
386 
387 	/*
388 	 * If any waiters have accumulated on the new page then
389 	 * wake them up.
390 	 */
391 	if (PageWriteback(newpage))
392 		end_page_writeback(newpage);
393 }
394 
395 /************************************************************
396  *                    Migration functions
397  ***********************************************************/
398 
399 /* Always fail migration. Used for mappings that are not movable */
400 int fail_migrate_page(struct address_space *mapping,
401 			struct page *newpage, struct page *page)
402 {
403 	return -EIO;
404 }
405 EXPORT_SYMBOL(fail_migrate_page);
406 
407 /*
408  * Common logic to directly migrate a single page suitable for
409  * pages that do not use PagePrivate.
410  *
411  * Pages are locked upon entry and exit.
412  */
413 int migrate_page(struct address_space *mapping,
414 		struct page *newpage, struct page *page)
415 {
416 	int rc;
417 
418 	BUG_ON(PageWriteback(page));	/* Writeback must be complete */
419 
420 	rc = migrate_page_move_mapping(mapping, newpage, page);
421 
422 	if (rc)
423 		return rc;
424 
425 	migrate_page_copy(newpage, page);
426 	return 0;
427 }
428 EXPORT_SYMBOL(migrate_page);
429 
430 #ifdef CONFIG_BLOCK
431 /*
432  * Migration function for pages with buffers. This function can only be used
433  * if the underlying filesystem guarantees that no other references to "page"
434  * exist.
435  */
436 int buffer_migrate_page(struct address_space *mapping,
437 		struct page *newpage, struct page *page)
438 {
439 	struct buffer_head *bh, *head;
440 	int rc;
441 
442 	if (!page_has_buffers(page))
443 		return migrate_page(mapping, newpage, page);
444 
445 	head = page_buffers(page);
446 
447 	rc = migrate_page_move_mapping(mapping, newpage, page);
448 
449 	if (rc)
450 		return rc;
451 
452 	bh = head;
453 	do {
454 		get_bh(bh);
455 		lock_buffer(bh);
456 		bh = bh->b_this_page;
457 
458 	} while (bh != head);
459 
460 	ClearPagePrivate(page);
461 	set_page_private(newpage, page_private(page));
462 	set_page_private(page, 0);
463 	put_page(page);
464 	get_page(newpage);
465 
466 	bh = head;
467 	do {
468 		set_bh_page(bh, newpage, bh_offset(bh));
469 		bh = bh->b_this_page;
470 
471 	} while (bh != head);
472 
473 	SetPagePrivate(newpage);
474 
475 	migrate_page_copy(newpage, page);
476 
477 	bh = head;
478 	do {
479 		unlock_buffer(bh);
480  		put_bh(bh);
481 		bh = bh->b_this_page;
482 
483 	} while (bh != head);
484 
485 	return 0;
486 }
487 EXPORT_SYMBOL(buffer_migrate_page);
488 #endif
489 
490 /*
491  * Writeback a page to clean the dirty state
492  */
493 static int writeout(struct address_space *mapping, struct page *page)
494 {
495 	struct writeback_control wbc = {
496 		.sync_mode = WB_SYNC_NONE,
497 		.nr_to_write = 1,
498 		.range_start = 0,
499 		.range_end = LLONG_MAX,
500 		.nonblocking = 1,
501 		.for_reclaim = 1
502 	};
503 	int rc;
504 
505 	if (!mapping->a_ops->writepage)
506 		/* No write method for the address space */
507 		return -EINVAL;
508 
509 	if (!clear_page_dirty_for_io(page))
510 		/* Someone else already triggered a write */
511 		return -EAGAIN;
512 
513 	/*
514 	 * A dirty page may imply that the underlying filesystem has
515 	 * the page on some queue. So the page must be clean for
516 	 * migration. Writeout may mean we loose the lock and the
517 	 * page state is no longer what we checked for earlier.
518 	 * At this point we know that the migration attempt cannot
519 	 * be successful.
520 	 */
521 	remove_migration_ptes(page, page);
522 
523 	rc = mapping->a_ops->writepage(page, &wbc);
524 	if (rc < 0)
525 		/* I/O Error writing */
526 		return -EIO;
527 
528 	if (rc != AOP_WRITEPAGE_ACTIVATE)
529 		/* unlocked. Relock */
530 		lock_page(page);
531 
532 	return -EAGAIN;
533 }
534 
535 /*
536  * Default handling if a filesystem does not provide a migration function.
537  */
538 static int fallback_migrate_page(struct address_space *mapping,
539 	struct page *newpage, struct page *page)
540 {
541 	if (PageDirty(page))
542 		return writeout(mapping, page);
543 
544 	/*
545 	 * Buffers may be managed in a filesystem specific way.
546 	 * We must have no buffers or drop them.
547 	 */
548 	if (PagePrivate(page) &&
549 	    !try_to_release_page(page, GFP_KERNEL))
550 		return -EAGAIN;
551 
552 	return migrate_page(mapping, newpage, page);
553 }
554 
555 /*
556  * Move a page to a newly allocated page
557  * The page is locked and all ptes have been successfully removed.
558  *
559  * The new page will have replaced the old page if this function
560  * is successful.
561  */
562 static int move_to_new_page(struct page *newpage, struct page *page)
563 {
564 	struct address_space *mapping;
565 	int rc;
566 
567 	/*
568 	 * Block others from accessing the page when we get around to
569 	 * establishing additional references. We are the only one
570 	 * holding a reference to the new page at this point.
571 	 */
572 	if (TestSetPageLocked(newpage))
573 		BUG();
574 
575 	/* Prepare mapping for the new page.*/
576 	newpage->index = page->index;
577 	newpage->mapping = page->mapping;
578 
579 	mapping = page_mapping(page);
580 	if (!mapping)
581 		rc = migrate_page(mapping, newpage, page);
582 	else if (mapping->a_ops->migratepage)
583 		/*
584 		 * Most pages have a mapping and most filesystems
585 		 * should provide a migration function. Anonymous
586 		 * pages are part of swap space which also has its
587 		 * own migration function. This is the most common
588 		 * path for page migration.
589 		 */
590 		rc = mapping->a_ops->migratepage(mapping,
591 						newpage, page);
592 	else
593 		rc = fallback_migrate_page(mapping, newpage, page);
594 
595 	if (!rc)
596 		remove_migration_ptes(page, newpage);
597 	else
598 		newpage->mapping = NULL;
599 
600 	unlock_page(newpage);
601 
602 	return rc;
603 }
604 
605 /*
606  * Obtain the lock on page, remove all ptes and migrate the page
607  * to the newly allocated page in newpage.
608  */
609 static int unmap_and_move(new_page_t get_new_page, unsigned long private,
610 			struct page *page, int force)
611 {
612 	int rc = 0;
613 	int *result = NULL;
614 	struct page *newpage = get_new_page(page, private, &result);
615 
616 	if (!newpage)
617 		return -ENOMEM;
618 
619 	if (page_count(page) == 1)
620 		/* page was freed from under us. So we are done. */
621 		goto move_newpage;
622 
623 	rc = -EAGAIN;
624 	if (TestSetPageLocked(page)) {
625 		if (!force)
626 			goto move_newpage;
627 		lock_page(page);
628 	}
629 
630 	if (PageWriteback(page)) {
631 		if (!force)
632 			goto unlock;
633 		wait_on_page_writeback(page);
634 	}
635 
636 	/*
637 	 * Establish migration ptes or remove ptes
638 	 */
639 	try_to_unmap(page, 1);
640 	if (!page_mapped(page))
641 		rc = move_to_new_page(newpage, page);
642 
643 	if (rc)
644 		remove_migration_ptes(page, page);
645 
646 unlock:
647 	unlock_page(page);
648 
649 	if (rc != -EAGAIN) {
650  		/*
651  		 * A page that has been migrated has all references
652  		 * removed and will be freed. A page that has not been
653  		 * migrated will have kepts its references and be
654  		 * restored.
655  		 */
656  		list_del(&page->lru);
657  		move_to_lru(page);
658 	}
659 
660 move_newpage:
661 	/*
662 	 * Move the new page to the LRU. If migration was not successful
663 	 * then this will free the page.
664 	 */
665 	move_to_lru(newpage);
666 	if (result) {
667 		if (rc)
668 			*result = rc;
669 		else
670 			*result = page_to_nid(newpage);
671 	}
672 	return rc;
673 }
674 
675 /*
676  * migrate_pages
677  *
678  * The function takes one list of pages to migrate and a function
679  * that determines from the page to be migrated and the private data
680  * the target of the move and allocates the page.
681  *
682  * The function returns after 10 attempts or if no pages
683  * are movable anymore because to has become empty
684  * or no retryable pages exist anymore. All pages will be
685  * retruned to the LRU or freed.
686  *
687  * Return: Number of pages not migrated or error code.
688  */
689 int migrate_pages(struct list_head *from,
690 		new_page_t get_new_page, unsigned long private)
691 {
692 	int retry = 1;
693 	int nr_failed = 0;
694 	int pass = 0;
695 	struct page *page;
696 	struct page *page2;
697 	int swapwrite = current->flags & PF_SWAPWRITE;
698 	int rc;
699 
700 	if (!swapwrite)
701 		current->flags |= PF_SWAPWRITE;
702 
703 	for(pass = 0; pass < 10 && retry; pass++) {
704 		retry = 0;
705 
706 		list_for_each_entry_safe(page, page2, from, lru) {
707 			cond_resched();
708 
709 			rc = unmap_and_move(get_new_page, private,
710 						page, pass > 2);
711 
712 			switch(rc) {
713 			case -ENOMEM:
714 				goto out;
715 			case -EAGAIN:
716 				retry++;
717 				break;
718 			case 0:
719 				break;
720 			default:
721 				/* Permanent failure */
722 				nr_failed++;
723 				break;
724 			}
725 		}
726 	}
727 	rc = 0;
728 out:
729 	if (!swapwrite)
730 		current->flags &= ~PF_SWAPWRITE;
731 
732 	putback_lru_pages(from);
733 
734 	if (rc)
735 		return rc;
736 
737 	return nr_failed + retry;
738 }
739 
740 #ifdef CONFIG_NUMA
741 /*
742  * Move a list of individual pages
743  */
744 struct page_to_node {
745 	unsigned long addr;
746 	struct page *page;
747 	int node;
748 	int status;
749 };
750 
751 static struct page *new_page_node(struct page *p, unsigned long private,
752 		int **result)
753 {
754 	struct page_to_node *pm = (struct page_to_node *)private;
755 
756 	while (pm->node != MAX_NUMNODES && pm->page != p)
757 		pm++;
758 
759 	if (pm->node == MAX_NUMNODES)
760 		return NULL;
761 
762 	*result = &pm->status;
763 
764 	return alloc_pages_node(pm->node, GFP_HIGHUSER | GFP_THISNODE, 0);
765 }
766 
767 /*
768  * Move a set of pages as indicated in the pm array. The addr
769  * field must be set to the virtual address of the page to be moved
770  * and the node number must contain a valid target node.
771  */
772 static int do_move_pages(struct mm_struct *mm, struct page_to_node *pm,
773 				int migrate_all)
774 {
775 	int err;
776 	struct page_to_node *pp;
777 	LIST_HEAD(pagelist);
778 
779 	down_read(&mm->mmap_sem);
780 
781 	/*
782 	 * Build a list of pages to migrate
783 	 */
784 	migrate_prep();
785 	for (pp = pm; pp->node != MAX_NUMNODES; pp++) {
786 		struct vm_area_struct *vma;
787 		struct page *page;
788 
789 		/*
790 		 * A valid page pointer that will not match any of the
791 		 * pages that will be moved.
792 		 */
793 		pp->page = ZERO_PAGE(0);
794 
795 		err = -EFAULT;
796 		vma = find_vma(mm, pp->addr);
797 		if (!vma || !vma_migratable(vma))
798 			goto set_status;
799 
800 		page = follow_page(vma, pp->addr, FOLL_GET);
801 		err = -ENOENT;
802 		if (!page)
803 			goto set_status;
804 
805 		if (PageReserved(page))		/* Check for zero page */
806 			goto put_and_set;
807 
808 		pp->page = page;
809 		err = page_to_nid(page);
810 
811 		if (err == pp->node)
812 			/*
813 			 * Node already in the right place
814 			 */
815 			goto put_and_set;
816 
817 		err = -EACCES;
818 		if (page_mapcount(page) > 1 &&
819 				!migrate_all)
820 			goto put_and_set;
821 
822 		err = isolate_lru_page(page, &pagelist);
823 put_and_set:
824 		/*
825 		 * Either remove the duplicate refcount from
826 		 * isolate_lru_page() or drop the page ref if it was
827 		 * not isolated.
828 		 */
829 		put_page(page);
830 set_status:
831 		pp->status = err;
832 	}
833 
834 	if (!list_empty(&pagelist))
835 		err = migrate_pages(&pagelist, new_page_node,
836 				(unsigned long)pm);
837 	else
838 		err = -ENOENT;
839 
840 	up_read(&mm->mmap_sem);
841 	return err;
842 }
843 
844 /*
845  * Determine the nodes of a list of pages. The addr in the pm array
846  * must have been set to the virtual address of which we want to determine
847  * the node number.
848  */
849 static int do_pages_stat(struct mm_struct *mm, struct page_to_node *pm)
850 {
851 	down_read(&mm->mmap_sem);
852 
853 	for ( ; pm->node != MAX_NUMNODES; pm++) {
854 		struct vm_area_struct *vma;
855 		struct page *page;
856 		int err;
857 
858 		err = -EFAULT;
859 		vma = find_vma(mm, pm->addr);
860 		if (!vma)
861 			goto set_status;
862 
863 		page = follow_page(vma, pm->addr, 0);
864 		err = -ENOENT;
865 		/* Use PageReserved to check for zero page */
866 		if (!page || PageReserved(page))
867 			goto set_status;
868 
869 		err = page_to_nid(page);
870 set_status:
871 		pm->status = err;
872 	}
873 
874 	up_read(&mm->mmap_sem);
875 	return 0;
876 }
877 
878 /*
879  * Move a list of pages in the address space of the currently executing
880  * process.
881  */
882 asmlinkage long sys_move_pages(pid_t pid, unsigned long nr_pages,
883 			const void __user * __user *pages,
884 			const int __user *nodes,
885 			int __user *status, int flags)
886 {
887 	int err = 0;
888 	int i;
889 	struct task_struct *task;
890 	nodemask_t task_nodes;
891 	struct mm_struct *mm;
892 	struct page_to_node *pm = NULL;
893 
894 	/* Check flags */
895 	if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL))
896 		return -EINVAL;
897 
898 	if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
899 		return -EPERM;
900 
901 	/* Find the mm_struct */
902 	read_lock(&tasklist_lock);
903 	task = pid ? find_task_by_pid(pid) : current;
904 	if (!task) {
905 		read_unlock(&tasklist_lock);
906 		return -ESRCH;
907 	}
908 	mm = get_task_mm(task);
909 	read_unlock(&tasklist_lock);
910 
911 	if (!mm)
912 		return -EINVAL;
913 
914 	/*
915 	 * Check if this process has the right to modify the specified
916 	 * process. The right exists if the process has administrative
917 	 * capabilities, superuser privileges or the same
918 	 * userid as the target process.
919 	 */
920 	if ((current->euid != task->suid) && (current->euid != task->uid) &&
921 	    (current->uid != task->suid) && (current->uid != task->uid) &&
922 	    !capable(CAP_SYS_NICE)) {
923 		err = -EPERM;
924 		goto out2;
925 	}
926 
927  	err = security_task_movememory(task);
928  	if (err)
929  		goto out2;
930 
931 
932 	task_nodes = cpuset_mems_allowed(task);
933 
934 	/* Limit nr_pages so that the multiplication may not overflow */
935 	if (nr_pages >= ULONG_MAX / sizeof(struct page_to_node) - 1) {
936 		err = -E2BIG;
937 		goto out2;
938 	}
939 
940 	pm = vmalloc((nr_pages + 1) * sizeof(struct page_to_node));
941 	if (!pm) {
942 		err = -ENOMEM;
943 		goto out2;
944 	}
945 
946 	/*
947 	 * Get parameters from user space and initialize the pm
948 	 * array. Return various errors if the user did something wrong.
949 	 */
950 	for (i = 0; i < nr_pages; i++) {
951 		const void *p;
952 
953 		err = -EFAULT;
954 		if (get_user(p, pages + i))
955 			goto out;
956 
957 		pm[i].addr = (unsigned long)p;
958 		if (nodes) {
959 			int node;
960 
961 			if (get_user(node, nodes + i))
962 				goto out;
963 
964 			err = -ENODEV;
965 			if (!node_online(node))
966 				goto out;
967 
968 			err = -EACCES;
969 			if (!node_isset(node, task_nodes))
970 				goto out;
971 
972 			pm[i].node = node;
973 		} else
974 			pm[i].node = 0;	/* anything to not match MAX_NUMNODES */
975 	}
976 	/* End marker */
977 	pm[nr_pages].node = MAX_NUMNODES;
978 
979 	if (nodes)
980 		err = do_move_pages(mm, pm, flags & MPOL_MF_MOVE_ALL);
981 	else
982 		err = do_pages_stat(mm, pm);
983 
984 	if (err >= 0)
985 		/* Return status information */
986 		for (i = 0; i < nr_pages; i++)
987 			if (put_user(pm[i].status, status + i))
988 				err = -EFAULT;
989 
990 out:
991 	vfree(pm);
992 out2:
993 	mmput(mm);
994 	return err;
995 }
996 #endif
997 
998 /*
999  * Call migration functions in the vma_ops that may prepare
1000  * memory in a vm for migration. migration functions may perform
1001  * the migration for vmas that do not have an underlying page struct.
1002  */
1003 int migrate_vmas(struct mm_struct *mm, const nodemask_t *to,
1004 	const nodemask_t *from, unsigned long flags)
1005 {
1006  	struct vm_area_struct *vma;
1007  	int err = 0;
1008 
1009  	for(vma = mm->mmap; vma->vm_next && !err; vma = vma->vm_next) {
1010  		if (vma->vm_ops && vma->vm_ops->migrate) {
1011  			err = vma->vm_ops->migrate(vma, to, from, flags);
1012  			if (err)
1013  				break;
1014  		}
1015  	}
1016  	return err;
1017 }
1018