xref: /openbmc/linux/mm/mempolicy.c (revision 636f13c174dd7c84a437d3c3e8fa66f03f7fda63)
11da177e4SLinus Torvalds /*
21da177e4SLinus Torvalds  * Simple NUMA memory policy for the Linux kernel.
31da177e4SLinus Torvalds  *
41da177e4SLinus Torvalds  * Copyright 2003,2004 Andi Kleen, SuSE Labs.
58bccd85fSChristoph Lameter  * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
61da177e4SLinus Torvalds  * Subject to the GNU Public License, version 2.
71da177e4SLinus Torvalds  *
81da177e4SLinus Torvalds  * NUMA policy allows the user to give hints in which node(s) memory should
91da177e4SLinus Torvalds  * be allocated.
101da177e4SLinus Torvalds  *
111da177e4SLinus Torvalds  * Support four policies per VMA and per process:
121da177e4SLinus Torvalds  *
131da177e4SLinus Torvalds  * The VMA policy has priority over the process policy for a page fault.
141da177e4SLinus Torvalds  *
151da177e4SLinus Torvalds  * interleave     Allocate memory interleaved over a set of nodes,
161da177e4SLinus Torvalds  *                with normal fallback if it fails.
171da177e4SLinus Torvalds  *                For VMA based allocations this interleaves based on the
181da177e4SLinus Torvalds  *                offset into the backing object or offset into the mapping
191da177e4SLinus Torvalds  *                for anonymous memory. For process policy an process counter
201da177e4SLinus Torvalds  *                is used.
218bccd85fSChristoph Lameter  *
221da177e4SLinus Torvalds  * bind           Only allocate memory on a specific set of nodes,
231da177e4SLinus Torvalds  *                no fallback.
248bccd85fSChristoph Lameter  *                FIXME: memory is allocated starting with the first node
258bccd85fSChristoph Lameter  *                to the last. It would be better if bind would truly restrict
268bccd85fSChristoph Lameter  *                the allocation to memory nodes instead
278bccd85fSChristoph Lameter  *
281da177e4SLinus Torvalds  * preferred       Try a specific node first before normal fallback.
291da177e4SLinus Torvalds  *                As a special case node -1 here means do the allocation
301da177e4SLinus Torvalds  *                on the local CPU. This is normally identical to default,
311da177e4SLinus Torvalds  *                but useful to set in a VMA when you have a non default
321da177e4SLinus Torvalds  *                process policy.
338bccd85fSChristoph Lameter  *
341da177e4SLinus Torvalds  * default        Allocate on the local node first, or when on a VMA
351da177e4SLinus Torvalds  *                use the process policy. This is what Linux always did
361da177e4SLinus Torvalds  *		  in a NUMA aware kernel and still does by, ahem, default.
371da177e4SLinus Torvalds  *
381da177e4SLinus Torvalds  * The process policy is applied for most non interrupt memory allocations
391da177e4SLinus Torvalds  * in that process' context. Interrupts ignore the policies and always
401da177e4SLinus Torvalds  * try to allocate on the local CPU. The VMA policy is only applied for memory
411da177e4SLinus Torvalds  * allocations for a VMA in the VM.
421da177e4SLinus Torvalds  *
431da177e4SLinus Torvalds  * Currently there are a few corner cases in swapping where the policy
441da177e4SLinus Torvalds  * is not applied, but the majority should be handled. When process policy
451da177e4SLinus Torvalds  * is used it is not remembered over swap outs/swap ins.
461da177e4SLinus Torvalds  *
471da177e4SLinus Torvalds  * Only the highest zone in the zone hierarchy gets policied. Allocations
481da177e4SLinus Torvalds  * requesting a lower zone just use default policy. This implies that
491da177e4SLinus Torvalds  * on systems with highmem kernel lowmem allocation don't get policied.
501da177e4SLinus Torvalds  * Same with GFP_DMA allocations.
511da177e4SLinus Torvalds  *
521da177e4SLinus Torvalds  * For shmfs/tmpfs/hugetlbfs shared memory the policy is shared between
531da177e4SLinus Torvalds  * all users and remembered even when nobody has memory mapped.
541da177e4SLinus Torvalds  */
551da177e4SLinus Torvalds 
561da177e4SLinus Torvalds /* Notebook:
571da177e4SLinus Torvalds    fix mmap readahead to honour policy and enable policy for any page cache
581da177e4SLinus Torvalds    object
591da177e4SLinus Torvalds    statistics for bigpages
601da177e4SLinus Torvalds    global policy for page cache? currently it uses process policy. Requires
611da177e4SLinus Torvalds    first item above.
621da177e4SLinus Torvalds    handle mremap for shared memory (currently ignored for the policy)
631da177e4SLinus Torvalds    grows down?
641da177e4SLinus Torvalds    make bind policy root only? It can trigger oom much faster and the
651da177e4SLinus Torvalds    kernel is not always grateful with that.
661da177e4SLinus Torvalds    could replace all the switch()es with a mempolicy_ops structure.
671da177e4SLinus Torvalds */
681da177e4SLinus Torvalds 
691da177e4SLinus Torvalds #include <linux/mempolicy.h>
701da177e4SLinus Torvalds #include <linux/mm.h>
711da177e4SLinus Torvalds #include <linux/highmem.h>
721da177e4SLinus Torvalds #include <linux/hugetlb.h>
731da177e4SLinus Torvalds #include <linux/kernel.h>
741da177e4SLinus Torvalds #include <linux/sched.h>
751da177e4SLinus Torvalds #include <linux/mm.h>
761da177e4SLinus Torvalds #include <linux/nodemask.h>
771da177e4SLinus Torvalds #include <linux/cpuset.h>
781da177e4SLinus Torvalds #include <linux/gfp.h>
791da177e4SLinus Torvalds #include <linux/slab.h>
801da177e4SLinus Torvalds #include <linux/string.h>
811da177e4SLinus Torvalds #include <linux/module.h>
821da177e4SLinus Torvalds #include <linux/interrupt.h>
831da177e4SLinus Torvalds #include <linux/init.h>
841da177e4SLinus Torvalds #include <linux/compat.h>
851da177e4SLinus Torvalds #include <linux/mempolicy.h>
86dc9aa5b9SChristoph Lameter #include <linux/swap.h>
871a75a6c8SChristoph Lameter #include <linux/seq_file.h>
881a75a6c8SChristoph Lameter #include <linux/proc_fs.h>
89dc9aa5b9SChristoph Lameter 
901da177e4SLinus Torvalds #include <asm/tlbflush.h>
911da177e4SLinus Torvalds #include <asm/uaccess.h>
921da177e4SLinus Torvalds 
9338e35860SChristoph Lameter /* Internal flags */
94dc9aa5b9SChristoph Lameter #define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0)	/* Skip checks for continuous vmas */
9538e35860SChristoph Lameter #define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1)		/* Invert check for nodemask */
961a75a6c8SChristoph Lameter #define MPOL_MF_STATS (MPOL_MF_INTERNAL << 2)		/* Gather statistics */
97dc9aa5b9SChristoph Lameter 
987e2ab150SChristoph Lameter /* The number of pages to migrate per call to migrate_pages() */
997e2ab150SChristoph Lameter #define MIGRATE_CHUNK_SIZE 256
1007e2ab150SChristoph Lameter 
1011da177e4SLinus Torvalds static kmem_cache_t *policy_cache;
1021da177e4SLinus Torvalds static kmem_cache_t *sn_cache;
1031da177e4SLinus Torvalds 
1041da177e4SLinus Torvalds #define PDprintk(fmt...)
1051da177e4SLinus Torvalds 
1061da177e4SLinus Torvalds /* Highest zone. An specific allocation for a zone below that is not
1071da177e4SLinus Torvalds    policied. */
1084be38e35SChristoph Lameter int policy_zone = ZONE_DMA;
1091da177e4SLinus Torvalds 
110d42c6997SAndi Kleen struct mempolicy default_policy = {
1111da177e4SLinus Torvalds 	.refcnt = ATOMIC_INIT(1), /* never free it */
1121da177e4SLinus Torvalds 	.policy = MPOL_DEFAULT,
1131da177e4SLinus Torvalds };
1141da177e4SLinus Torvalds 
1151da177e4SLinus Torvalds /* Do sanity checking on a policy */
116dfcd3c0dSAndi Kleen static int mpol_check_policy(int mode, nodemask_t *nodes)
1171da177e4SLinus Torvalds {
118dfcd3c0dSAndi Kleen 	int empty = nodes_empty(*nodes);
1191da177e4SLinus Torvalds 
1201da177e4SLinus Torvalds 	switch (mode) {
1211da177e4SLinus Torvalds 	case MPOL_DEFAULT:
1221da177e4SLinus Torvalds 		if (!empty)
1231da177e4SLinus Torvalds 			return -EINVAL;
1241da177e4SLinus Torvalds 		break;
1251da177e4SLinus Torvalds 	case MPOL_BIND:
1261da177e4SLinus Torvalds 	case MPOL_INTERLEAVE:
1271da177e4SLinus Torvalds 		/* Preferred will only use the first bit, but allow
1281da177e4SLinus Torvalds 		   more for now. */
1291da177e4SLinus Torvalds 		if (empty)
1301da177e4SLinus Torvalds 			return -EINVAL;
1311da177e4SLinus Torvalds 		break;
1321da177e4SLinus Torvalds 	}
133dfcd3c0dSAndi Kleen 	return nodes_subset(*nodes, node_online_map) ? 0 : -EINVAL;
1341da177e4SLinus Torvalds }
135dd942ae3SAndi Kleen 
1361da177e4SLinus Torvalds /* Generate a custom zonelist for the BIND policy. */
137dfcd3c0dSAndi Kleen static struct zonelist *bind_zonelist(nodemask_t *nodes)
1381da177e4SLinus Torvalds {
1391da177e4SLinus Torvalds 	struct zonelist *zl;
140dd942ae3SAndi Kleen 	int num, max, nd, k;
1411da177e4SLinus Torvalds 
142dfcd3c0dSAndi Kleen 	max = 1 + MAX_NR_ZONES * nodes_weight(*nodes);
143dd942ae3SAndi Kleen 	zl = kmalloc(sizeof(struct zone *) * max, GFP_KERNEL);
1441da177e4SLinus Torvalds 	if (!zl)
1451da177e4SLinus Torvalds 		return NULL;
1461da177e4SLinus Torvalds 	num = 0;
147dd942ae3SAndi Kleen 	/* First put in the highest zones from all nodes, then all the next
148dd942ae3SAndi Kleen 	   lower zones etc. Avoid empty zones because the memory allocator
149dd942ae3SAndi Kleen 	   doesn't like them. If you implement node hot removal you
150dd942ae3SAndi Kleen 	   have to fix that. */
151dd942ae3SAndi Kleen 	for (k = policy_zone; k >= 0; k--) {
152dd942ae3SAndi Kleen 		for_each_node_mask(nd, *nodes) {
153dd942ae3SAndi Kleen 			struct zone *z = &NODE_DATA(nd)->node_zones[k];
154dd942ae3SAndi Kleen 			if (z->present_pages > 0)
155dd942ae3SAndi Kleen 				zl->zones[num++] = z;
156dd942ae3SAndi Kleen 		}
157dd942ae3SAndi Kleen 	}
1581da177e4SLinus Torvalds 	zl->zones[num] = NULL;
1591da177e4SLinus Torvalds 	return zl;
1601da177e4SLinus Torvalds }
1611da177e4SLinus Torvalds 
1621da177e4SLinus Torvalds /* Create a new policy */
163dfcd3c0dSAndi Kleen static struct mempolicy *mpol_new(int mode, nodemask_t *nodes)
1641da177e4SLinus Torvalds {
1651da177e4SLinus Torvalds 	struct mempolicy *policy;
1661da177e4SLinus Torvalds 
167dfcd3c0dSAndi Kleen 	PDprintk("setting mode %d nodes[0] %lx\n", mode, nodes_addr(*nodes)[0]);
1681da177e4SLinus Torvalds 	if (mode == MPOL_DEFAULT)
1691da177e4SLinus Torvalds 		return NULL;
1701da177e4SLinus Torvalds 	policy = kmem_cache_alloc(policy_cache, GFP_KERNEL);
1711da177e4SLinus Torvalds 	if (!policy)
1721da177e4SLinus Torvalds 		return ERR_PTR(-ENOMEM);
1731da177e4SLinus Torvalds 	atomic_set(&policy->refcnt, 1);
1741da177e4SLinus Torvalds 	switch (mode) {
1751da177e4SLinus Torvalds 	case MPOL_INTERLEAVE:
176dfcd3c0dSAndi Kleen 		policy->v.nodes = *nodes;
1778f493d79SAndi Kleen 		if (nodes_weight(*nodes) == 0) {
1788f493d79SAndi Kleen 			kmem_cache_free(policy_cache, policy);
1798f493d79SAndi Kleen 			return ERR_PTR(-EINVAL);
1808f493d79SAndi Kleen 		}
1811da177e4SLinus Torvalds 		break;
1821da177e4SLinus Torvalds 	case MPOL_PREFERRED:
183dfcd3c0dSAndi Kleen 		policy->v.preferred_node = first_node(*nodes);
1841da177e4SLinus Torvalds 		if (policy->v.preferred_node >= MAX_NUMNODES)
1851da177e4SLinus Torvalds 			policy->v.preferred_node = -1;
1861da177e4SLinus Torvalds 		break;
1871da177e4SLinus Torvalds 	case MPOL_BIND:
1881da177e4SLinus Torvalds 		policy->v.zonelist = bind_zonelist(nodes);
1891da177e4SLinus Torvalds 		if (policy->v.zonelist == NULL) {
1901da177e4SLinus Torvalds 			kmem_cache_free(policy_cache, policy);
1911da177e4SLinus Torvalds 			return ERR_PTR(-ENOMEM);
1921da177e4SLinus Torvalds 		}
1931da177e4SLinus Torvalds 		break;
1941da177e4SLinus Torvalds 	}
1951da177e4SLinus Torvalds 	policy->policy = mode;
19674cb2155SPaul Jackson 	policy->cpuset_mems_allowed = cpuset_mems_allowed(current);
1971da177e4SLinus Torvalds 	return policy;
1981da177e4SLinus Torvalds }
1991da177e4SLinus Torvalds 
2001a75a6c8SChristoph Lameter static void gather_stats(struct page *, void *);
201fc301289SChristoph Lameter static void migrate_page_add(struct page *page, struct list_head *pagelist,
202fc301289SChristoph Lameter 				unsigned long flags);
2031a75a6c8SChristoph Lameter 
20438e35860SChristoph Lameter /* Scan through pages checking if pages follow certain conditions. */
205b5810039SNick Piggin static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
206dc9aa5b9SChristoph Lameter 		unsigned long addr, unsigned long end,
207dc9aa5b9SChristoph Lameter 		const nodemask_t *nodes, unsigned long flags,
20838e35860SChristoph Lameter 		void *private)
2091da177e4SLinus Torvalds {
21091612e0dSHugh Dickins 	pte_t *orig_pte;
21191612e0dSHugh Dickins 	pte_t *pte;
212705e87c0SHugh Dickins 	spinlock_t *ptl;
213941150a3SHugh Dickins 
214705e87c0SHugh Dickins 	orig_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
21591612e0dSHugh Dickins 	do {
2166aab341eSLinus Torvalds 		struct page *page;
21791612e0dSHugh Dickins 		unsigned int nid;
21891612e0dSHugh Dickins 
21991612e0dSHugh Dickins 		if (!pte_present(*pte))
22091612e0dSHugh Dickins 			continue;
2216aab341eSLinus Torvalds 		page = vm_normal_page(vma, addr, *pte);
2226aab341eSLinus Torvalds 		if (!page)
22391612e0dSHugh Dickins 			continue;
224053837fcSNick Piggin 		/*
225053837fcSNick Piggin 		 * The check for PageReserved here is important to avoid
226053837fcSNick Piggin 		 * handling zero pages and other pages that may have been
227053837fcSNick Piggin 		 * marked special by the system.
228053837fcSNick Piggin 		 *
229053837fcSNick Piggin 		 * If the PageReserved would not be checked here then f.e.
230053837fcSNick Piggin 		 * the location of the zero page could have an influence
231053837fcSNick Piggin 		 * on MPOL_MF_STRICT, zero pages would be counted for
232053837fcSNick Piggin 		 * the per node stats, and there would be useless attempts
233053837fcSNick Piggin 		 * to put zero pages on the migration list.
234053837fcSNick Piggin 		 */
235f4598c8bSChristoph Lameter 		if (PageReserved(page))
236f4598c8bSChristoph Lameter 			continue;
2376aab341eSLinus Torvalds 		nid = page_to_nid(page);
23838e35860SChristoph Lameter 		if (node_isset(nid, *nodes) == !!(flags & MPOL_MF_INVERT))
23938e35860SChristoph Lameter 			continue;
24038e35860SChristoph Lameter 
2411a75a6c8SChristoph Lameter 		if (flags & MPOL_MF_STATS)
2421a75a6c8SChristoph Lameter 			gather_stats(page, private);
243053837fcSNick Piggin 		else if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
244fc301289SChristoph Lameter 			migrate_page_add(page, private, flags);
245dc9aa5b9SChristoph Lameter 		else
2461da177e4SLinus Torvalds 			break;
24791612e0dSHugh Dickins 	} while (pte++, addr += PAGE_SIZE, addr != end);
248705e87c0SHugh Dickins 	pte_unmap_unlock(orig_pte, ptl);
24991612e0dSHugh Dickins 	return addr != end;
25091612e0dSHugh Dickins }
25191612e0dSHugh Dickins 
252b5810039SNick Piggin static inline int check_pmd_range(struct vm_area_struct *vma, pud_t *pud,
253dc9aa5b9SChristoph Lameter 		unsigned long addr, unsigned long end,
254dc9aa5b9SChristoph Lameter 		const nodemask_t *nodes, unsigned long flags,
25538e35860SChristoph Lameter 		void *private)
25691612e0dSHugh Dickins {
25791612e0dSHugh Dickins 	pmd_t *pmd;
25891612e0dSHugh Dickins 	unsigned long next;
25991612e0dSHugh Dickins 
26091612e0dSHugh Dickins 	pmd = pmd_offset(pud, addr);
26191612e0dSHugh Dickins 	do {
26291612e0dSHugh Dickins 		next = pmd_addr_end(addr, end);
26391612e0dSHugh Dickins 		if (pmd_none_or_clear_bad(pmd))
26491612e0dSHugh Dickins 			continue;
265dc9aa5b9SChristoph Lameter 		if (check_pte_range(vma, pmd, addr, next, nodes,
26638e35860SChristoph Lameter 				    flags, private))
26791612e0dSHugh Dickins 			return -EIO;
26891612e0dSHugh Dickins 	} while (pmd++, addr = next, addr != end);
26991612e0dSHugh Dickins 	return 0;
27091612e0dSHugh Dickins }
27191612e0dSHugh Dickins 
272b5810039SNick Piggin static inline int check_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
273dc9aa5b9SChristoph Lameter 		unsigned long addr, unsigned long end,
274dc9aa5b9SChristoph Lameter 		const nodemask_t *nodes, unsigned long flags,
27538e35860SChristoph Lameter 		void *private)
27691612e0dSHugh Dickins {
27791612e0dSHugh Dickins 	pud_t *pud;
27891612e0dSHugh Dickins 	unsigned long next;
27991612e0dSHugh Dickins 
28091612e0dSHugh Dickins 	pud = pud_offset(pgd, addr);
28191612e0dSHugh Dickins 	do {
28291612e0dSHugh Dickins 		next = pud_addr_end(addr, end);
28391612e0dSHugh Dickins 		if (pud_none_or_clear_bad(pud))
28491612e0dSHugh Dickins 			continue;
285dc9aa5b9SChristoph Lameter 		if (check_pmd_range(vma, pud, addr, next, nodes,
28638e35860SChristoph Lameter 				    flags, private))
28791612e0dSHugh Dickins 			return -EIO;
28891612e0dSHugh Dickins 	} while (pud++, addr = next, addr != end);
28991612e0dSHugh Dickins 	return 0;
29091612e0dSHugh Dickins }
29191612e0dSHugh Dickins 
292b5810039SNick Piggin static inline int check_pgd_range(struct vm_area_struct *vma,
293dc9aa5b9SChristoph Lameter 		unsigned long addr, unsigned long end,
294dc9aa5b9SChristoph Lameter 		const nodemask_t *nodes, unsigned long flags,
29538e35860SChristoph Lameter 		void *private)
29691612e0dSHugh Dickins {
29791612e0dSHugh Dickins 	pgd_t *pgd;
29891612e0dSHugh Dickins 	unsigned long next;
29991612e0dSHugh Dickins 
300b5810039SNick Piggin 	pgd = pgd_offset(vma->vm_mm, addr);
30191612e0dSHugh Dickins 	do {
30291612e0dSHugh Dickins 		next = pgd_addr_end(addr, end);
30391612e0dSHugh Dickins 		if (pgd_none_or_clear_bad(pgd))
30491612e0dSHugh Dickins 			continue;
305dc9aa5b9SChristoph Lameter 		if (check_pud_range(vma, pgd, addr, next, nodes,
30638e35860SChristoph Lameter 				    flags, private))
30791612e0dSHugh Dickins 			return -EIO;
30891612e0dSHugh Dickins 	} while (pgd++, addr = next, addr != end);
30991612e0dSHugh Dickins 	return 0;
3101da177e4SLinus Torvalds }
3111da177e4SLinus Torvalds 
312dc9aa5b9SChristoph Lameter /* Check if a vma is migratable */
313dc9aa5b9SChristoph Lameter static inline int vma_migratable(struct vm_area_struct *vma)
314dc9aa5b9SChristoph Lameter {
315dc9aa5b9SChristoph Lameter 	if (vma->vm_flags & (
316f4598c8bSChristoph Lameter 		VM_LOCKED|VM_IO|VM_HUGETLB|VM_PFNMAP|VM_RESERVED))
317dc9aa5b9SChristoph Lameter 		return 0;
318dc9aa5b9SChristoph Lameter 	return 1;
319dc9aa5b9SChristoph Lameter }
320dc9aa5b9SChristoph Lameter 
321dc9aa5b9SChristoph Lameter /*
322dc9aa5b9SChristoph Lameter  * Check if all pages in a range are on a set of nodes.
323dc9aa5b9SChristoph Lameter  * If pagelist != NULL then isolate pages from the LRU and
324dc9aa5b9SChristoph Lameter  * put them on the pagelist.
325dc9aa5b9SChristoph Lameter  */
3261da177e4SLinus Torvalds static struct vm_area_struct *
3271da177e4SLinus Torvalds check_range(struct mm_struct *mm, unsigned long start, unsigned long end,
32838e35860SChristoph Lameter 		const nodemask_t *nodes, unsigned long flags, void *private)
3291da177e4SLinus Torvalds {
3301da177e4SLinus Torvalds 	int err;
3311da177e4SLinus Torvalds 	struct vm_area_struct *first, *vma, *prev;
3321da177e4SLinus Torvalds 
333053837fcSNick Piggin 	/* Clear the LRU lists so pages can be isolated */
334053837fcSNick Piggin 	if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
335053837fcSNick Piggin 		lru_add_drain_all();
336053837fcSNick Piggin 
3371da177e4SLinus Torvalds 	first = find_vma(mm, start);
3381da177e4SLinus Torvalds 	if (!first)
3391da177e4SLinus Torvalds 		return ERR_PTR(-EFAULT);
3401da177e4SLinus Torvalds 	prev = NULL;
3411da177e4SLinus Torvalds 	for (vma = first; vma && vma->vm_start < end; vma = vma->vm_next) {
342dc9aa5b9SChristoph Lameter 		if (!(flags & MPOL_MF_DISCONTIG_OK)) {
3431da177e4SLinus Torvalds 			if (!vma->vm_next && vma->vm_end < end)
3441da177e4SLinus Torvalds 				return ERR_PTR(-EFAULT);
3451da177e4SLinus Torvalds 			if (prev && prev->vm_end < vma->vm_start)
3461da177e4SLinus Torvalds 				return ERR_PTR(-EFAULT);
347dc9aa5b9SChristoph Lameter 		}
348dc9aa5b9SChristoph Lameter 		if (!is_vm_hugetlb_page(vma) &&
349dc9aa5b9SChristoph Lameter 		    ((flags & MPOL_MF_STRICT) ||
350dc9aa5b9SChristoph Lameter 		     ((flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) &&
351dc9aa5b9SChristoph Lameter 				vma_migratable(vma)))) {
3525b952b3cSAndi Kleen 			unsigned long endvma = vma->vm_end;
353dc9aa5b9SChristoph Lameter 
3545b952b3cSAndi Kleen 			if (endvma > end)
3555b952b3cSAndi Kleen 				endvma = end;
3565b952b3cSAndi Kleen 			if (vma->vm_start > start)
3575b952b3cSAndi Kleen 				start = vma->vm_start;
358dc9aa5b9SChristoph Lameter 			err = check_pgd_range(vma, start, endvma, nodes,
35938e35860SChristoph Lameter 						flags, private);
3601da177e4SLinus Torvalds 			if (err) {
3611da177e4SLinus Torvalds 				first = ERR_PTR(err);
3621da177e4SLinus Torvalds 				break;
3631da177e4SLinus Torvalds 			}
3641da177e4SLinus Torvalds 		}
3651da177e4SLinus Torvalds 		prev = vma;
3661da177e4SLinus Torvalds 	}
3671da177e4SLinus Torvalds 	return first;
3681da177e4SLinus Torvalds }
3691da177e4SLinus Torvalds 
3701da177e4SLinus Torvalds /* Apply policy to a single VMA */
3711da177e4SLinus Torvalds static int policy_vma(struct vm_area_struct *vma, struct mempolicy *new)
3721da177e4SLinus Torvalds {
3731da177e4SLinus Torvalds 	int err = 0;
3741da177e4SLinus Torvalds 	struct mempolicy *old = vma->vm_policy;
3751da177e4SLinus Torvalds 
3761da177e4SLinus Torvalds 	PDprintk("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n",
3771da177e4SLinus Torvalds 		 vma->vm_start, vma->vm_end, vma->vm_pgoff,
3781da177e4SLinus Torvalds 		 vma->vm_ops, vma->vm_file,
3791da177e4SLinus Torvalds 		 vma->vm_ops ? vma->vm_ops->set_policy : NULL);
3801da177e4SLinus Torvalds 
3811da177e4SLinus Torvalds 	if (vma->vm_ops && vma->vm_ops->set_policy)
3821da177e4SLinus Torvalds 		err = vma->vm_ops->set_policy(vma, new);
3831da177e4SLinus Torvalds 	if (!err) {
3841da177e4SLinus Torvalds 		mpol_get(new);
3851da177e4SLinus Torvalds 		vma->vm_policy = new;
3861da177e4SLinus Torvalds 		mpol_free(old);
3871da177e4SLinus Torvalds 	}
3881da177e4SLinus Torvalds 	return err;
3891da177e4SLinus Torvalds }
3901da177e4SLinus Torvalds 
3911da177e4SLinus Torvalds /* Step 2: apply policy to a range and do splits. */
3921da177e4SLinus Torvalds static int mbind_range(struct vm_area_struct *vma, unsigned long start,
3931da177e4SLinus Torvalds 		       unsigned long end, struct mempolicy *new)
3941da177e4SLinus Torvalds {
3951da177e4SLinus Torvalds 	struct vm_area_struct *next;
3961da177e4SLinus Torvalds 	int err;
3971da177e4SLinus Torvalds 
3981da177e4SLinus Torvalds 	err = 0;
3991da177e4SLinus Torvalds 	for (; vma && vma->vm_start < end; vma = next) {
4001da177e4SLinus Torvalds 		next = vma->vm_next;
4011da177e4SLinus Torvalds 		if (vma->vm_start < start)
4021da177e4SLinus Torvalds 			err = split_vma(vma->vm_mm, vma, start, 1);
4031da177e4SLinus Torvalds 		if (!err && vma->vm_end > end)
4041da177e4SLinus Torvalds 			err = split_vma(vma->vm_mm, vma, end, 0);
4051da177e4SLinus Torvalds 		if (!err)
4061da177e4SLinus Torvalds 			err = policy_vma(vma, new);
4071da177e4SLinus Torvalds 		if (err)
4081da177e4SLinus Torvalds 			break;
4091da177e4SLinus Torvalds 	}
4101da177e4SLinus Torvalds 	return err;
4111da177e4SLinus Torvalds }
4121da177e4SLinus Torvalds 
4138bccd85fSChristoph Lameter static int contextualize_policy(int mode, nodemask_t *nodes)
4148bccd85fSChristoph Lameter {
4158bccd85fSChristoph Lameter 	if (!nodes)
4168bccd85fSChristoph Lameter 		return 0;
4178bccd85fSChristoph Lameter 
418cf2a473cSPaul Jackson 	cpuset_update_task_memory_state();
4195966514dSPaul Jackson 	if (!cpuset_nodes_subset_current_mems_allowed(*nodes))
4205966514dSPaul Jackson 		return -EINVAL;
4218bccd85fSChristoph Lameter 	return mpol_check_policy(mode, nodes);
4228bccd85fSChristoph Lameter }
4238bccd85fSChristoph Lameter 
4241da177e4SLinus Torvalds /* Set the process memory policy */
4258bccd85fSChristoph Lameter long do_set_mempolicy(int mode, nodemask_t *nodes)
4261da177e4SLinus Torvalds {
4271da177e4SLinus Torvalds 	struct mempolicy *new;
4281da177e4SLinus Torvalds 
4298bccd85fSChristoph Lameter 	if (contextualize_policy(mode, nodes))
4301da177e4SLinus Torvalds 		return -EINVAL;
4318bccd85fSChristoph Lameter 	new = mpol_new(mode, nodes);
4321da177e4SLinus Torvalds 	if (IS_ERR(new))
4331da177e4SLinus Torvalds 		return PTR_ERR(new);
4341da177e4SLinus Torvalds 	mpol_free(current->mempolicy);
4351da177e4SLinus Torvalds 	current->mempolicy = new;
4361da177e4SLinus Torvalds 	if (new && new->policy == MPOL_INTERLEAVE)
437dfcd3c0dSAndi Kleen 		current->il_next = first_node(new->v.nodes);
4381da177e4SLinus Torvalds 	return 0;
4391da177e4SLinus Torvalds }
4401da177e4SLinus Torvalds 
4411da177e4SLinus Torvalds /* Fill a zone bitmap for a policy */
442dfcd3c0dSAndi Kleen static void get_zonemask(struct mempolicy *p, nodemask_t *nodes)
4431da177e4SLinus Torvalds {
4441da177e4SLinus Torvalds 	int i;
4451da177e4SLinus Torvalds 
446dfcd3c0dSAndi Kleen 	nodes_clear(*nodes);
4471da177e4SLinus Torvalds 	switch (p->policy) {
4481da177e4SLinus Torvalds 	case MPOL_BIND:
4491da177e4SLinus Torvalds 		for (i = 0; p->v.zonelist->zones[i]; i++)
4508bccd85fSChristoph Lameter 			node_set(p->v.zonelist->zones[i]->zone_pgdat->node_id,
4518bccd85fSChristoph Lameter 				*nodes);
4521da177e4SLinus Torvalds 		break;
4531da177e4SLinus Torvalds 	case MPOL_DEFAULT:
4541da177e4SLinus Torvalds 		break;
4551da177e4SLinus Torvalds 	case MPOL_INTERLEAVE:
456dfcd3c0dSAndi Kleen 		*nodes = p->v.nodes;
4571da177e4SLinus Torvalds 		break;
4581da177e4SLinus Torvalds 	case MPOL_PREFERRED:
4591da177e4SLinus Torvalds 		/* or use current node instead of online map? */
4601da177e4SLinus Torvalds 		if (p->v.preferred_node < 0)
461dfcd3c0dSAndi Kleen 			*nodes = node_online_map;
4621da177e4SLinus Torvalds 		else
463dfcd3c0dSAndi Kleen 			node_set(p->v.preferred_node, *nodes);
4641da177e4SLinus Torvalds 		break;
4651da177e4SLinus Torvalds 	default:
4661da177e4SLinus Torvalds 		BUG();
4671da177e4SLinus Torvalds 	}
4681da177e4SLinus Torvalds }
4691da177e4SLinus Torvalds 
4701da177e4SLinus Torvalds static int lookup_node(struct mm_struct *mm, unsigned long addr)
4711da177e4SLinus Torvalds {
4721da177e4SLinus Torvalds 	struct page *p;
4731da177e4SLinus Torvalds 	int err;
4741da177e4SLinus Torvalds 
4751da177e4SLinus Torvalds 	err = get_user_pages(current, mm, addr & PAGE_MASK, 1, 0, 0, &p, NULL);
4761da177e4SLinus Torvalds 	if (err >= 0) {
4771da177e4SLinus Torvalds 		err = page_to_nid(p);
4781da177e4SLinus Torvalds 		put_page(p);
4791da177e4SLinus Torvalds 	}
4801da177e4SLinus Torvalds 	return err;
4811da177e4SLinus Torvalds }
4821da177e4SLinus Torvalds 
4831da177e4SLinus Torvalds /* Retrieve NUMA policy */
4848bccd85fSChristoph Lameter long do_get_mempolicy(int *policy, nodemask_t *nmask,
4851da177e4SLinus Torvalds 			unsigned long addr, unsigned long flags)
4861da177e4SLinus Torvalds {
4878bccd85fSChristoph Lameter 	int err;
4881da177e4SLinus Torvalds 	struct mm_struct *mm = current->mm;
4891da177e4SLinus Torvalds 	struct vm_area_struct *vma = NULL;
4901da177e4SLinus Torvalds 	struct mempolicy *pol = current->mempolicy;
4911da177e4SLinus Torvalds 
492cf2a473cSPaul Jackson 	cpuset_update_task_memory_state();
4931da177e4SLinus Torvalds 	if (flags & ~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR))
4941da177e4SLinus Torvalds 		return -EINVAL;
4951da177e4SLinus Torvalds 	if (flags & MPOL_F_ADDR) {
4961da177e4SLinus Torvalds 		down_read(&mm->mmap_sem);
4971da177e4SLinus Torvalds 		vma = find_vma_intersection(mm, addr, addr+1);
4981da177e4SLinus Torvalds 		if (!vma) {
4991da177e4SLinus Torvalds 			up_read(&mm->mmap_sem);
5001da177e4SLinus Torvalds 			return -EFAULT;
5011da177e4SLinus Torvalds 		}
5021da177e4SLinus Torvalds 		if (vma->vm_ops && vma->vm_ops->get_policy)
5031da177e4SLinus Torvalds 			pol = vma->vm_ops->get_policy(vma, addr);
5041da177e4SLinus Torvalds 		else
5051da177e4SLinus Torvalds 			pol = vma->vm_policy;
5061da177e4SLinus Torvalds 	} else if (addr)
5071da177e4SLinus Torvalds 		return -EINVAL;
5081da177e4SLinus Torvalds 
5091da177e4SLinus Torvalds 	if (!pol)
5101da177e4SLinus Torvalds 		pol = &default_policy;
5111da177e4SLinus Torvalds 
5121da177e4SLinus Torvalds 	if (flags & MPOL_F_NODE) {
5131da177e4SLinus Torvalds 		if (flags & MPOL_F_ADDR) {
5141da177e4SLinus Torvalds 			err = lookup_node(mm, addr);
5151da177e4SLinus Torvalds 			if (err < 0)
5161da177e4SLinus Torvalds 				goto out;
5178bccd85fSChristoph Lameter 			*policy = err;
5181da177e4SLinus Torvalds 		} else if (pol == current->mempolicy &&
5191da177e4SLinus Torvalds 				pol->policy == MPOL_INTERLEAVE) {
5208bccd85fSChristoph Lameter 			*policy = current->il_next;
5211da177e4SLinus Torvalds 		} else {
5221da177e4SLinus Torvalds 			err = -EINVAL;
5231da177e4SLinus Torvalds 			goto out;
5241da177e4SLinus Torvalds 		}
5251da177e4SLinus Torvalds 	} else
5268bccd85fSChristoph Lameter 		*policy = pol->policy;
5271da177e4SLinus Torvalds 
5281da177e4SLinus Torvalds 	if (vma) {
5291da177e4SLinus Torvalds 		up_read(&current->mm->mmap_sem);
5301da177e4SLinus Torvalds 		vma = NULL;
5311da177e4SLinus Torvalds 	}
5321da177e4SLinus Torvalds 
5331da177e4SLinus Torvalds 	err = 0;
5348bccd85fSChristoph Lameter 	if (nmask)
5358bccd85fSChristoph Lameter 		get_zonemask(pol, nmask);
5361da177e4SLinus Torvalds 
5371da177e4SLinus Torvalds  out:
5381da177e4SLinus Torvalds 	if (vma)
5391da177e4SLinus Torvalds 		up_read(&current->mm->mmap_sem);
5401da177e4SLinus Torvalds 	return err;
5411da177e4SLinus Torvalds }
5421da177e4SLinus Torvalds 
5438bccd85fSChristoph Lameter /*
5446ce3c4c0SChristoph Lameter  * page migration
5456ce3c4c0SChristoph Lameter  */
5466ce3c4c0SChristoph Lameter 
547fc301289SChristoph Lameter static void migrate_page_add(struct page *page, struct list_head *pagelist,
548fc301289SChristoph Lameter 				unsigned long flags)
5496ce3c4c0SChristoph Lameter {
5506ce3c4c0SChristoph Lameter 	/*
551fc301289SChristoph Lameter 	 * Avoid migrating a page that is shared with others.
5526ce3c4c0SChristoph Lameter 	 */
553fc301289SChristoph Lameter 	if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1) {
554053837fcSNick Piggin 		if (isolate_lru_page(page))
5556ce3c4c0SChristoph Lameter 			list_add(&page->lru, pagelist);
5566ce3c4c0SChristoph Lameter 	}
5576ce3c4c0SChristoph Lameter }
5586ce3c4c0SChristoph Lameter 
5597e2ab150SChristoph Lameter /*
5607e2ab150SChristoph Lameter  * Migrate the list 'pagelist' of pages to a certain destination.
5617e2ab150SChristoph Lameter  *
5627e2ab150SChristoph Lameter  * Specify destination with either non-NULL vma or dest_node >= 0
5637e2ab150SChristoph Lameter  * Return the number of pages not migrated or error code
5647e2ab150SChristoph Lameter  */
5657e2ab150SChristoph Lameter static int migrate_pages_to(struct list_head *pagelist,
5667e2ab150SChristoph Lameter 			struct vm_area_struct *vma, int dest)
5676ce3c4c0SChristoph Lameter {
5687e2ab150SChristoph Lameter 	LIST_HEAD(newlist);
5696ce3c4c0SChristoph Lameter 	LIST_HEAD(moved);
5706ce3c4c0SChristoph Lameter 	LIST_HEAD(failed);
5717e2ab150SChristoph Lameter 	int err = 0;
5727e2ab150SChristoph Lameter 	int nr_pages;
5737e2ab150SChristoph Lameter 	struct page *page;
5747e2ab150SChristoph Lameter 	struct list_head *p;
5756ce3c4c0SChristoph Lameter 
5767e2ab150SChristoph Lameter redo:
5777e2ab150SChristoph Lameter 	nr_pages = 0;
5787e2ab150SChristoph Lameter 	list_for_each(p, pagelist) {
5797e2ab150SChristoph Lameter 		if (vma)
5807e2ab150SChristoph Lameter 			page = alloc_page_vma(GFP_HIGHUSER, vma, vma->vm_start);
5817e2ab150SChristoph Lameter 		else
5827e2ab150SChristoph Lameter 			page = alloc_pages_node(dest, GFP_HIGHUSER, 0);
5836ce3c4c0SChristoph Lameter 
5847e2ab150SChristoph Lameter 		if (!page) {
5857e2ab150SChristoph Lameter 			err = -ENOMEM;
5867e2ab150SChristoph Lameter 			goto out;
5877e2ab150SChristoph Lameter 		}
5887e2ab150SChristoph Lameter 		list_add(&page->lru, &newlist);
5897e2ab150SChristoph Lameter 		nr_pages++;
5907e2ab150SChristoph Lameter 		if (nr_pages > MIGRATE_CHUNK_SIZE);
5917e2ab150SChristoph Lameter 			break;
5927e2ab150SChristoph Lameter 	}
5937e2ab150SChristoph Lameter 	err = migrate_pages(pagelist, &newlist, &moved, &failed);
5947e2ab150SChristoph Lameter 
5957e2ab150SChristoph Lameter 	putback_lru_pages(&moved);	/* Call release pages instead ?? */
5967e2ab150SChristoph Lameter 
5977e2ab150SChristoph Lameter 	if (err >= 0 && list_empty(&newlist) && !list_empty(pagelist))
5987e2ab150SChristoph Lameter 		goto redo;
5997e2ab150SChristoph Lameter out:
6007e2ab150SChristoph Lameter 	/* Return leftover allocated pages */
6017e2ab150SChristoph Lameter 	while (!list_empty(&newlist)) {
6027e2ab150SChristoph Lameter 		page = list_entry(newlist.next, struct page, lru);
6037e2ab150SChristoph Lameter 		list_del(&page->lru);
6047e2ab150SChristoph Lameter 		__free_page(page);
6057e2ab150SChristoph Lameter 	}
6067e2ab150SChristoph Lameter 	list_splice(&failed, pagelist);
6077e2ab150SChristoph Lameter 	if (err < 0)
6087e2ab150SChristoph Lameter 		return err;
6097e2ab150SChristoph Lameter 
6107e2ab150SChristoph Lameter 	/* Calculate number of leftover pages */
6117e2ab150SChristoph Lameter 	nr_pages = 0;
6127e2ab150SChristoph Lameter 	list_for_each(p, pagelist)
6137e2ab150SChristoph Lameter 		nr_pages++;
6147e2ab150SChristoph Lameter 	return nr_pages;
6156ce3c4c0SChristoph Lameter }
6166ce3c4c0SChristoph Lameter 
6176ce3c4c0SChristoph Lameter /*
6187e2ab150SChristoph Lameter  * Migrate pages from one node to a target node.
6197e2ab150SChristoph Lameter  * Returns error or the number of pages not migrated.
6207e2ab150SChristoph Lameter  */
6217e2ab150SChristoph Lameter int migrate_to_node(struct mm_struct *mm, int source, int dest, int flags)
6227e2ab150SChristoph Lameter {
6237e2ab150SChristoph Lameter 	nodemask_t nmask;
6247e2ab150SChristoph Lameter 	LIST_HEAD(pagelist);
6257e2ab150SChristoph Lameter 	int err = 0;
6267e2ab150SChristoph Lameter 
6277e2ab150SChristoph Lameter 	nodes_clear(nmask);
6287e2ab150SChristoph Lameter 	node_set(source, nmask);
6297e2ab150SChristoph Lameter 
6307e2ab150SChristoph Lameter 	check_range(mm, mm->mmap->vm_start, TASK_SIZE, &nmask,
6317e2ab150SChristoph Lameter 			flags | MPOL_MF_DISCONTIG_OK, &pagelist);
6327e2ab150SChristoph Lameter 
6337e2ab150SChristoph Lameter 	if (!list_empty(&pagelist)) {
6347e2ab150SChristoph Lameter 		err = migrate_pages_to(&pagelist, NULL, dest);
6357e2ab150SChristoph Lameter 		if (!list_empty(&pagelist))
6367e2ab150SChristoph Lameter 			putback_lru_pages(&pagelist);
6377e2ab150SChristoph Lameter 	}
6387e2ab150SChristoph Lameter 	return err;
6397e2ab150SChristoph Lameter }
6407e2ab150SChristoph Lameter 
6417e2ab150SChristoph Lameter /*
6427e2ab150SChristoph Lameter  * Move pages between the two nodesets so as to preserve the physical
6437e2ab150SChristoph Lameter  * layout as much as possible.
64439743889SChristoph Lameter  *
64539743889SChristoph Lameter  * Returns the number of page that could not be moved.
64639743889SChristoph Lameter  */
64739743889SChristoph Lameter int do_migrate_pages(struct mm_struct *mm,
64839743889SChristoph Lameter 	const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags)
64939743889SChristoph Lameter {
65039743889SChristoph Lameter 	LIST_HEAD(pagelist);
6517e2ab150SChristoph Lameter 	int busy = 0;
6527e2ab150SChristoph Lameter 	int err = 0;
6537e2ab150SChristoph Lameter 	nodemask_t tmp;
65439743889SChristoph Lameter 
65539743889SChristoph Lameter   	down_read(&mm->mmap_sem);
656d4984711SChristoph Lameter 
6577e2ab150SChristoph Lameter /*
6587e2ab150SChristoph Lameter  * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
6597e2ab150SChristoph Lameter  * bit in 'to' is not also set in 'tmp'.  Clear the found 'source'
6607e2ab150SChristoph Lameter  * bit in 'tmp', and return that <source, dest> pair for migration.
6617e2ab150SChristoph Lameter  * The pair of nodemasks 'to' and 'from' define the map.
6627e2ab150SChristoph Lameter  *
6637e2ab150SChristoph Lameter  * If no pair of bits is found that way, fallback to picking some
6647e2ab150SChristoph Lameter  * pair of 'source' and 'dest' bits that are not the same.  If the
6657e2ab150SChristoph Lameter  * 'source' and 'dest' bits are the same, this represents a node
6667e2ab150SChristoph Lameter  * that will be migrating to itself, so no pages need move.
6677e2ab150SChristoph Lameter  *
6687e2ab150SChristoph Lameter  * If no bits are left in 'tmp', or if all remaining bits left
6697e2ab150SChristoph Lameter  * in 'tmp' correspond to the same bit in 'to', return false
6707e2ab150SChristoph Lameter  * (nothing left to migrate).
6717e2ab150SChristoph Lameter  *
6727e2ab150SChristoph Lameter  * This lets us pick a pair of nodes to migrate between, such that
6737e2ab150SChristoph Lameter  * if possible the dest node is not already occupied by some other
6747e2ab150SChristoph Lameter  * source node, minimizing the risk of overloading the memory on a
6757e2ab150SChristoph Lameter  * node that would happen if we migrated incoming memory to a node
6767e2ab150SChristoph Lameter  * before migrating outgoing memory source that same node.
6777e2ab150SChristoph Lameter  *
6787e2ab150SChristoph Lameter  * A single scan of tmp is sufficient.  As we go, we remember the
6797e2ab150SChristoph Lameter  * most recent <s, d> pair that moved (s != d).  If we find a pair
6807e2ab150SChristoph Lameter  * that not only moved, but what's better, moved to an empty slot
6817e2ab150SChristoph Lameter  * (d is not set in tmp), then we break out then, with that pair.
6827e2ab150SChristoph Lameter  * Otherwise when we finish scannng from_tmp, we at least have the
6837e2ab150SChristoph Lameter  * most recent <s, d> pair that moved.  If we get all the way through
6847e2ab150SChristoph Lameter  * the scan of tmp without finding any node that moved, much less
6857e2ab150SChristoph Lameter  * moved to an empty node, then there is nothing left worth migrating.
6867e2ab150SChristoph Lameter  */
6877e2ab150SChristoph Lameter 
6887e2ab150SChristoph Lameter 	tmp = *from_nodes;
6897e2ab150SChristoph Lameter 	while (!nodes_empty(tmp)) {
6907e2ab150SChristoph Lameter 		int s,d;
6917e2ab150SChristoph Lameter 		int source = -1;
6927e2ab150SChristoph Lameter 		int dest = 0;
6937e2ab150SChristoph Lameter 
6947e2ab150SChristoph Lameter 		for_each_node_mask(s, tmp) {
6957e2ab150SChristoph Lameter 			d = node_remap(s, *from_nodes, *to_nodes);
6967e2ab150SChristoph Lameter 			if (s == d)
6977e2ab150SChristoph Lameter 				continue;
6987e2ab150SChristoph Lameter 
6997e2ab150SChristoph Lameter 			source = s;	/* Node moved. Memorize */
7007e2ab150SChristoph Lameter 			dest = d;
7017e2ab150SChristoph Lameter 
7027e2ab150SChristoph Lameter 			/* dest not in remaining from nodes? */
7037e2ab150SChristoph Lameter 			if (!node_isset(dest, tmp))
7047e2ab150SChristoph Lameter 				break;
7057e2ab150SChristoph Lameter 		}
7067e2ab150SChristoph Lameter 		if (source == -1)
7077e2ab150SChristoph Lameter 			break;
7087e2ab150SChristoph Lameter 
7097e2ab150SChristoph Lameter 		node_clear(source, tmp);
7107e2ab150SChristoph Lameter 		err = migrate_to_node(mm, source, dest, flags);
7117e2ab150SChristoph Lameter 		if (err > 0)
7127e2ab150SChristoph Lameter 			busy += err;
7137e2ab150SChristoph Lameter 		if (err < 0)
7147e2ab150SChristoph Lameter 			break;
71539743889SChristoph Lameter 	}
716d4984711SChristoph Lameter 
71739743889SChristoph Lameter 	up_read(&mm->mmap_sem);
7187e2ab150SChristoph Lameter 	if (err < 0)
7197e2ab150SChristoph Lameter 		return err;
7207e2ab150SChristoph Lameter 	return busy;
72139743889SChristoph Lameter }
72239743889SChristoph Lameter 
7236ce3c4c0SChristoph Lameter long do_mbind(unsigned long start, unsigned long len,
7246ce3c4c0SChristoph Lameter 		unsigned long mode, nodemask_t *nmask, unsigned long flags)
7256ce3c4c0SChristoph Lameter {
7266ce3c4c0SChristoph Lameter 	struct vm_area_struct *vma;
7276ce3c4c0SChristoph Lameter 	struct mm_struct *mm = current->mm;
7286ce3c4c0SChristoph Lameter 	struct mempolicy *new;
7296ce3c4c0SChristoph Lameter 	unsigned long end;
7306ce3c4c0SChristoph Lameter 	int err;
7316ce3c4c0SChristoph Lameter 	LIST_HEAD(pagelist);
7326ce3c4c0SChristoph Lameter 
7336ce3c4c0SChristoph Lameter 	if ((flags & ~(unsigned long)(MPOL_MF_STRICT |
7346ce3c4c0SChristoph Lameter 				      MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
7356ce3c4c0SChristoph Lameter 	    || mode > MPOL_MAX)
7366ce3c4c0SChristoph Lameter 		return -EINVAL;
7376ce3c4c0SChristoph Lameter 	if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_RESOURCE))
7386ce3c4c0SChristoph Lameter 		return -EPERM;
7396ce3c4c0SChristoph Lameter 
7406ce3c4c0SChristoph Lameter 	if (start & ~PAGE_MASK)
7416ce3c4c0SChristoph Lameter 		return -EINVAL;
7426ce3c4c0SChristoph Lameter 
7436ce3c4c0SChristoph Lameter 	if (mode == MPOL_DEFAULT)
7446ce3c4c0SChristoph Lameter 		flags &= ~MPOL_MF_STRICT;
7456ce3c4c0SChristoph Lameter 
7466ce3c4c0SChristoph Lameter 	len = (len + PAGE_SIZE - 1) & PAGE_MASK;
7476ce3c4c0SChristoph Lameter 	end = start + len;
7486ce3c4c0SChristoph Lameter 
7496ce3c4c0SChristoph Lameter 	if (end < start)
7506ce3c4c0SChristoph Lameter 		return -EINVAL;
7516ce3c4c0SChristoph Lameter 	if (end == start)
7526ce3c4c0SChristoph Lameter 		return 0;
7536ce3c4c0SChristoph Lameter 
7546ce3c4c0SChristoph Lameter 	if (mpol_check_policy(mode, nmask))
7556ce3c4c0SChristoph Lameter 		return -EINVAL;
7566ce3c4c0SChristoph Lameter 
7576ce3c4c0SChristoph Lameter 	new = mpol_new(mode, nmask);
7586ce3c4c0SChristoph Lameter 	if (IS_ERR(new))
7596ce3c4c0SChristoph Lameter 		return PTR_ERR(new);
7606ce3c4c0SChristoph Lameter 
7616ce3c4c0SChristoph Lameter 	/*
7626ce3c4c0SChristoph Lameter 	 * If we are using the default policy then operation
7636ce3c4c0SChristoph Lameter 	 * on discontinuous address spaces is okay after all
7646ce3c4c0SChristoph Lameter 	 */
7656ce3c4c0SChristoph Lameter 	if (!new)
7666ce3c4c0SChristoph Lameter 		flags |= MPOL_MF_DISCONTIG_OK;
7676ce3c4c0SChristoph Lameter 
7686ce3c4c0SChristoph Lameter 	PDprintk("mbind %lx-%lx mode:%ld nodes:%lx\n",start,start+len,
7696ce3c4c0SChristoph Lameter 			mode,nodes_addr(nodes)[0]);
7706ce3c4c0SChristoph Lameter 
7716ce3c4c0SChristoph Lameter 	down_write(&mm->mmap_sem);
7726ce3c4c0SChristoph Lameter 	vma = check_range(mm, start, end, nmask,
7736ce3c4c0SChristoph Lameter 			  flags | MPOL_MF_INVERT, &pagelist);
7746ce3c4c0SChristoph Lameter 
7756ce3c4c0SChristoph Lameter 	err = PTR_ERR(vma);
7766ce3c4c0SChristoph Lameter 	if (!IS_ERR(vma)) {
7776ce3c4c0SChristoph Lameter 		int nr_failed = 0;
7786ce3c4c0SChristoph Lameter 
7796ce3c4c0SChristoph Lameter 		err = mbind_range(vma, start, end, new);
7807e2ab150SChristoph Lameter 
7816ce3c4c0SChristoph Lameter 		if (!list_empty(&pagelist))
7827e2ab150SChristoph Lameter 			nr_failed = migrate_pages_to(&pagelist, vma, -1);
7836ce3c4c0SChristoph Lameter 
7846ce3c4c0SChristoph Lameter 		if (!err && nr_failed && (flags & MPOL_MF_STRICT))
7856ce3c4c0SChristoph Lameter 			err = -EIO;
7866ce3c4c0SChristoph Lameter 	}
7876ce3c4c0SChristoph Lameter 	if (!list_empty(&pagelist))
7886ce3c4c0SChristoph Lameter 		putback_lru_pages(&pagelist);
7896ce3c4c0SChristoph Lameter 
7906ce3c4c0SChristoph Lameter 	up_write(&mm->mmap_sem);
7916ce3c4c0SChristoph Lameter 	mpol_free(new);
7926ce3c4c0SChristoph Lameter 	return err;
7936ce3c4c0SChristoph Lameter }
7946ce3c4c0SChristoph Lameter 
79539743889SChristoph Lameter /*
7968bccd85fSChristoph Lameter  * User space interface with variable sized bitmaps for nodelists.
7978bccd85fSChristoph Lameter  */
7988bccd85fSChristoph Lameter 
7998bccd85fSChristoph Lameter /* Copy a node mask from user space. */
80039743889SChristoph Lameter static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask,
8018bccd85fSChristoph Lameter 		     unsigned long maxnode)
8028bccd85fSChristoph Lameter {
8038bccd85fSChristoph Lameter 	unsigned long k;
8048bccd85fSChristoph Lameter 	unsigned long nlongs;
8058bccd85fSChristoph Lameter 	unsigned long endmask;
8068bccd85fSChristoph Lameter 
8078bccd85fSChristoph Lameter 	--maxnode;
8088bccd85fSChristoph Lameter 	nodes_clear(*nodes);
8098bccd85fSChristoph Lameter 	if (maxnode == 0 || !nmask)
8108bccd85fSChristoph Lameter 		return 0;
811*636f13c1SChris Wright 	if (maxnode > PAGE_SIZE)
812*636f13c1SChris Wright 		return -EINVAL;
8138bccd85fSChristoph Lameter 
8148bccd85fSChristoph Lameter 	nlongs = BITS_TO_LONGS(maxnode);
8158bccd85fSChristoph Lameter 	if ((maxnode % BITS_PER_LONG) == 0)
8168bccd85fSChristoph Lameter 		endmask = ~0UL;
8178bccd85fSChristoph Lameter 	else
8188bccd85fSChristoph Lameter 		endmask = (1UL << (maxnode % BITS_PER_LONG)) - 1;
8198bccd85fSChristoph Lameter 
8208bccd85fSChristoph Lameter 	/* When the user specified more nodes than supported just check
8218bccd85fSChristoph Lameter 	   if the non supported part is all zero. */
8228bccd85fSChristoph Lameter 	if (nlongs > BITS_TO_LONGS(MAX_NUMNODES)) {
8238bccd85fSChristoph Lameter 		if (nlongs > PAGE_SIZE/sizeof(long))
8248bccd85fSChristoph Lameter 			return -EINVAL;
8258bccd85fSChristoph Lameter 		for (k = BITS_TO_LONGS(MAX_NUMNODES); k < nlongs; k++) {
8268bccd85fSChristoph Lameter 			unsigned long t;
8278bccd85fSChristoph Lameter 			if (get_user(t, nmask + k))
8288bccd85fSChristoph Lameter 				return -EFAULT;
8298bccd85fSChristoph Lameter 			if (k == nlongs - 1) {
8308bccd85fSChristoph Lameter 				if (t & endmask)
8318bccd85fSChristoph Lameter 					return -EINVAL;
8328bccd85fSChristoph Lameter 			} else if (t)
8338bccd85fSChristoph Lameter 				return -EINVAL;
8348bccd85fSChristoph Lameter 		}
8358bccd85fSChristoph Lameter 		nlongs = BITS_TO_LONGS(MAX_NUMNODES);
8368bccd85fSChristoph Lameter 		endmask = ~0UL;
8378bccd85fSChristoph Lameter 	}
8388bccd85fSChristoph Lameter 
8398bccd85fSChristoph Lameter 	if (copy_from_user(nodes_addr(*nodes), nmask, nlongs*sizeof(unsigned long)))
8408bccd85fSChristoph Lameter 		return -EFAULT;
8418bccd85fSChristoph Lameter 	nodes_addr(*nodes)[nlongs-1] &= endmask;
8428bccd85fSChristoph Lameter 	return 0;
8438bccd85fSChristoph Lameter }
8448bccd85fSChristoph Lameter 
8458bccd85fSChristoph Lameter /* Copy a kernel node mask to user space */
8468bccd85fSChristoph Lameter static int copy_nodes_to_user(unsigned long __user *mask, unsigned long maxnode,
8478bccd85fSChristoph Lameter 			      nodemask_t *nodes)
8488bccd85fSChristoph Lameter {
8498bccd85fSChristoph Lameter 	unsigned long copy = ALIGN(maxnode-1, 64) / 8;
8508bccd85fSChristoph Lameter 	const int nbytes = BITS_TO_LONGS(MAX_NUMNODES) * sizeof(long);
8518bccd85fSChristoph Lameter 
8528bccd85fSChristoph Lameter 	if (copy > nbytes) {
8538bccd85fSChristoph Lameter 		if (copy > PAGE_SIZE)
8548bccd85fSChristoph Lameter 			return -EINVAL;
8558bccd85fSChristoph Lameter 		if (clear_user((char __user *)mask + nbytes, copy - nbytes))
8568bccd85fSChristoph Lameter 			return -EFAULT;
8578bccd85fSChristoph Lameter 		copy = nbytes;
8588bccd85fSChristoph Lameter 	}
8598bccd85fSChristoph Lameter 	return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0;
8608bccd85fSChristoph Lameter }
8618bccd85fSChristoph Lameter 
8628bccd85fSChristoph Lameter asmlinkage long sys_mbind(unsigned long start, unsigned long len,
8638bccd85fSChristoph Lameter 			unsigned long mode,
8648bccd85fSChristoph Lameter 			unsigned long __user *nmask, unsigned long maxnode,
8658bccd85fSChristoph Lameter 			unsigned flags)
8668bccd85fSChristoph Lameter {
8678bccd85fSChristoph Lameter 	nodemask_t nodes;
8688bccd85fSChristoph Lameter 	int err;
8698bccd85fSChristoph Lameter 
8708bccd85fSChristoph Lameter 	err = get_nodes(&nodes, nmask, maxnode);
8718bccd85fSChristoph Lameter 	if (err)
8728bccd85fSChristoph Lameter 		return err;
8738bccd85fSChristoph Lameter 	return do_mbind(start, len, mode, &nodes, flags);
8748bccd85fSChristoph Lameter }
8758bccd85fSChristoph Lameter 
8768bccd85fSChristoph Lameter /* Set the process memory policy */
8778bccd85fSChristoph Lameter asmlinkage long sys_set_mempolicy(int mode, unsigned long __user *nmask,
8788bccd85fSChristoph Lameter 		unsigned long maxnode)
8798bccd85fSChristoph Lameter {
8808bccd85fSChristoph Lameter 	int err;
8818bccd85fSChristoph Lameter 	nodemask_t nodes;
8828bccd85fSChristoph Lameter 
8838bccd85fSChristoph Lameter 	if (mode < 0 || mode > MPOL_MAX)
8848bccd85fSChristoph Lameter 		return -EINVAL;
8858bccd85fSChristoph Lameter 	err = get_nodes(&nodes, nmask, maxnode);
8868bccd85fSChristoph Lameter 	if (err)
8878bccd85fSChristoph Lameter 		return err;
8888bccd85fSChristoph Lameter 	return do_set_mempolicy(mode, &nodes);
8898bccd85fSChristoph Lameter }
8908bccd85fSChristoph Lameter 
89139743889SChristoph Lameter asmlinkage long sys_migrate_pages(pid_t pid, unsigned long maxnode,
89239743889SChristoph Lameter 		const unsigned long __user *old_nodes,
89339743889SChristoph Lameter 		const unsigned long __user *new_nodes)
89439743889SChristoph Lameter {
89539743889SChristoph Lameter 	struct mm_struct *mm;
89639743889SChristoph Lameter 	struct task_struct *task;
89739743889SChristoph Lameter 	nodemask_t old;
89839743889SChristoph Lameter 	nodemask_t new;
89939743889SChristoph Lameter 	nodemask_t task_nodes;
90039743889SChristoph Lameter 	int err;
90139743889SChristoph Lameter 
90239743889SChristoph Lameter 	err = get_nodes(&old, old_nodes, maxnode);
90339743889SChristoph Lameter 	if (err)
90439743889SChristoph Lameter 		return err;
90539743889SChristoph Lameter 
90639743889SChristoph Lameter 	err = get_nodes(&new, new_nodes, maxnode);
90739743889SChristoph Lameter 	if (err)
90839743889SChristoph Lameter 		return err;
90939743889SChristoph Lameter 
91039743889SChristoph Lameter 	/* Find the mm_struct */
91139743889SChristoph Lameter 	read_lock(&tasklist_lock);
91239743889SChristoph Lameter 	task = pid ? find_task_by_pid(pid) : current;
91339743889SChristoph Lameter 	if (!task) {
91439743889SChristoph Lameter 		read_unlock(&tasklist_lock);
91539743889SChristoph Lameter 		return -ESRCH;
91639743889SChristoph Lameter 	}
91739743889SChristoph Lameter 	mm = get_task_mm(task);
91839743889SChristoph Lameter 	read_unlock(&tasklist_lock);
91939743889SChristoph Lameter 
92039743889SChristoph Lameter 	if (!mm)
92139743889SChristoph Lameter 		return -EINVAL;
92239743889SChristoph Lameter 
92339743889SChristoph Lameter 	/*
92439743889SChristoph Lameter 	 * Check if this process has the right to modify the specified
92539743889SChristoph Lameter 	 * process. The right exists if the process has administrative
92639743889SChristoph Lameter 	 * capabilities, superuser priviledges or the same
92739743889SChristoph Lameter 	 * userid as the target process.
92839743889SChristoph Lameter 	 */
92939743889SChristoph Lameter 	if ((current->euid != task->suid) && (current->euid != task->uid) &&
93039743889SChristoph Lameter 	    (current->uid != task->suid) && (current->uid != task->uid) &&
93139743889SChristoph Lameter 	    !capable(CAP_SYS_ADMIN)) {
93239743889SChristoph Lameter 		err = -EPERM;
93339743889SChristoph Lameter 		goto out;
93439743889SChristoph Lameter 	}
93539743889SChristoph Lameter 
93639743889SChristoph Lameter 	task_nodes = cpuset_mems_allowed(task);
93739743889SChristoph Lameter 	/* Is the user allowed to access the target nodes? */
93839743889SChristoph Lameter 	if (!nodes_subset(new, task_nodes) && !capable(CAP_SYS_ADMIN)) {
93939743889SChristoph Lameter 		err = -EPERM;
94039743889SChristoph Lameter 		goto out;
94139743889SChristoph Lameter 	}
94239743889SChristoph Lameter 
94339743889SChristoph Lameter 	err = do_migrate_pages(mm, &old, &new, MPOL_MF_MOVE);
94439743889SChristoph Lameter out:
94539743889SChristoph Lameter 	mmput(mm);
94639743889SChristoph Lameter 	return err;
94739743889SChristoph Lameter }
94839743889SChristoph Lameter 
94939743889SChristoph Lameter 
9508bccd85fSChristoph Lameter /* Retrieve NUMA policy */
9518bccd85fSChristoph Lameter asmlinkage long sys_get_mempolicy(int __user *policy,
9528bccd85fSChristoph Lameter 				unsigned long __user *nmask,
9538bccd85fSChristoph Lameter 				unsigned long maxnode,
9548bccd85fSChristoph Lameter 				unsigned long addr, unsigned long flags)
9558bccd85fSChristoph Lameter {
9568bccd85fSChristoph Lameter 	int err, pval;
9578bccd85fSChristoph Lameter 	nodemask_t nodes;
9588bccd85fSChristoph Lameter 
9598bccd85fSChristoph Lameter 	if (nmask != NULL && maxnode < MAX_NUMNODES)
9608bccd85fSChristoph Lameter 		return -EINVAL;
9618bccd85fSChristoph Lameter 
9628bccd85fSChristoph Lameter 	err = do_get_mempolicy(&pval, &nodes, addr, flags);
9638bccd85fSChristoph Lameter 
9648bccd85fSChristoph Lameter 	if (err)
9658bccd85fSChristoph Lameter 		return err;
9668bccd85fSChristoph Lameter 
9678bccd85fSChristoph Lameter 	if (policy && put_user(pval, policy))
9688bccd85fSChristoph Lameter 		return -EFAULT;
9698bccd85fSChristoph Lameter 
9708bccd85fSChristoph Lameter 	if (nmask)
9718bccd85fSChristoph Lameter 		err = copy_nodes_to_user(nmask, maxnode, &nodes);
9728bccd85fSChristoph Lameter 
9738bccd85fSChristoph Lameter 	return err;
9748bccd85fSChristoph Lameter }
9758bccd85fSChristoph Lameter 
9761da177e4SLinus Torvalds #ifdef CONFIG_COMPAT
9771da177e4SLinus Torvalds 
9781da177e4SLinus Torvalds asmlinkage long compat_sys_get_mempolicy(int __user *policy,
9791da177e4SLinus Torvalds 				     compat_ulong_t __user *nmask,
9801da177e4SLinus Torvalds 				     compat_ulong_t maxnode,
9811da177e4SLinus Torvalds 				     compat_ulong_t addr, compat_ulong_t flags)
9821da177e4SLinus Torvalds {
9831da177e4SLinus Torvalds 	long err;
9841da177e4SLinus Torvalds 	unsigned long __user *nm = NULL;
9851da177e4SLinus Torvalds 	unsigned long nr_bits, alloc_size;
9861da177e4SLinus Torvalds 	DECLARE_BITMAP(bm, MAX_NUMNODES);
9871da177e4SLinus Torvalds 
9881da177e4SLinus Torvalds 	nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
9891da177e4SLinus Torvalds 	alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
9901da177e4SLinus Torvalds 
9911da177e4SLinus Torvalds 	if (nmask)
9921da177e4SLinus Torvalds 		nm = compat_alloc_user_space(alloc_size);
9931da177e4SLinus Torvalds 
9941da177e4SLinus Torvalds 	err = sys_get_mempolicy(policy, nm, nr_bits+1, addr, flags);
9951da177e4SLinus Torvalds 
9961da177e4SLinus Torvalds 	if (!err && nmask) {
9971da177e4SLinus Torvalds 		err = copy_from_user(bm, nm, alloc_size);
9981da177e4SLinus Torvalds 		/* ensure entire bitmap is zeroed */
9991da177e4SLinus Torvalds 		err |= clear_user(nmask, ALIGN(maxnode-1, 8) / 8);
10001da177e4SLinus Torvalds 		err |= compat_put_bitmap(nmask, bm, nr_bits);
10011da177e4SLinus Torvalds 	}
10021da177e4SLinus Torvalds 
10031da177e4SLinus Torvalds 	return err;
10041da177e4SLinus Torvalds }
10051da177e4SLinus Torvalds 
10061da177e4SLinus Torvalds asmlinkage long compat_sys_set_mempolicy(int mode, compat_ulong_t __user *nmask,
10071da177e4SLinus Torvalds 				     compat_ulong_t maxnode)
10081da177e4SLinus Torvalds {
10091da177e4SLinus Torvalds 	long err = 0;
10101da177e4SLinus Torvalds 	unsigned long __user *nm = NULL;
10111da177e4SLinus Torvalds 	unsigned long nr_bits, alloc_size;
10121da177e4SLinus Torvalds 	DECLARE_BITMAP(bm, MAX_NUMNODES);
10131da177e4SLinus Torvalds 
10141da177e4SLinus Torvalds 	nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
10151da177e4SLinus Torvalds 	alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
10161da177e4SLinus Torvalds 
10171da177e4SLinus Torvalds 	if (nmask) {
10181da177e4SLinus Torvalds 		err = compat_get_bitmap(bm, nmask, nr_bits);
10191da177e4SLinus Torvalds 		nm = compat_alloc_user_space(alloc_size);
10201da177e4SLinus Torvalds 		err |= copy_to_user(nm, bm, alloc_size);
10211da177e4SLinus Torvalds 	}
10221da177e4SLinus Torvalds 
10231da177e4SLinus Torvalds 	if (err)
10241da177e4SLinus Torvalds 		return -EFAULT;
10251da177e4SLinus Torvalds 
10261da177e4SLinus Torvalds 	return sys_set_mempolicy(mode, nm, nr_bits+1);
10271da177e4SLinus Torvalds }
10281da177e4SLinus Torvalds 
10291da177e4SLinus Torvalds asmlinkage long compat_sys_mbind(compat_ulong_t start, compat_ulong_t len,
10301da177e4SLinus Torvalds 			     compat_ulong_t mode, compat_ulong_t __user *nmask,
10311da177e4SLinus Torvalds 			     compat_ulong_t maxnode, compat_ulong_t flags)
10321da177e4SLinus Torvalds {
10331da177e4SLinus Torvalds 	long err = 0;
10341da177e4SLinus Torvalds 	unsigned long __user *nm = NULL;
10351da177e4SLinus Torvalds 	unsigned long nr_bits, alloc_size;
1036dfcd3c0dSAndi Kleen 	nodemask_t bm;
10371da177e4SLinus Torvalds 
10381da177e4SLinus Torvalds 	nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
10391da177e4SLinus Torvalds 	alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
10401da177e4SLinus Torvalds 
10411da177e4SLinus Torvalds 	if (nmask) {
1042dfcd3c0dSAndi Kleen 		err = compat_get_bitmap(nodes_addr(bm), nmask, nr_bits);
10431da177e4SLinus Torvalds 		nm = compat_alloc_user_space(alloc_size);
1044dfcd3c0dSAndi Kleen 		err |= copy_to_user(nm, nodes_addr(bm), alloc_size);
10451da177e4SLinus Torvalds 	}
10461da177e4SLinus Torvalds 
10471da177e4SLinus Torvalds 	if (err)
10481da177e4SLinus Torvalds 		return -EFAULT;
10491da177e4SLinus Torvalds 
10501da177e4SLinus Torvalds 	return sys_mbind(start, len, mode, nm, nr_bits+1, flags);
10511da177e4SLinus Torvalds }
10521da177e4SLinus Torvalds 
10531da177e4SLinus Torvalds #endif
10541da177e4SLinus Torvalds 
10551da177e4SLinus Torvalds /* Return effective policy for a VMA */
105648fce342SChristoph Lameter static struct mempolicy * get_vma_policy(struct task_struct *task,
105748fce342SChristoph Lameter 		struct vm_area_struct *vma, unsigned long addr)
10581da177e4SLinus Torvalds {
10596e21c8f1SChristoph Lameter 	struct mempolicy *pol = task->mempolicy;
10601da177e4SLinus Torvalds 
10611da177e4SLinus Torvalds 	if (vma) {
10621da177e4SLinus Torvalds 		if (vma->vm_ops && vma->vm_ops->get_policy)
10631da177e4SLinus Torvalds 			pol = vma->vm_ops->get_policy(vma, addr);
10641da177e4SLinus Torvalds 		else if (vma->vm_policy &&
10651da177e4SLinus Torvalds 				vma->vm_policy->policy != MPOL_DEFAULT)
10661da177e4SLinus Torvalds 			pol = vma->vm_policy;
10671da177e4SLinus Torvalds 	}
10681da177e4SLinus Torvalds 	if (!pol)
10691da177e4SLinus Torvalds 		pol = &default_policy;
10701da177e4SLinus Torvalds 	return pol;
10711da177e4SLinus Torvalds }
10721da177e4SLinus Torvalds 
10731da177e4SLinus Torvalds /* Return a zonelist representing a mempolicy */
1074dd0fc66fSAl Viro static struct zonelist *zonelist_policy(gfp_t gfp, struct mempolicy *policy)
10751da177e4SLinus Torvalds {
10761da177e4SLinus Torvalds 	int nd;
10771da177e4SLinus Torvalds 
10781da177e4SLinus Torvalds 	switch (policy->policy) {
10791da177e4SLinus Torvalds 	case MPOL_PREFERRED:
10801da177e4SLinus Torvalds 		nd = policy->v.preferred_node;
10811da177e4SLinus Torvalds 		if (nd < 0)
10821da177e4SLinus Torvalds 			nd = numa_node_id();
10831da177e4SLinus Torvalds 		break;
10841da177e4SLinus Torvalds 	case MPOL_BIND:
10851da177e4SLinus Torvalds 		/* Lower zones don't get a policy applied */
10861da177e4SLinus Torvalds 		/* Careful: current->mems_allowed might have moved */
1087af4ca457SAl Viro 		if (gfp_zone(gfp) >= policy_zone)
10881da177e4SLinus Torvalds 			if (cpuset_zonelist_valid_mems_allowed(policy->v.zonelist))
10891da177e4SLinus Torvalds 				return policy->v.zonelist;
10901da177e4SLinus Torvalds 		/*FALL THROUGH*/
10911da177e4SLinus Torvalds 	case MPOL_INTERLEAVE: /* should not happen */
10921da177e4SLinus Torvalds 	case MPOL_DEFAULT:
10931da177e4SLinus Torvalds 		nd = numa_node_id();
10941da177e4SLinus Torvalds 		break;
10951da177e4SLinus Torvalds 	default:
10961da177e4SLinus Torvalds 		nd = 0;
10971da177e4SLinus Torvalds 		BUG();
10981da177e4SLinus Torvalds 	}
1099af4ca457SAl Viro 	return NODE_DATA(nd)->node_zonelists + gfp_zone(gfp);
11001da177e4SLinus Torvalds }
11011da177e4SLinus Torvalds 
11021da177e4SLinus Torvalds /* Do dynamic interleaving for a process */
11031da177e4SLinus Torvalds static unsigned interleave_nodes(struct mempolicy *policy)
11041da177e4SLinus Torvalds {
11051da177e4SLinus Torvalds 	unsigned nid, next;
11061da177e4SLinus Torvalds 	struct task_struct *me = current;
11071da177e4SLinus Torvalds 
11081da177e4SLinus Torvalds 	nid = me->il_next;
1109dfcd3c0dSAndi Kleen 	next = next_node(nid, policy->v.nodes);
11101da177e4SLinus Torvalds 	if (next >= MAX_NUMNODES)
1111dfcd3c0dSAndi Kleen 		next = first_node(policy->v.nodes);
11121da177e4SLinus Torvalds 	me->il_next = next;
11131da177e4SLinus Torvalds 	return nid;
11141da177e4SLinus Torvalds }
11151da177e4SLinus Torvalds 
1116dc85da15SChristoph Lameter /*
1117dc85da15SChristoph Lameter  * Depending on the memory policy provide a node from which to allocate the
1118dc85da15SChristoph Lameter  * next slab entry.
1119dc85da15SChristoph Lameter  */
1120dc85da15SChristoph Lameter unsigned slab_node(struct mempolicy *policy)
1121dc85da15SChristoph Lameter {
1122dc85da15SChristoph Lameter 	switch (policy->policy) {
1123dc85da15SChristoph Lameter 	case MPOL_INTERLEAVE:
1124dc85da15SChristoph Lameter 		return interleave_nodes(policy);
1125dc85da15SChristoph Lameter 
1126dc85da15SChristoph Lameter 	case MPOL_BIND:
1127dc85da15SChristoph Lameter 		/*
1128dc85da15SChristoph Lameter 		 * Follow bind policy behavior and start allocation at the
1129dc85da15SChristoph Lameter 		 * first node.
1130dc85da15SChristoph Lameter 		 */
1131dc85da15SChristoph Lameter 		return policy->v.zonelist->zones[0]->zone_pgdat->node_id;
1132dc85da15SChristoph Lameter 
1133dc85da15SChristoph Lameter 	case MPOL_PREFERRED:
1134dc85da15SChristoph Lameter 		if (policy->v.preferred_node >= 0)
1135dc85da15SChristoph Lameter 			return policy->v.preferred_node;
1136dc85da15SChristoph Lameter 		/* Fall through */
1137dc85da15SChristoph Lameter 
1138dc85da15SChristoph Lameter 	default:
1139dc85da15SChristoph Lameter 		return numa_node_id();
1140dc85da15SChristoph Lameter 	}
1141dc85da15SChristoph Lameter }
1142dc85da15SChristoph Lameter 
11431da177e4SLinus Torvalds /* Do static interleaving for a VMA with known offset. */
11441da177e4SLinus Torvalds static unsigned offset_il_node(struct mempolicy *pol,
11451da177e4SLinus Torvalds 		struct vm_area_struct *vma, unsigned long off)
11461da177e4SLinus Torvalds {
1147dfcd3c0dSAndi Kleen 	unsigned nnodes = nodes_weight(pol->v.nodes);
11481da177e4SLinus Torvalds 	unsigned target = (unsigned)off % nnodes;
11491da177e4SLinus Torvalds 	int c;
11501da177e4SLinus Torvalds 	int nid = -1;
11511da177e4SLinus Torvalds 
11521da177e4SLinus Torvalds 	c = 0;
11531da177e4SLinus Torvalds 	do {
1154dfcd3c0dSAndi Kleen 		nid = next_node(nid, pol->v.nodes);
11551da177e4SLinus Torvalds 		c++;
11561da177e4SLinus Torvalds 	} while (c <= target);
11571da177e4SLinus Torvalds 	return nid;
11581da177e4SLinus Torvalds }
11591da177e4SLinus Torvalds 
11605da7ca86SChristoph Lameter /* Determine a node number for interleave */
11615da7ca86SChristoph Lameter static inline unsigned interleave_nid(struct mempolicy *pol,
11625da7ca86SChristoph Lameter 		 struct vm_area_struct *vma, unsigned long addr, int shift)
11635da7ca86SChristoph Lameter {
11645da7ca86SChristoph Lameter 	if (vma) {
11655da7ca86SChristoph Lameter 		unsigned long off;
11665da7ca86SChristoph Lameter 
11675da7ca86SChristoph Lameter 		off = vma->vm_pgoff;
11685da7ca86SChristoph Lameter 		off += (addr - vma->vm_start) >> shift;
11695da7ca86SChristoph Lameter 		return offset_il_node(pol, vma, off);
11705da7ca86SChristoph Lameter 	} else
11715da7ca86SChristoph Lameter 		return interleave_nodes(pol);
11725da7ca86SChristoph Lameter }
11735da7ca86SChristoph Lameter 
117400ac59adSChen, Kenneth W #ifdef CONFIG_HUGETLBFS
11755da7ca86SChristoph Lameter /* Return a zonelist suitable for a huge page allocation. */
11765da7ca86SChristoph Lameter struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr)
11775da7ca86SChristoph Lameter {
11785da7ca86SChristoph Lameter 	struct mempolicy *pol = get_vma_policy(current, vma, addr);
11795da7ca86SChristoph Lameter 
11805da7ca86SChristoph Lameter 	if (pol->policy == MPOL_INTERLEAVE) {
11815da7ca86SChristoph Lameter 		unsigned nid;
11825da7ca86SChristoph Lameter 
11835da7ca86SChristoph Lameter 		nid = interleave_nid(pol, vma, addr, HPAGE_SHIFT);
11845da7ca86SChristoph Lameter 		return NODE_DATA(nid)->node_zonelists + gfp_zone(GFP_HIGHUSER);
11855da7ca86SChristoph Lameter 	}
11865da7ca86SChristoph Lameter 	return zonelist_policy(GFP_HIGHUSER, pol);
11875da7ca86SChristoph Lameter }
118800ac59adSChen, Kenneth W #endif
11895da7ca86SChristoph Lameter 
11901da177e4SLinus Torvalds /* Allocate a page in interleaved policy.
11911da177e4SLinus Torvalds    Own path because it needs to do special accounting. */
1192662f3a0bSAndi Kleen static struct page *alloc_page_interleave(gfp_t gfp, unsigned order,
1193662f3a0bSAndi Kleen 					unsigned nid)
11941da177e4SLinus Torvalds {
11951da177e4SLinus Torvalds 	struct zonelist *zl;
11961da177e4SLinus Torvalds 	struct page *page;
11971da177e4SLinus Torvalds 
1198af4ca457SAl Viro 	zl = NODE_DATA(nid)->node_zonelists + gfp_zone(gfp);
11991da177e4SLinus Torvalds 	page = __alloc_pages(gfp, order, zl);
12001da177e4SLinus Torvalds 	if (page && page_zone(page) == zl->zones[0]) {
1201e7c8d5c9SChristoph Lameter 		zone_pcp(zl->zones[0],get_cpu())->interleave_hit++;
12021da177e4SLinus Torvalds 		put_cpu();
12031da177e4SLinus Torvalds 	}
12041da177e4SLinus Torvalds 	return page;
12051da177e4SLinus Torvalds }
12061da177e4SLinus Torvalds 
12071da177e4SLinus Torvalds /**
12081da177e4SLinus Torvalds  * 	alloc_page_vma	- Allocate a page for a VMA.
12091da177e4SLinus Torvalds  *
12101da177e4SLinus Torvalds  * 	@gfp:
12111da177e4SLinus Torvalds  *      %GFP_USER    user allocation.
12121da177e4SLinus Torvalds  *      %GFP_KERNEL  kernel allocations,
12131da177e4SLinus Torvalds  *      %GFP_HIGHMEM highmem/user allocations,
12141da177e4SLinus Torvalds  *      %GFP_FS      allocation should not call back into a file system.
12151da177e4SLinus Torvalds  *      %GFP_ATOMIC  don't sleep.
12161da177e4SLinus Torvalds  *
12171da177e4SLinus Torvalds  * 	@vma:  Pointer to VMA or NULL if not available.
12181da177e4SLinus Torvalds  *	@addr: Virtual Address of the allocation. Must be inside the VMA.
12191da177e4SLinus Torvalds  *
12201da177e4SLinus Torvalds  * 	This function allocates a page from the kernel page pool and applies
12211da177e4SLinus Torvalds  *	a NUMA policy associated with the VMA or the current process.
12221da177e4SLinus Torvalds  *	When VMA is not NULL caller must hold down_read on the mmap_sem of the
12231da177e4SLinus Torvalds  *	mm_struct of the VMA to prevent it from going away. Should be used for
12241da177e4SLinus Torvalds  *	all allocations for pages that will be mapped into
12251da177e4SLinus Torvalds  * 	user space. Returns NULL when no page can be allocated.
12261da177e4SLinus Torvalds  *
12271da177e4SLinus Torvalds  *	Should be called with the mm_sem of the vma hold.
12281da177e4SLinus Torvalds  */
12291da177e4SLinus Torvalds struct page *
1230dd0fc66fSAl Viro alloc_page_vma(gfp_t gfp, struct vm_area_struct *vma, unsigned long addr)
12311da177e4SLinus Torvalds {
12326e21c8f1SChristoph Lameter 	struct mempolicy *pol = get_vma_policy(current, vma, addr);
12331da177e4SLinus Torvalds 
1234cf2a473cSPaul Jackson 	cpuset_update_task_memory_state();
12351da177e4SLinus Torvalds 
12361da177e4SLinus Torvalds 	if (unlikely(pol->policy == MPOL_INTERLEAVE)) {
12371da177e4SLinus Torvalds 		unsigned nid;
12385da7ca86SChristoph Lameter 
12395da7ca86SChristoph Lameter 		nid = interleave_nid(pol, vma, addr, PAGE_SHIFT);
12401da177e4SLinus Torvalds 		return alloc_page_interleave(gfp, 0, nid);
12411da177e4SLinus Torvalds 	}
12421da177e4SLinus Torvalds 	return __alloc_pages(gfp, 0, zonelist_policy(gfp, pol));
12431da177e4SLinus Torvalds }
12441da177e4SLinus Torvalds 
12451da177e4SLinus Torvalds /**
12461da177e4SLinus Torvalds  * 	alloc_pages_current - Allocate pages.
12471da177e4SLinus Torvalds  *
12481da177e4SLinus Torvalds  *	@gfp:
12491da177e4SLinus Torvalds  *		%GFP_USER   user allocation,
12501da177e4SLinus Torvalds  *      	%GFP_KERNEL kernel allocation,
12511da177e4SLinus Torvalds  *      	%GFP_HIGHMEM highmem allocation,
12521da177e4SLinus Torvalds  *      	%GFP_FS     don't call back into a file system.
12531da177e4SLinus Torvalds  *      	%GFP_ATOMIC don't sleep.
12541da177e4SLinus Torvalds  *	@order: Power of two of allocation size in pages. 0 is a single page.
12551da177e4SLinus Torvalds  *
12561da177e4SLinus Torvalds  *	Allocate a page from the kernel page pool.  When not in
12571da177e4SLinus Torvalds  *	interrupt context and apply the current process NUMA policy.
12581da177e4SLinus Torvalds  *	Returns NULL when no page can be allocated.
12591da177e4SLinus Torvalds  *
1260cf2a473cSPaul Jackson  *	Don't call cpuset_update_task_memory_state() unless
12611da177e4SLinus Torvalds  *	1) it's ok to take cpuset_sem (can WAIT), and
12621da177e4SLinus Torvalds  *	2) allocating for current task (not interrupt).
12631da177e4SLinus Torvalds  */
1264dd0fc66fSAl Viro struct page *alloc_pages_current(gfp_t gfp, unsigned order)
12651da177e4SLinus Torvalds {
12661da177e4SLinus Torvalds 	struct mempolicy *pol = current->mempolicy;
12671da177e4SLinus Torvalds 
12681da177e4SLinus Torvalds 	if ((gfp & __GFP_WAIT) && !in_interrupt())
1269cf2a473cSPaul Jackson 		cpuset_update_task_memory_state();
12701da177e4SLinus Torvalds 	if (!pol || in_interrupt())
12711da177e4SLinus Torvalds 		pol = &default_policy;
12721da177e4SLinus Torvalds 	if (pol->policy == MPOL_INTERLEAVE)
12731da177e4SLinus Torvalds 		return alloc_page_interleave(gfp, order, interleave_nodes(pol));
12741da177e4SLinus Torvalds 	return __alloc_pages(gfp, order, zonelist_policy(gfp, pol));
12751da177e4SLinus Torvalds }
12761da177e4SLinus Torvalds EXPORT_SYMBOL(alloc_pages_current);
12771da177e4SLinus Torvalds 
12784225399aSPaul Jackson /*
12794225399aSPaul Jackson  * If mpol_copy() sees current->cpuset == cpuset_being_rebound, then it
12804225399aSPaul Jackson  * rebinds the mempolicy its copying by calling mpol_rebind_policy()
12814225399aSPaul Jackson  * with the mems_allowed returned by cpuset_mems_allowed().  This
12824225399aSPaul Jackson  * keeps mempolicies cpuset relative after its cpuset moves.  See
12834225399aSPaul Jackson  * further kernel/cpuset.c update_nodemask().
12844225399aSPaul Jackson  */
12854225399aSPaul Jackson void *cpuset_being_rebound;
12864225399aSPaul Jackson 
12871da177e4SLinus Torvalds /* Slow path of a mempolicy copy */
12881da177e4SLinus Torvalds struct mempolicy *__mpol_copy(struct mempolicy *old)
12891da177e4SLinus Torvalds {
12901da177e4SLinus Torvalds 	struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
12911da177e4SLinus Torvalds 
12921da177e4SLinus Torvalds 	if (!new)
12931da177e4SLinus Torvalds 		return ERR_PTR(-ENOMEM);
12944225399aSPaul Jackson 	if (current_cpuset_is_being_rebound()) {
12954225399aSPaul Jackson 		nodemask_t mems = cpuset_mems_allowed(current);
12964225399aSPaul Jackson 		mpol_rebind_policy(old, &mems);
12974225399aSPaul Jackson 	}
12981da177e4SLinus Torvalds 	*new = *old;
12991da177e4SLinus Torvalds 	atomic_set(&new->refcnt, 1);
13001da177e4SLinus Torvalds 	if (new->policy == MPOL_BIND) {
13011da177e4SLinus Torvalds 		int sz = ksize(old->v.zonelist);
13021da177e4SLinus Torvalds 		new->v.zonelist = kmalloc(sz, SLAB_KERNEL);
13031da177e4SLinus Torvalds 		if (!new->v.zonelist) {
13041da177e4SLinus Torvalds 			kmem_cache_free(policy_cache, new);
13051da177e4SLinus Torvalds 			return ERR_PTR(-ENOMEM);
13061da177e4SLinus Torvalds 		}
13071da177e4SLinus Torvalds 		memcpy(new->v.zonelist, old->v.zonelist, sz);
13081da177e4SLinus Torvalds 	}
13091da177e4SLinus Torvalds 	return new;
13101da177e4SLinus Torvalds }
13111da177e4SLinus Torvalds 
13121da177e4SLinus Torvalds /* Slow path of a mempolicy comparison */
13131da177e4SLinus Torvalds int __mpol_equal(struct mempolicy *a, struct mempolicy *b)
13141da177e4SLinus Torvalds {
13151da177e4SLinus Torvalds 	if (!a || !b)
13161da177e4SLinus Torvalds 		return 0;
13171da177e4SLinus Torvalds 	if (a->policy != b->policy)
13181da177e4SLinus Torvalds 		return 0;
13191da177e4SLinus Torvalds 	switch (a->policy) {
13201da177e4SLinus Torvalds 	case MPOL_DEFAULT:
13211da177e4SLinus Torvalds 		return 1;
13221da177e4SLinus Torvalds 	case MPOL_INTERLEAVE:
1323dfcd3c0dSAndi Kleen 		return nodes_equal(a->v.nodes, b->v.nodes);
13241da177e4SLinus Torvalds 	case MPOL_PREFERRED:
13251da177e4SLinus Torvalds 		return a->v.preferred_node == b->v.preferred_node;
13261da177e4SLinus Torvalds 	case MPOL_BIND: {
13271da177e4SLinus Torvalds 		int i;
13281da177e4SLinus Torvalds 		for (i = 0; a->v.zonelist->zones[i]; i++)
13291da177e4SLinus Torvalds 			if (a->v.zonelist->zones[i] != b->v.zonelist->zones[i])
13301da177e4SLinus Torvalds 				return 0;
13311da177e4SLinus Torvalds 		return b->v.zonelist->zones[i] == NULL;
13321da177e4SLinus Torvalds 	}
13331da177e4SLinus Torvalds 	default:
13341da177e4SLinus Torvalds 		BUG();
13351da177e4SLinus Torvalds 		return 0;
13361da177e4SLinus Torvalds 	}
13371da177e4SLinus Torvalds }
13381da177e4SLinus Torvalds 
13391da177e4SLinus Torvalds /* Slow path of a mpol destructor. */
13401da177e4SLinus Torvalds void __mpol_free(struct mempolicy *p)
13411da177e4SLinus Torvalds {
13421da177e4SLinus Torvalds 	if (!atomic_dec_and_test(&p->refcnt))
13431da177e4SLinus Torvalds 		return;
13441da177e4SLinus Torvalds 	if (p->policy == MPOL_BIND)
13451da177e4SLinus Torvalds 		kfree(p->v.zonelist);
13461da177e4SLinus Torvalds 	p->policy = MPOL_DEFAULT;
13471da177e4SLinus Torvalds 	kmem_cache_free(policy_cache, p);
13481da177e4SLinus Torvalds }
13491da177e4SLinus Torvalds 
13501da177e4SLinus Torvalds /*
13511da177e4SLinus Torvalds  * Shared memory backing store policy support.
13521da177e4SLinus Torvalds  *
13531da177e4SLinus Torvalds  * Remember policies even when nobody has shared memory mapped.
13541da177e4SLinus Torvalds  * The policies are kept in Red-Black tree linked from the inode.
13551da177e4SLinus Torvalds  * They are protected by the sp->lock spinlock, which should be held
13561da177e4SLinus Torvalds  * for any accesses to the tree.
13571da177e4SLinus Torvalds  */
13581da177e4SLinus Torvalds 
13591da177e4SLinus Torvalds /* lookup first element intersecting start-end */
13601da177e4SLinus Torvalds /* Caller holds sp->lock */
13611da177e4SLinus Torvalds static struct sp_node *
13621da177e4SLinus Torvalds sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end)
13631da177e4SLinus Torvalds {
13641da177e4SLinus Torvalds 	struct rb_node *n = sp->root.rb_node;
13651da177e4SLinus Torvalds 
13661da177e4SLinus Torvalds 	while (n) {
13671da177e4SLinus Torvalds 		struct sp_node *p = rb_entry(n, struct sp_node, nd);
13681da177e4SLinus Torvalds 
13691da177e4SLinus Torvalds 		if (start >= p->end)
13701da177e4SLinus Torvalds 			n = n->rb_right;
13711da177e4SLinus Torvalds 		else if (end <= p->start)
13721da177e4SLinus Torvalds 			n = n->rb_left;
13731da177e4SLinus Torvalds 		else
13741da177e4SLinus Torvalds 			break;
13751da177e4SLinus Torvalds 	}
13761da177e4SLinus Torvalds 	if (!n)
13771da177e4SLinus Torvalds 		return NULL;
13781da177e4SLinus Torvalds 	for (;;) {
13791da177e4SLinus Torvalds 		struct sp_node *w = NULL;
13801da177e4SLinus Torvalds 		struct rb_node *prev = rb_prev(n);
13811da177e4SLinus Torvalds 		if (!prev)
13821da177e4SLinus Torvalds 			break;
13831da177e4SLinus Torvalds 		w = rb_entry(prev, struct sp_node, nd);
13841da177e4SLinus Torvalds 		if (w->end <= start)
13851da177e4SLinus Torvalds 			break;
13861da177e4SLinus Torvalds 		n = prev;
13871da177e4SLinus Torvalds 	}
13881da177e4SLinus Torvalds 	return rb_entry(n, struct sp_node, nd);
13891da177e4SLinus Torvalds }
13901da177e4SLinus Torvalds 
13911da177e4SLinus Torvalds /* Insert a new shared policy into the list. */
13921da177e4SLinus Torvalds /* Caller holds sp->lock */
13931da177e4SLinus Torvalds static void sp_insert(struct shared_policy *sp, struct sp_node *new)
13941da177e4SLinus Torvalds {
13951da177e4SLinus Torvalds 	struct rb_node **p = &sp->root.rb_node;
13961da177e4SLinus Torvalds 	struct rb_node *parent = NULL;
13971da177e4SLinus Torvalds 	struct sp_node *nd;
13981da177e4SLinus Torvalds 
13991da177e4SLinus Torvalds 	while (*p) {
14001da177e4SLinus Torvalds 		parent = *p;
14011da177e4SLinus Torvalds 		nd = rb_entry(parent, struct sp_node, nd);
14021da177e4SLinus Torvalds 		if (new->start < nd->start)
14031da177e4SLinus Torvalds 			p = &(*p)->rb_left;
14041da177e4SLinus Torvalds 		else if (new->end > nd->end)
14051da177e4SLinus Torvalds 			p = &(*p)->rb_right;
14061da177e4SLinus Torvalds 		else
14071da177e4SLinus Torvalds 			BUG();
14081da177e4SLinus Torvalds 	}
14091da177e4SLinus Torvalds 	rb_link_node(&new->nd, parent, p);
14101da177e4SLinus Torvalds 	rb_insert_color(&new->nd, &sp->root);
14111da177e4SLinus Torvalds 	PDprintk("inserting %lx-%lx: %d\n", new->start, new->end,
14121da177e4SLinus Torvalds 		 new->policy ? new->policy->policy : 0);
14131da177e4SLinus Torvalds }
14141da177e4SLinus Torvalds 
14151da177e4SLinus Torvalds /* Find shared policy intersecting idx */
14161da177e4SLinus Torvalds struct mempolicy *
14171da177e4SLinus Torvalds mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx)
14181da177e4SLinus Torvalds {
14191da177e4SLinus Torvalds 	struct mempolicy *pol = NULL;
14201da177e4SLinus Torvalds 	struct sp_node *sn;
14211da177e4SLinus Torvalds 
14221da177e4SLinus Torvalds 	if (!sp->root.rb_node)
14231da177e4SLinus Torvalds 		return NULL;
14241da177e4SLinus Torvalds 	spin_lock(&sp->lock);
14251da177e4SLinus Torvalds 	sn = sp_lookup(sp, idx, idx+1);
14261da177e4SLinus Torvalds 	if (sn) {
14271da177e4SLinus Torvalds 		mpol_get(sn->policy);
14281da177e4SLinus Torvalds 		pol = sn->policy;
14291da177e4SLinus Torvalds 	}
14301da177e4SLinus Torvalds 	spin_unlock(&sp->lock);
14311da177e4SLinus Torvalds 	return pol;
14321da177e4SLinus Torvalds }
14331da177e4SLinus Torvalds 
14341da177e4SLinus Torvalds static void sp_delete(struct shared_policy *sp, struct sp_node *n)
14351da177e4SLinus Torvalds {
14361da177e4SLinus Torvalds 	PDprintk("deleting %lx-l%x\n", n->start, n->end);
14371da177e4SLinus Torvalds 	rb_erase(&n->nd, &sp->root);
14381da177e4SLinus Torvalds 	mpol_free(n->policy);
14391da177e4SLinus Torvalds 	kmem_cache_free(sn_cache, n);
14401da177e4SLinus Torvalds }
14411da177e4SLinus Torvalds 
14421da177e4SLinus Torvalds struct sp_node *
14431da177e4SLinus Torvalds sp_alloc(unsigned long start, unsigned long end, struct mempolicy *pol)
14441da177e4SLinus Torvalds {
14451da177e4SLinus Torvalds 	struct sp_node *n = kmem_cache_alloc(sn_cache, GFP_KERNEL);
14461da177e4SLinus Torvalds 
14471da177e4SLinus Torvalds 	if (!n)
14481da177e4SLinus Torvalds 		return NULL;
14491da177e4SLinus Torvalds 	n->start = start;
14501da177e4SLinus Torvalds 	n->end = end;
14511da177e4SLinus Torvalds 	mpol_get(pol);
14521da177e4SLinus Torvalds 	n->policy = pol;
14531da177e4SLinus Torvalds 	return n;
14541da177e4SLinus Torvalds }
14551da177e4SLinus Torvalds 
14561da177e4SLinus Torvalds /* Replace a policy range. */
14571da177e4SLinus Torvalds static int shared_policy_replace(struct shared_policy *sp, unsigned long start,
14581da177e4SLinus Torvalds 				 unsigned long end, struct sp_node *new)
14591da177e4SLinus Torvalds {
14601da177e4SLinus Torvalds 	struct sp_node *n, *new2 = NULL;
14611da177e4SLinus Torvalds 
14621da177e4SLinus Torvalds restart:
14631da177e4SLinus Torvalds 	spin_lock(&sp->lock);
14641da177e4SLinus Torvalds 	n = sp_lookup(sp, start, end);
14651da177e4SLinus Torvalds 	/* Take care of old policies in the same range. */
14661da177e4SLinus Torvalds 	while (n && n->start < end) {
14671da177e4SLinus Torvalds 		struct rb_node *next = rb_next(&n->nd);
14681da177e4SLinus Torvalds 		if (n->start >= start) {
14691da177e4SLinus Torvalds 			if (n->end <= end)
14701da177e4SLinus Torvalds 				sp_delete(sp, n);
14711da177e4SLinus Torvalds 			else
14721da177e4SLinus Torvalds 				n->start = end;
14731da177e4SLinus Torvalds 		} else {
14741da177e4SLinus Torvalds 			/* Old policy spanning whole new range. */
14751da177e4SLinus Torvalds 			if (n->end > end) {
14761da177e4SLinus Torvalds 				if (!new2) {
14771da177e4SLinus Torvalds 					spin_unlock(&sp->lock);
14781da177e4SLinus Torvalds 					new2 = sp_alloc(end, n->end, n->policy);
14791da177e4SLinus Torvalds 					if (!new2)
14801da177e4SLinus Torvalds 						return -ENOMEM;
14811da177e4SLinus Torvalds 					goto restart;
14821da177e4SLinus Torvalds 				}
14831da177e4SLinus Torvalds 				n->end = start;
14841da177e4SLinus Torvalds 				sp_insert(sp, new2);
14851da177e4SLinus Torvalds 				new2 = NULL;
14861da177e4SLinus Torvalds 				break;
14871da177e4SLinus Torvalds 			} else
14881da177e4SLinus Torvalds 				n->end = start;
14891da177e4SLinus Torvalds 		}
14901da177e4SLinus Torvalds 		if (!next)
14911da177e4SLinus Torvalds 			break;
14921da177e4SLinus Torvalds 		n = rb_entry(next, struct sp_node, nd);
14931da177e4SLinus Torvalds 	}
14941da177e4SLinus Torvalds 	if (new)
14951da177e4SLinus Torvalds 		sp_insert(sp, new);
14961da177e4SLinus Torvalds 	spin_unlock(&sp->lock);
14971da177e4SLinus Torvalds 	if (new2) {
14981da177e4SLinus Torvalds 		mpol_free(new2->policy);
14991da177e4SLinus Torvalds 		kmem_cache_free(sn_cache, new2);
15001da177e4SLinus Torvalds 	}
15011da177e4SLinus Torvalds 	return 0;
15021da177e4SLinus Torvalds }
15031da177e4SLinus Torvalds 
15047339ff83SRobin Holt void mpol_shared_policy_init(struct shared_policy *info, int policy,
15057339ff83SRobin Holt 				nodemask_t *policy_nodes)
15067339ff83SRobin Holt {
15077339ff83SRobin Holt 	info->root = RB_ROOT;
15087339ff83SRobin Holt 	spin_lock_init(&info->lock);
15097339ff83SRobin Holt 
15107339ff83SRobin Holt 	if (policy != MPOL_DEFAULT) {
15117339ff83SRobin Holt 		struct mempolicy *newpol;
15127339ff83SRobin Holt 
15137339ff83SRobin Holt 		/* Falls back to MPOL_DEFAULT on any error */
15147339ff83SRobin Holt 		newpol = mpol_new(policy, policy_nodes);
15157339ff83SRobin Holt 		if (!IS_ERR(newpol)) {
15167339ff83SRobin Holt 			/* Create pseudo-vma that contains just the policy */
15177339ff83SRobin Holt 			struct vm_area_struct pvma;
15187339ff83SRobin Holt 
15197339ff83SRobin Holt 			memset(&pvma, 0, sizeof(struct vm_area_struct));
15207339ff83SRobin Holt 			/* Policy covers entire file */
15217339ff83SRobin Holt 			pvma.vm_end = TASK_SIZE;
15227339ff83SRobin Holt 			mpol_set_shared_policy(info, &pvma, newpol);
15237339ff83SRobin Holt 			mpol_free(newpol);
15247339ff83SRobin Holt 		}
15257339ff83SRobin Holt 	}
15267339ff83SRobin Holt }
15277339ff83SRobin Holt 
15281da177e4SLinus Torvalds int mpol_set_shared_policy(struct shared_policy *info,
15291da177e4SLinus Torvalds 			struct vm_area_struct *vma, struct mempolicy *npol)
15301da177e4SLinus Torvalds {
15311da177e4SLinus Torvalds 	int err;
15321da177e4SLinus Torvalds 	struct sp_node *new = NULL;
15331da177e4SLinus Torvalds 	unsigned long sz = vma_pages(vma);
15341da177e4SLinus Torvalds 
15351da177e4SLinus Torvalds 	PDprintk("set_shared_policy %lx sz %lu %d %lx\n",
15361da177e4SLinus Torvalds 		 vma->vm_pgoff,
15371da177e4SLinus Torvalds 		 sz, npol? npol->policy : -1,
1538dfcd3c0dSAndi Kleen 		npol ? nodes_addr(npol->v.nodes)[0] : -1);
15391da177e4SLinus Torvalds 
15401da177e4SLinus Torvalds 	if (npol) {
15411da177e4SLinus Torvalds 		new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, npol);
15421da177e4SLinus Torvalds 		if (!new)
15431da177e4SLinus Torvalds 			return -ENOMEM;
15441da177e4SLinus Torvalds 	}
15451da177e4SLinus Torvalds 	err = shared_policy_replace(info, vma->vm_pgoff, vma->vm_pgoff+sz, new);
15461da177e4SLinus Torvalds 	if (err && new)
15471da177e4SLinus Torvalds 		kmem_cache_free(sn_cache, new);
15481da177e4SLinus Torvalds 	return err;
15491da177e4SLinus Torvalds }
15501da177e4SLinus Torvalds 
15511da177e4SLinus Torvalds /* Free a backing policy store on inode delete. */
15521da177e4SLinus Torvalds void mpol_free_shared_policy(struct shared_policy *p)
15531da177e4SLinus Torvalds {
15541da177e4SLinus Torvalds 	struct sp_node *n;
15551da177e4SLinus Torvalds 	struct rb_node *next;
15561da177e4SLinus Torvalds 
15571da177e4SLinus Torvalds 	if (!p->root.rb_node)
15581da177e4SLinus Torvalds 		return;
15591da177e4SLinus Torvalds 	spin_lock(&p->lock);
15601da177e4SLinus Torvalds 	next = rb_first(&p->root);
15611da177e4SLinus Torvalds 	while (next) {
15621da177e4SLinus Torvalds 		n = rb_entry(next, struct sp_node, nd);
15631da177e4SLinus Torvalds 		next = rb_next(&n->nd);
156490c5029eSAndi Kleen 		rb_erase(&n->nd, &p->root);
15651da177e4SLinus Torvalds 		mpol_free(n->policy);
15661da177e4SLinus Torvalds 		kmem_cache_free(sn_cache, n);
15671da177e4SLinus Torvalds 	}
15681da177e4SLinus Torvalds 	spin_unlock(&p->lock);
15691da177e4SLinus Torvalds }
15701da177e4SLinus Torvalds 
15711da177e4SLinus Torvalds /* assumes fs == KERNEL_DS */
15721da177e4SLinus Torvalds void __init numa_policy_init(void)
15731da177e4SLinus Torvalds {
15741da177e4SLinus Torvalds 	policy_cache = kmem_cache_create("numa_policy",
15751da177e4SLinus Torvalds 					 sizeof(struct mempolicy),
15761da177e4SLinus Torvalds 					 0, SLAB_PANIC, NULL, NULL);
15771da177e4SLinus Torvalds 
15781da177e4SLinus Torvalds 	sn_cache = kmem_cache_create("shared_policy_node",
15791da177e4SLinus Torvalds 				     sizeof(struct sp_node),
15801da177e4SLinus Torvalds 				     0, SLAB_PANIC, NULL, NULL);
15811da177e4SLinus Torvalds 
15821da177e4SLinus Torvalds 	/* Set interleaving policy for system init. This way not all
15831da177e4SLinus Torvalds 	   the data structures allocated at system boot end up in node zero. */
15841da177e4SLinus Torvalds 
15858bccd85fSChristoph Lameter 	if (do_set_mempolicy(MPOL_INTERLEAVE, &node_online_map))
15861da177e4SLinus Torvalds 		printk("numa_policy_init: interleaving failed\n");
15871da177e4SLinus Torvalds }
15881da177e4SLinus Torvalds 
15898bccd85fSChristoph Lameter /* Reset policy of current process to default */
15901da177e4SLinus Torvalds void numa_default_policy(void)
15911da177e4SLinus Torvalds {
15928bccd85fSChristoph Lameter 	do_set_mempolicy(MPOL_DEFAULT, NULL);
15931da177e4SLinus Torvalds }
159468860ec1SPaul Jackson 
159568860ec1SPaul Jackson /* Migrate a policy to a different set of nodes */
159674cb2155SPaul Jackson void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask)
159768860ec1SPaul Jackson {
159874cb2155SPaul Jackson 	nodemask_t *mpolmask;
159968860ec1SPaul Jackson 	nodemask_t tmp;
160068860ec1SPaul Jackson 
160168860ec1SPaul Jackson 	if (!pol)
160268860ec1SPaul Jackson 		return;
160374cb2155SPaul Jackson 	mpolmask = &pol->cpuset_mems_allowed;
160474cb2155SPaul Jackson 	if (nodes_equal(*mpolmask, *newmask))
160574cb2155SPaul Jackson 		return;
160668860ec1SPaul Jackson 
160768860ec1SPaul Jackson 	switch (pol->policy) {
160868860ec1SPaul Jackson 	case MPOL_DEFAULT:
160968860ec1SPaul Jackson 		break;
161068860ec1SPaul Jackson 	case MPOL_INTERLEAVE:
161174cb2155SPaul Jackson 		nodes_remap(tmp, pol->v.nodes, *mpolmask, *newmask);
161268860ec1SPaul Jackson 		pol->v.nodes = tmp;
161374cb2155SPaul Jackson 		*mpolmask = *newmask;
161474cb2155SPaul Jackson 		current->il_next = node_remap(current->il_next,
161574cb2155SPaul Jackson 						*mpolmask, *newmask);
161668860ec1SPaul Jackson 		break;
161768860ec1SPaul Jackson 	case MPOL_PREFERRED:
161868860ec1SPaul Jackson 		pol->v.preferred_node = node_remap(pol->v.preferred_node,
161974cb2155SPaul Jackson 						*mpolmask, *newmask);
162074cb2155SPaul Jackson 		*mpolmask = *newmask;
162168860ec1SPaul Jackson 		break;
162268860ec1SPaul Jackson 	case MPOL_BIND: {
162368860ec1SPaul Jackson 		nodemask_t nodes;
162468860ec1SPaul Jackson 		struct zone **z;
162568860ec1SPaul Jackson 		struct zonelist *zonelist;
162668860ec1SPaul Jackson 
162768860ec1SPaul Jackson 		nodes_clear(nodes);
162868860ec1SPaul Jackson 		for (z = pol->v.zonelist->zones; *z; z++)
162968860ec1SPaul Jackson 			node_set((*z)->zone_pgdat->node_id, nodes);
163074cb2155SPaul Jackson 		nodes_remap(tmp, nodes, *mpolmask, *newmask);
163168860ec1SPaul Jackson 		nodes = tmp;
163268860ec1SPaul Jackson 
163368860ec1SPaul Jackson 		zonelist = bind_zonelist(&nodes);
163468860ec1SPaul Jackson 
163568860ec1SPaul Jackson 		/* If no mem, then zonelist is NULL and we keep old zonelist.
163668860ec1SPaul Jackson 		 * If that old zonelist has no remaining mems_allowed nodes,
163768860ec1SPaul Jackson 		 * then zonelist_policy() will "FALL THROUGH" to MPOL_DEFAULT.
163868860ec1SPaul Jackson 		 */
163968860ec1SPaul Jackson 
164068860ec1SPaul Jackson 		if (zonelist) {
164168860ec1SPaul Jackson 			/* Good - got mem - substitute new zonelist */
164268860ec1SPaul Jackson 			kfree(pol->v.zonelist);
164368860ec1SPaul Jackson 			pol->v.zonelist = zonelist;
164468860ec1SPaul Jackson 		}
164574cb2155SPaul Jackson 		*mpolmask = *newmask;
164668860ec1SPaul Jackson 		break;
164768860ec1SPaul Jackson 	}
164868860ec1SPaul Jackson 	default:
164968860ec1SPaul Jackson 		BUG();
165068860ec1SPaul Jackson 		break;
165168860ec1SPaul Jackson 	}
165268860ec1SPaul Jackson }
165368860ec1SPaul Jackson 
165468860ec1SPaul Jackson /*
165574cb2155SPaul Jackson  * Wrapper for mpol_rebind_policy() that just requires task
165674cb2155SPaul Jackson  * pointer, and updates task mempolicy.
165768860ec1SPaul Jackson  */
165874cb2155SPaul Jackson 
165974cb2155SPaul Jackson void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new)
166068860ec1SPaul Jackson {
166174cb2155SPaul Jackson 	mpol_rebind_policy(tsk->mempolicy, new);
166268860ec1SPaul Jackson }
16631a75a6c8SChristoph Lameter 
16641a75a6c8SChristoph Lameter /*
16654225399aSPaul Jackson  * Rebind each vma in mm to new nodemask.
16664225399aSPaul Jackson  *
16674225399aSPaul Jackson  * Call holding a reference to mm.  Takes mm->mmap_sem during call.
16684225399aSPaul Jackson  */
16694225399aSPaul Jackson 
16704225399aSPaul Jackson void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
16714225399aSPaul Jackson {
16724225399aSPaul Jackson 	struct vm_area_struct *vma;
16734225399aSPaul Jackson 
16744225399aSPaul Jackson 	down_write(&mm->mmap_sem);
16754225399aSPaul Jackson 	for (vma = mm->mmap; vma; vma = vma->vm_next)
16764225399aSPaul Jackson 		mpol_rebind_policy(vma->vm_policy, new);
16774225399aSPaul Jackson 	up_write(&mm->mmap_sem);
16784225399aSPaul Jackson }
16794225399aSPaul Jackson 
16804225399aSPaul Jackson /*
16811a75a6c8SChristoph Lameter  * Display pages allocated per node and memory policy via /proc.
16821a75a6c8SChristoph Lameter  */
16831a75a6c8SChristoph Lameter 
16841a75a6c8SChristoph Lameter static const char *policy_types[] = { "default", "prefer", "bind",
16851a75a6c8SChristoph Lameter 				      "interleave" };
16861a75a6c8SChristoph Lameter 
16871a75a6c8SChristoph Lameter /*
16881a75a6c8SChristoph Lameter  * Convert a mempolicy into a string.
16891a75a6c8SChristoph Lameter  * Returns the number of characters in buffer (if positive)
16901a75a6c8SChristoph Lameter  * or an error (negative)
16911a75a6c8SChristoph Lameter  */
16921a75a6c8SChristoph Lameter static inline int mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)
16931a75a6c8SChristoph Lameter {
16941a75a6c8SChristoph Lameter 	char *p = buffer;
16951a75a6c8SChristoph Lameter 	int l;
16961a75a6c8SChristoph Lameter 	nodemask_t nodes;
16971a75a6c8SChristoph Lameter 	int mode = pol ? pol->policy : MPOL_DEFAULT;
16981a75a6c8SChristoph Lameter 
16991a75a6c8SChristoph Lameter 	switch (mode) {
17001a75a6c8SChristoph Lameter 	case MPOL_DEFAULT:
17011a75a6c8SChristoph Lameter 		nodes_clear(nodes);
17021a75a6c8SChristoph Lameter 		break;
17031a75a6c8SChristoph Lameter 
17041a75a6c8SChristoph Lameter 	case MPOL_PREFERRED:
17051a75a6c8SChristoph Lameter 		nodes_clear(nodes);
17061a75a6c8SChristoph Lameter 		node_set(pol->v.preferred_node, nodes);
17071a75a6c8SChristoph Lameter 		break;
17081a75a6c8SChristoph Lameter 
17091a75a6c8SChristoph Lameter 	case MPOL_BIND:
17101a75a6c8SChristoph Lameter 		get_zonemask(pol, &nodes);
17111a75a6c8SChristoph Lameter 		break;
17121a75a6c8SChristoph Lameter 
17131a75a6c8SChristoph Lameter 	case MPOL_INTERLEAVE:
17141a75a6c8SChristoph Lameter 		nodes = pol->v.nodes;
17151a75a6c8SChristoph Lameter 		break;
17161a75a6c8SChristoph Lameter 
17171a75a6c8SChristoph Lameter 	default:
17181a75a6c8SChristoph Lameter 		BUG();
17191a75a6c8SChristoph Lameter 		return -EFAULT;
17201a75a6c8SChristoph Lameter 	}
17211a75a6c8SChristoph Lameter 
17221a75a6c8SChristoph Lameter 	l = strlen(policy_types[mode]);
17231a75a6c8SChristoph Lameter  	if (buffer + maxlen < p + l + 1)
17241a75a6c8SChristoph Lameter  		return -ENOSPC;
17251a75a6c8SChristoph Lameter 
17261a75a6c8SChristoph Lameter 	strcpy(p, policy_types[mode]);
17271a75a6c8SChristoph Lameter 	p += l;
17281a75a6c8SChristoph Lameter 
17291a75a6c8SChristoph Lameter 	if (!nodes_empty(nodes)) {
17301a75a6c8SChristoph Lameter 		if (buffer + maxlen < p + 2)
17311a75a6c8SChristoph Lameter 			return -ENOSPC;
17321a75a6c8SChristoph Lameter 		*p++ = '=';
17331a75a6c8SChristoph Lameter 	 	p += nodelist_scnprintf(p, buffer + maxlen - p, nodes);
17341a75a6c8SChristoph Lameter 	}
17351a75a6c8SChristoph Lameter 	return p - buffer;
17361a75a6c8SChristoph Lameter }
17371a75a6c8SChristoph Lameter 
17381a75a6c8SChristoph Lameter struct numa_maps {
17391a75a6c8SChristoph Lameter 	unsigned long pages;
17401a75a6c8SChristoph Lameter 	unsigned long anon;
17411a75a6c8SChristoph Lameter 	unsigned long mapped;
17421a75a6c8SChristoph Lameter 	unsigned long mapcount_max;
17431a75a6c8SChristoph Lameter 	unsigned long node[MAX_NUMNODES];
17441a75a6c8SChristoph Lameter };
17451a75a6c8SChristoph Lameter 
17461a75a6c8SChristoph Lameter static void gather_stats(struct page *page, void *private)
17471a75a6c8SChristoph Lameter {
17481a75a6c8SChristoph Lameter 	struct numa_maps *md = private;
17491a75a6c8SChristoph Lameter 	int count = page_mapcount(page);
17501a75a6c8SChristoph Lameter 
17511a75a6c8SChristoph Lameter 	if (count)
17521a75a6c8SChristoph Lameter 		md->mapped++;
17531a75a6c8SChristoph Lameter 
17541a75a6c8SChristoph Lameter 	if (count > md->mapcount_max)
17551a75a6c8SChristoph Lameter 		md->mapcount_max = count;
17561a75a6c8SChristoph Lameter 
17571a75a6c8SChristoph Lameter 	md->pages++;
17581a75a6c8SChristoph Lameter 
17591a75a6c8SChristoph Lameter 	if (PageAnon(page))
17601a75a6c8SChristoph Lameter 		md->anon++;
17611a75a6c8SChristoph Lameter 
17621a75a6c8SChristoph Lameter 	md->node[page_to_nid(page)]++;
17631a75a6c8SChristoph Lameter 	cond_resched();
17641a75a6c8SChristoph Lameter }
17651a75a6c8SChristoph Lameter 
17661a75a6c8SChristoph Lameter int show_numa_map(struct seq_file *m, void *v)
17671a75a6c8SChristoph Lameter {
17681a75a6c8SChristoph Lameter 	struct task_struct *task = m->private;
17691a75a6c8SChristoph Lameter 	struct vm_area_struct *vma = v;
17701a75a6c8SChristoph Lameter 	struct numa_maps *md;
17711a75a6c8SChristoph Lameter 	int n;
17721a75a6c8SChristoph Lameter 	char buffer[50];
17731a75a6c8SChristoph Lameter 
17741a75a6c8SChristoph Lameter 	if (!vma->vm_mm)
17751a75a6c8SChristoph Lameter 		return 0;
17761a75a6c8SChristoph Lameter 
17771a75a6c8SChristoph Lameter 	md = kzalloc(sizeof(struct numa_maps), GFP_KERNEL);
17781a75a6c8SChristoph Lameter 	if (!md)
17791a75a6c8SChristoph Lameter 		return 0;
17801a75a6c8SChristoph Lameter 
17811a75a6c8SChristoph Lameter 	check_pgd_range(vma, vma->vm_start, vma->vm_end,
17821a75a6c8SChristoph Lameter 		    &node_online_map, MPOL_MF_STATS, md);
17831a75a6c8SChristoph Lameter 
17841a75a6c8SChristoph Lameter 	if (md->pages) {
17851a75a6c8SChristoph Lameter 		mpol_to_str(buffer, sizeof(buffer),
17861a75a6c8SChristoph Lameter 			    get_vma_policy(task, vma, vma->vm_start));
17871a75a6c8SChristoph Lameter 
17881a75a6c8SChristoph Lameter 		seq_printf(m, "%08lx %s pages=%lu mapped=%lu maxref=%lu",
17891a75a6c8SChristoph Lameter 			   vma->vm_start, buffer, md->pages,
17901a75a6c8SChristoph Lameter 			   md->mapped, md->mapcount_max);
17911a75a6c8SChristoph Lameter 
17921a75a6c8SChristoph Lameter 		if (md->anon)
17931a75a6c8SChristoph Lameter 			seq_printf(m," anon=%lu",md->anon);
17941a75a6c8SChristoph Lameter 
17951a75a6c8SChristoph Lameter 		for_each_online_node(n)
17961a75a6c8SChristoph Lameter 			if (md->node[n])
17971a75a6c8SChristoph Lameter 				seq_printf(m, " N%d=%lu", n, md->node[n]);
17981a75a6c8SChristoph Lameter 
17991a75a6c8SChristoph Lameter 		seq_putc(m, '\n');
18001a75a6c8SChristoph Lameter 	}
18011a75a6c8SChristoph Lameter 	kfree(md);
18021a75a6c8SChristoph Lameter 
18031a75a6c8SChristoph Lameter 	if (m->count < m->size)
18041a75a6c8SChristoph Lameter 		m->version = (vma != get_gate_vma(task)) ? vma->vm_start : 0;
18051a75a6c8SChristoph Lameter 	return 0;
18061a75a6c8SChristoph Lameter }
18071a75a6c8SChristoph Lameter 
1808