xref: /openbmc/linux/mm/mempolicy.c (revision f15ca78e33b0bb5acc0c5d9a5d5be3c55c4f0bb7)
11da177e4SLinus Torvalds /*
21da177e4SLinus Torvalds  * Simple NUMA memory policy for the Linux kernel.
31da177e4SLinus Torvalds  *
41da177e4SLinus Torvalds  * Copyright 2003,2004 Andi Kleen, SuSE Labs.
58bccd85fSChristoph Lameter  * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
61da177e4SLinus Torvalds  * Subject to the GNU Public License, version 2.
71da177e4SLinus Torvalds  *
81da177e4SLinus Torvalds  * NUMA policy allows the user to give hints in which node(s) memory should
91da177e4SLinus Torvalds  * be allocated.
101da177e4SLinus Torvalds  *
111da177e4SLinus Torvalds  * Support four policies per VMA and per process:
121da177e4SLinus Torvalds  *
131da177e4SLinus Torvalds  * The VMA policy has priority over the process policy for a page fault.
141da177e4SLinus Torvalds  *
151da177e4SLinus Torvalds  * interleave     Allocate memory interleaved over a set of nodes,
161da177e4SLinus Torvalds  *                with normal fallback if it fails.
171da177e4SLinus Torvalds  *                For VMA based allocations this interleaves based on the
181da177e4SLinus Torvalds  *                offset into the backing object or offset into the mapping
191da177e4SLinus Torvalds  *                for anonymous memory. For process policy an process counter
201da177e4SLinus Torvalds  *                is used.
218bccd85fSChristoph Lameter  *
221da177e4SLinus Torvalds  * bind           Only allocate memory on a specific set of nodes,
231da177e4SLinus Torvalds  *                no fallback.
248bccd85fSChristoph Lameter  *                FIXME: memory is allocated starting with the first node
258bccd85fSChristoph Lameter  *                to the last. It would be better if bind would truly restrict
268bccd85fSChristoph Lameter  *                the allocation to memory nodes instead
278bccd85fSChristoph Lameter  *
281da177e4SLinus Torvalds  * preferred       Try a specific node first before normal fallback.
2900ef2d2fSDavid Rientjes  *                As a special case NUMA_NO_NODE here means do the allocation
301da177e4SLinus Torvalds  *                on the local CPU. This is normally identical to default,
311da177e4SLinus Torvalds  *                but useful to set in a VMA when you have a non default
321da177e4SLinus Torvalds  *                process policy.
338bccd85fSChristoph Lameter  *
341da177e4SLinus Torvalds  * default        Allocate on the local node first, or when on a VMA
351da177e4SLinus Torvalds  *                use the process policy. This is what Linux always did
361da177e4SLinus Torvalds  *		  in a NUMA aware kernel and still does by, ahem, default.
371da177e4SLinus Torvalds  *
381da177e4SLinus Torvalds  * The process policy is applied for most non interrupt memory allocations
391da177e4SLinus Torvalds  * in that process' context. Interrupts ignore the policies and always
401da177e4SLinus Torvalds  * try to allocate on the local CPU. The VMA policy is only applied for memory
411da177e4SLinus Torvalds  * allocations for a VMA in the VM.
421da177e4SLinus Torvalds  *
431da177e4SLinus Torvalds  * Currently there are a few corner cases in swapping where the policy
441da177e4SLinus Torvalds  * is not applied, but the majority should be handled. When process policy
451da177e4SLinus Torvalds  * is used it is not remembered over swap outs/swap ins.
461da177e4SLinus Torvalds  *
471da177e4SLinus Torvalds  * Only the highest zone in the zone hierarchy gets policied. Allocations
481da177e4SLinus Torvalds  * requesting a lower zone just use default policy. This implies that
491da177e4SLinus Torvalds  * on systems with highmem kernel lowmem allocation don't get policied.
501da177e4SLinus Torvalds  * Same with GFP_DMA allocations.
511da177e4SLinus Torvalds  *
521da177e4SLinus Torvalds  * For shmfs/tmpfs/hugetlbfs shared memory the policy is shared between
531da177e4SLinus Torvalds  * all users and remembered even when nobody has memory mapped.
541da177e4SLinus Torvalds  */
551da177e4SLinus Torvalds 
561da177e4SLinus Torvalds /* Notebook:
571da177e4SLinus Torvalds    fix mmap readahead to honour policy and enable policy for any page cache
581da177e4SLinus Torvalds    object
591da177e4SLinus Torvalds    statistics for bigpages
601da177e4SLinus Torvalds    global policy for page cache? currently it uses process policy. Requires
611da177e4SLinus Torvalds    first item above.
621da177e4SLinus Torvalds    handle mremap for shared memory (currently ignored for the policy)
631da177e4SLinus Torvalds    grows down?
641da177e4SLinus Torvalds    make bind policy root only? It can trigger oom much faster and the
651da177e4SLinus Torvalds    kernel is not always grateful with that.
661da177e4SLinus Torvalds */
671da177e4SLinus Torvalds 
68b1de0d13SMitchel Humpherys #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
69b1de0d13SMitchel Humpherys 
701da177e4SLinus Torvalds #include <linux/mempolicy.h>
711da177e4SLinus Torvalds #include <linux/mm.h>
721da177e4SLinus Torvalds #include <linux/highmem.h>
731da177e4SLinus Torvalds #include <linux/hugetlb.h>
741da177e4SLinus Torvalds #include <linux/kernel.h>
751da177e4SLinus Torvalds #include <linux/sched.h>
761da177e4SLinus Torvalds #include <linux/nodemask.h>
771da177e4SLinus Torvalds #include <linux/cpuset.h>
781da177e4SLinus Torvalds #include <linux/slab.h>
791da177e4SLinus Torvalds #include <linux/string.h>
80b95f1b31SPaul Gortmaker #include <linux/export.h>
81b488893aSPavel Emelyanov #include <linux/nsproxy.h>
821da177e4SLinus Torvalds #include <linux/interrupt.h>
831da177e4SLinus Torvalds #include <linux/init.h>
841da177e4SLinus Torvalds #include <linux/compat.h>
85dc9aa5b9SChristoph Lameter #include <linux/swap.h>
861a75a6c8SChristoph Lameter #include <linux/seq_file.h>
871a75a6c8SChristoph Lameter #include <linux/proc_fs.h>
88b20a3503SChristoph Lameter #include <linux/migrate.h>
8962b61f61SHugh Dickins #include <linux/ksm.h>
9095a402c3SChristoph Lameter #include <linux/rmap.h>
9186c3a764SDavid Quigley #include <linux/security.h>
92dbcb0f19SAdrian Bunk #include <linux/syscalls.h>
93095f1fc4SLee Schermerhorn #include <linux/ctype.h>
946d9c285aSKOSAKI Motohiro #include <linux/mm_inline.h>
95b24f53a0SLee Schermerhorn #include <linux/mmu_notifier.h>
96b1de0d13SMitchel Humpherys #include <linux/printk.h>
97dc9aa5b9SChristoph Lameter 
981da177e4SLinus Torvalds #include <asm/tlbflush.h>
991da177e4SLinus Torvalds #include <asm/uaccess.h>
100778d3b0fSMichal Hocko #include <linux/random.h>
1011da177e4SLinus Torvalds 
10262695a84SNick Piggin #include "internal.h"
10362695a84SNick Piggin 
10438e35860SChristoph Lameter /* Internal flags */
105dc9aa5b9SChristoph Lameter #define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0)	/* Skip checks for continuous vmas */
10638e35860SChristoph Lameter #define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1)		/* Invert check for nodemask */
107dc9aa5b9SChristoph Lameter 
108fcc234f8SPekka Enberg static struct kmem_cache *policy_cache;
109fcc234f8SPekka Enberg static struct kmem_cache *sn_cache;
1101da177e4SLinus Torvalds 
1111da177e4SLinus Torvalds /* Highest zone. An specific allocation for a zone below that is not
1121da177e4SLinus Torvalds    policied. */
1136267276fSChristoph Lameter enum zone_type policy_zone = 0;
1141da177e4SLinus Torvalds 
115bea904d5SLee Schermerhorn /*
116bea904d5SLee Schermerhorn  * run-time system-wide default policy => local allocation
117bea904d5SLee Schermerhorn  */
118e754d79dSH Hartley Sweeten static struct mempolicy default_policy = {
1191da177e4SLinus Torvalds 	.refcnt = ATOMIC_INIT(1), /* never free it */
120bea904d5SLee Schermerhorn 	.mode = MPOL_PREFERRED,
121fc36b8d3SLee Schermerhorn 	.flags = MPOL_F_LOCAL,
1221da177e4SLinus Torvalds };
1231da177e4SLinus Torvalds 
1245606e387SMel Gorman static struct mempolicy preferred_node_policy[MAX_NUMNODES];
1255606e387SMel Gorman 
1265606e387SMel Gorman static struct mempolicy *get_task_policy(struct task_struct *p)
1275606e387SMel Gorman {
1285606e387SMel Gorman 	struct mempolicy *pol = p->mempolicy;
129*f15ca78eSOleg Nesterov 	int node;
1305606e387SMel Gorman 
131*f15ca78eSOleg Nesterov 	if (pol)
132*f15ca78eSOleg Nesterov 		return pol;
1335606e387SMel Gorman 
134*f15ca78eSOleg Nesterov 	node = numa_node_id();
1351da6f0e1SJianguo Wu 	if (node != NUMA_NO_NODE) {
1361da6f0e1SJianguo Wu 		pol = &preferred_node_policy[node];
137*f15ca78eSOleg Nesterov 		/* preferred_node_policy is not initialised early in boot */
138*f15ca78eSOleg Nesterov 		if (pol->mode)
139*f15ca78eSOleg Nesterov 			return pol;
1401da6f0e1SJianguo Wu 	}
1415606e387SMel Gorman 
142*f15ca78eSOleg Nesterov 	return &default_policy;
1435606e387SMel Gorman }
1445606e387SMel Gorman 
14537012946SDavid Rientjes static const struct mempolicy_operations {
14637012946SDavid Rientjes 	int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
147708c1bbcSMiao Xie 	/*
148708c1bbcSMiao Xie 	 * If read-side task has no lock to protect task->mempolicy, write-side
149708c1bbcSMiao Xie 	 * task will rebind the task->mempolicy by two step. The first step is
150708c1bbcSMiao Xie 	 * setting all the newly nodes, and the second step is cleaning all the
151708c1bbcSMiao Xie 	 * disallowed nodes. In this way, we can avoid finding no node to alloc
152708c1bbcSMiao Xie 	 * page.
153708c1bbcSMiao Xie 	 * If we have a lock to protect task->mempolicy in read-side, we do
154708c1bbcSMiao Xie 	 * rebind directly.
155708c1bbcSMiao Xie 	 *
156708c1bbcSMiao Xie 	 * step:
157708c1bbcSMiao Xie 	 * 	MPOL_REBIND_ONCE - do rebind work at once
158708c1bbcSMiao Xie 	 * 	MPOL_REBIND_STEP1 - set all the newly nodes
159708c1bbcSMiao Xie 	 * 	MPOL_REBIND_STEP2 - clean all the disallowed nodes
160708c1bbcSMiao Xie 	 */
161708c1bbcSMiao Xie 	void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes,
162708c1bbcSMiao Xie 			enum mpol_rebind_step step);
16337012946SDavid Rientjes } mpol_ops[MPOL_MAX];
16437012946SDavid Rientjes 
16519770b32SMel Gorman /* Check that the nodemask contains at least one populated zone */
16637012946SDavid Rientjes static int is_valid_nodemask(const nodemask_t *nodemask)
1671da177e4SLinus Torvalds {
168d3eb1570SLai Jiangshan 	return nodes_intersects(*nodemask, node_states[N_MEMORY]);
1691da177e4SLinus Torvalds }
1701da177e4SLinus Torvalds 
171f5b087b5SDavid Rientjes static inline int mpol_store_user_nodemask(const struct mempolicy *pol)
172f5b087b5SDavid Rientjes {
1736d556294SBob Liu 	return pol->flags & MPOL_MODE_FLAGS;
1744c50bc01SDavid Rientjes }
1754c50bc01SDavid Rientjes 
1764c50bc01SDavid Rientjes static void mpol_relative_nodemask(nodemask_t *ret, const nodemask_t *orig,
1774c50bc01SDavid Rientjes 				   const nodemask_t *rel)
1784c50bc01SDavid Rientjes {
1794c50bc01SDavid Rientjes 	nodemask_t tmp;
1804c50bc01SDavid Rientjes 	nodes_fold(tmp, *orig, nodes_weight(*rel));
1814c50bc01SDavid Rientjes 	nodes_onto(*ret, tmp, *rel);
182f5b087b5SDavid Rientjes }
183f5b087b5SDavid Rientjes 
18437012946SDavid Rientjes static int mpol_new_interleave(struct mempolicy *pol, const nodemask_t *nodes)
18537012946SDavid Rientjes {
18637012946SDavid Rientjes 	if (nodes_empty(*nodes))
18737012946SDavid Rientjes 		return -EINVAL;
18837012946SDavid Rientjes 	pol->v.nodes = *nodes;
18937012946SDavid Rientjes 	return 0;
19037012946SDavid Rientjes }
19137012946SDavid Rientjes 
19237012946SDavid Rientjes static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes)
19337012946SDavid Rientjes {
19437012946SDavid Rientjes 	if (!nodes)
195fc36b8d3SLee Schermerhorn 		pol->flags |= MPOL_F_LOCAL;	/* local allocation */
19637012946SDavid Rientjes 	else if (nodes_empty(*nodes))
19737012946SDavid Rientjes 		return -EINVAL;			/*  no allowed nodes */
19837012946SDavid Rientjes 	else
19937012946SDavid Rientjes 		pol->v.preferred_node = first_node(*nodes);
20037012946SDavid Rientjes 	return 0;
20137012946SDavid Rientjes }
20237012946SDavid Rientjes 
20337012946SDavid Rientjes static int mpol_new_bind(struct mempolicy *pol, const nodemask_t *nodes)
20437012946SDavid Rientjes {
20537012946SDavid Rientjes 	if (!is_valid_nodemask(nodes))
20637012946SDavid Rientjes 		return -EINVAL;
20737012946SDavid Rientjes 	pol->v.nodes = *nodes;
20837012946SDavid Rientjes 	return 0;
20937012946SDavid Rientjes }
21037012946SDavid Rientjes 
21158568d2aSMiao Xie /*
21258568d2aSMiao Xie  * mpol_set_nodemask is called after mpol_new() to set up the nodemask, if
21358568d2aSMiao Xie  * any, for the new policy.  mpol_new() has already validated the nodes
21458568d2aSMiao Xie  * parameter with respect to the policy mode and flags.  But, we need to
21558568d2aSMiao Xie  * handle an empty nodemask with MPOL_PREFERRED here.
21658568d2aSMiao Xie  *
21758568d2aSMiao Xie  * Must be called holding task's alloc_lock to protect task's mems_allowed
21858568d2aSMiao Xie  * and mempolicy.  May also be called holding the mmap_semaphore for write.
21958568d2aSMiao Xie  */
2204bfc4495SKAMEZAWA Hiroyuki static int mpol_set_nodemask(struct mempolicy *pol,
2214bfc4495SKAMEZAWA Hiroyuki 		     const nodemask_t *nodes, struct nodemask_scratch *nsc)
22258568d2aSMiao Xie {
22358568d2aSMiao Xie 	int ret;
22458568d2aSMiao Xie 
22558568d2aSMiao Xie 	/* if mode is MPOL_DEFAULT, pol is NULL. This is right. */
22658568d2aSMiao Xie 	if (pol == NULL)
22758568d2aSMiao Xie 		return 0;
22801f13bd6SLai Jiangshan 	/* Check N_MEMORY */
2294bfc4495SKAMEZAWA Hiroyuki 	nodes_and(nsc->mask1,
23001f13bd6SLai Jiangshan 		  cpuset_current_mems_allowed, node_states[N_MEMORY]);
23158568d2aSMiao Xie 
23258568d2aSMiao Xie 	VM_BUG_ON(!nodes);
23358568d2aSMiao Xie 	if (pol->mode == MPOL_PREFERRED && nodes_empty(*nodes))
23458568d2aSMiao Xie 		nodes = NULL;	/* explicit local allocation */
23558568d2aSMiao Xie 	else {
23658568d2aSMiao Xie 		if (pol->flags & MPOL_F_RELATIVE_NODES)
2374bfc4495SKAMEZAWA Hiroyuki 			mpol_relative_nodemask(&nsc->mask2, nodes,&nsc->mask1);
23858568d2aSMiao Xie 		else
2394bfc4495SKAMEZAWA Hiroyuki 			nodes_and(nsc->mask2, *nodes, nsc->mask1);
2404bfc4495SKAMEZAWA Hiroyuki 
24158568d2aSMiao Xie 		if (mpol_store_user_nodemask(pol))
24258568d2aSMiao Xie 			pol->w.user_nodemask = *nodes;
24358568d2aSMiao Xie 		else
24458568d2aSMiao Xie 			pol->w.cpuset_mems_allowed =
24558568d2aSMiao Xie 						cpuset_current_mems_allowed;
24658568d2aSMiao Xie 	}
24758568d2aSMiao Xie 
2484bfc4495SKAMEZAWA Hiroyuki 	if (nodes)
2494bfc4495SKAMEZAWA Hiroyuki 		ret = mpol_ops[pol->mode].create(pol, &nsc->mask2);
2504bfc4495SKAMEZAWA Hiroyuki 	else
2514bfc4495SKAMEZAWA Hiroyuki 		ret = mpol_ops[pol->mode].create(pol, NULL);
25258568d2aSMiao Xie 	return ret;
25358568d2aSMiao Xie }
25458568d2aSMiao Xie 
25558568d2aSMiao Xie /*
25658568d2aSMiao Xie  * This function just creates a new policy, does some check and simple
25758568d2aSMiao Xie  * initialization. You must invoke mpol_set_nodemask() to set nodes.
25858568d2aSMiao Xie  */
259028fec41SDavid Rientjes static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags,
260028fec41SDavid Rientjes 				  nodemask_t *nodes)
2611da177e4SLinus Torvalds {
2621da177e4SLinus Torvalds 	struct mempolicy *policy;
2631da177e4SLinus Torvalds 
264028fec41SDavid Rientjes 	pr_debug("setting mode %d flags %d nodes[0] %lx\n",
26500ef2d2fSDavid Rientjes 		 mode, flags, nodes ? nodes_addr(*nodes)[0] : NUMA_NO_NODE);
266140d5a49SPaul Mundt 
2673e1f0645SDavid Rientjes 	if (mode == MPOL_DEFAULT) {
2683e1f0645SDavid Rientjes 		if (nodes && !nodes_empty(*nodes))
26937012946SDavid Rientjes 			return ERR_PTR(-EINVAL);
270d3a71033SLee Schermerhorn 		return NULL;
27137012946SDavid Rientjes 	}
2723e1f0645SDavid Rientjes 	VM_BUG_ON(!nodes);
2733e1f0645SDavid Rientjes 
2743e1f0645SDavid Rientjes 	/*
2753e1f0645SDavid Rientjes 	 * MPOL_PREFERRED cannot be used with MPOL_F_STATIC_NODES or
2763e1f0645SDavid Rientjes 	 * MPOL_F_RELATIVE_NODES if the nodemask is empty (local allocation).
2773e1f0645SDavid Rientjes 	 * All other modes require a valid pointer to a non-empty nodemask.
2783e1f0645SDavid Rientjes 	 */
2793e1f0645SDavid Rientjes 	if (mode == MPOL_PREFERRED) {
2803e1f0645SDavid Rientjes 		if (nodes_empty(*nodes)) {
2813e1f0645SDavid Rientjes 			if (((flags & MPOL_F_STATIC_NODES) ||
2823e1f0645SDavid Rientjes 			     (flags & MPOL_F_RELATIVE_NODES)))
2833e1f0645SDavid Rientjes 				return ERR_PTR(-EINVAL);
2843e1f0645SDavid Rientjes 		}
285479e2802SPeter Zijlstra 	} else if (mode == MPOL_LOCAL) {
286479e2802SPeter Zijlstra 		if (!nodes_empty(*nodes))
287479e2802SPeter Zijlstra 			return ERR_PTR(-EINVAL);
288479e2802SPeter Zijlstra 		mode = MPOL_PREFERRED;
2893e1f0645SDavid Rientjes 	} else if (nodes_empty(*nodes))
2903e1f0645SDavid Rientjes 		return ERR_PTR(-EINVAL);
2911da177e4SLinus Torvalds 	policy = kmem_cache_alloc(policy_cache, GFP_KERNEL);
2921da177e4SLinus Torvalds 	if (!policy)
2931da177e4SLinus Torvalds 		return ERR_PTR(-ENOMEM);
2941da177e4SLinus Torvalds 	atomic_set(&policy->refcnt, 1);
29545c4745aSLee Schermerhorn 	policy->mode = mode;
29637012946SDavid Rientjes 	policy->flags = flags;
2973e1f0645SDavid Rientjes 
29837012946SDavid Rientjes 	return policy;
29937012946SDavid Rientjes }
30037012946SDavid Rientjes 
30152cd3b07SLee Schermerhorn /* Slow path of a mpol destructor. */
30252cd3b07SLee Schermerhorn void __mpol_put(struct mempolicy *p)
30352cd3b07SLee Schermerhorn {
30452cd3b07SLee Schermerhorn 	if (!atomic_dec_and_test(&p->refcnt))
30552cd3b07SLee Schermerhorn 		return;
30652cd3b07SLee Schermerhorn 	kmem_cache_free(policy_cache, p);
30752cd3b07SLee Schermerhorn }
30852cd3b07SLee Schermerhorn 
309708c1bbcSMiao Xie static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes,
310708c1bbcSMiao Xie 				enum mpol_rebind_step step)
31137012946SDavid Rientjes {
31237012946SDavid Rientjes }
31337012946SDavid Rientjes 
314708c1bbcSMiao Xie /*
315708c1bbcSMiao Xie  * step:
316708c1bbcSMiao Xie  * 	MPOL_REBIND_ONCE  - do rebind work at once
317708c1bbcSMiao Xie  * 	MPOL_REBIND_STEP1 - set all the newly nodes
318708c1bbcSMiao Xie  * 	MPOL_REBIND_STEP2 - clean all the disallowed nodes
319708c1bbcSMiao Xie  */
320708c1bbcSMiao Xie static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes,
321708c1bbcSMiao Xie 				 enum mpol_rebind_step step)
3221d0d2680SDavid Rientjes {
3231d0d2680SDavid Rientjes 	nodemask_t tmp;
3241d0d2680SDavid Rientjes 
32537012946SDavid Rientjes 	if (pol->flags & MPOL_F_STATIC_NODES)
32637012946SDavid Rientjes 		nodes_and(tmp, pol->w.user_nodemask, *nodes);
32737012946SDavid Rientjes 	else if (pol->flags & MPOL_F_RELATIVE_NODES)
32837012946SDavid Rientjes 		mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
3291d0d2680SDavid Rientjes 	else {
330708c1bbcSMiao Xie 		/*
331708c1bbcSMiao Xie 		 * if step == 1, we use ->w.cpuset_mems_allowed to cache the
332708c1bbcSMiao Xie 		 * result
333708c1bbcSMiao Xie 		 */
334708c1bbcSMiao Xie 		if (step == MPOL_REBIND_ONCE || step == MPOL_REBIND_STEP1) {
335708c1bbcSMiao Xie 			nodes_remap(tmp, pol->v.nodes,
336708c1bbcSMiao Xie 					pol->w.cpuset_mems_allowed, *nodes);
337708c1bbcSMiao Xie 			pol->w.cpuset_mems_allowed = step ? tmp : *nodes;
338708c1bbcSMiao Xie 		} else if (step == MPOL_REBIND_STEP2) {
339708c1bbcSMiao Xie 			tmp = pol->w.cpuset_mems_allowed;
34037012946SDavid Rientjes 			pol->w.cpuset_mems_allowed = *nodes;
341708c1bbcSMiao Xie 		} else
342708c1bbcSMiao Xie 			BUG();
3431d0d2680SDavid Rientjes 	}
34437012946SDavid Rientjes 
345708c1bbcSMiao Xie 	if (nodes_empty(tmp))
346708c1bbcSMiao Xie 		tmp = *nodes;
347708c1bbcSMiao Xie 
348708c1bbcSMiao Xie 	if (step == MPOL_REBIND_STEP1)
349708c1bbcSMiao Xie 		nodes_or(pol->v.nodes, pol->v.nodes, tmp);
350708c1bbcSMiao Xie 	else if (step == MPOL_REBIND_ONCE || step == MPOL_REBIND_STEP2)
3511d0d2680SDavid Rientjes 		pol->v.nodes = tmp;
352708c1bbcSMiao Xie 	else
353708c1bbcSMiao Xie 		BUG();
354708c1bbcSMiao Xie 
3551d0d2680SDavid Rientjes 	if (!node_isset(current->il_next, tmp)) {
3561d0d2680SDavid Rientjes 		current->il_next = next_node(current->il_next, tmp);
3571d0d2680SDavid Rientjes 		if (current->il_next >= MAX_NUMNODES)
3581d0d2680SDavid Rientjes 			current->il_next = first_node(tmp);
3591d0d2680SDavid Rientjes 		if (current->il_next >= MAX_NUMNODES)
3601d0d2680SDavid Rientjes 			current->il_next = numa_node_id();
3611d0d2680SDavid Rientjes 	}
36237012946SDavid Rientjes }
36337012946SDavid Rientjes 
36437012946SDavid Rientjes static void mpol_rebind_preferred(struct mempolicy *pol,
365708c1bbcSMiao Xie 				  const nodemask_t *nodes,
366708c1bbcSMiao Xie 				  enum mpol_rebind_step step)
36737012946SDavid Rientjes {
36837012946SDavid Rientjes 	nodemask_t tmp;
36937012946SDavid Rientjes 
37037012946SDavid Rientjes 	if (pol->flags & MPOL_F_STATIC_NODES) {
3711d0d2680SDavid Rientjes 		int node = first_node(pol->w.user_nodemask);
3721d0d2680SDavid Rientjes 
373fc36b8d3SLee Schermerhorn 		if (node_isset(node, *nodes)) {
3741d0d2680SDavid Rientjes 			pol->v.preferred_node = node;
375fc36b8d3SLee Schermerhorn 			pol->flags &= ~MPOL_F_LOCAL;
376fc36b8d3SLee Schermerhorn 		} else
377fc36b8d3SLee Schermerhorn 			pol->flags |= MPOL_F_LOCAL;
37837012946SDavid Rientjes 	} else if (pol->flags & MPOL_F_RELATIVE_NODES) {
37937012946SDavid Rientjes 		mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
3801d0d2680SDavid Rientjes 		pol->v.preferred_node = first_node(tmp);
381fc36b8d3SLee Schermerhorn 	} else if (!(pol->flags & MPOL_F_LOCAL)) {
3821d0d2680SDavid Rientjes 		pol->v.preferred_node = node_remap(pol->v.preferred_node,
38337012946SDavid Rientjes 						   pol->w.cpuset_mems_allowed,
38437012946SDavid Rientjes 						   *nodes);
38537012946SDavid Rientjes 		pol->w.cpuset_mems_allowed = *nodes;
3861d0d2680SDavid Rientjes 	}
3871d0d2680SDavid Rientjes }
38837012946SDavid Rientjes 
389708c1bbcSMiao Xie /*
390708c1bbcSMiao Xie  * mpol_rebind_policy - Migrate a policy to a different set of nodes
391708c1bbcSMiao Xie  *
392708c1bbcSMiao Xie  * If read-side task has no lock to protect task->mempolicy, write-side
393708c1bbcSMiao Xie  * task will rebind the task->mempolicy by two step. The first step is
394708c1bbcSMiao Xie  * setting all the newly nodes, and the second step is cleaning all the
395708c1bbcSMiao Xie  * disallowed nodes. In this way, we can avoid finding no node to alloc
396708c1bbcSMiao Xie  * page.
397708c1bbcSMiao Xie  * If we have a lock to protect task->mempolicy in read-side, we do
398708c1bbcSMiao Xie  * rebind directly.
399708c1bbcSMiao Xie  *
400708c1bbcSMiao Xie  * step:
401708c1bbcSMiao Xie  * 	MPOL_REBIND_ONCE  - do rebind work at once
402708c1bbcSMiao Xie  * 	MPOL_REBIND_STEP1 - set all the newly nodes
403708c1bbcSMiao Xie  * 	MPOL_REBIND_STEP2 - clean all the disallowed nodes
404708c1bbcSMiao Xie  */
405708c1bbcSMiao Xie static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask,
406708c1bbcSMiao Xie 				enum mpol_rebind_step step)
40737012946SDavid Rientjes {
40837012946SDavid Rientjes 	if (!pol)
40937012946SDavid Rientjes 		return;
41089c522c7SWang Sheng-Hui 	if (!mpol_store_user_nodemask(pol) && step == MPOL_REBIND_ONCE &&
41137012946SDavid Rientjes 	    nodes_equal(pol->w.cpuset_mems_allowed, *newmask))
41237012946SDavid Rientjes 		return;
413708c1bbcSMiao Xie 
414708c1bbcSMiao Xie 	if (step == MPOL_REBIND_STEP1 && (pol->flags & MPOL_F_REBINDING))
415708c1bbcSMiao Xie 		return;
416708c1bbcSMiao Xie 
417708c1bbcSMiao Xie 	if (step == MPOL_REBIND_STEP2 && !(pol->flags & MPOL_F_REBINDING))
418708c1bbcSMiao Xie 		BUG();
419708c1bbcSMiao Xie 
420708c1bbcSMiao Xie 	if (step == MPOL_REBIND_STEP1)
421708c1bbcSMiao Xie 		pol->flags |= MPOL_F_REBINDING;
422708c1bbcSMiao Xie 	else if (step == MPOL_REBIND_STEP2)
423708c1bbcSMiao Xie 		pol->flags &= ~MPOL_F_REBINDING;
424708c1bbcSMiao Xie 	else if (step >= MPOL_REBIND_NSTEP)
425708c1bbcSMiao Xie 		BUG();
426708c1bbcSMiao Xie 
427708c1bbcSMiao Xie 	mpol_ops[pol->mode].rebind(pol, newmask, step);
4281d0d2680SDavid Rientjes }
4291d0d2680SDavid Rientjes 
4301d0d2680SDavid Rientjes /*
4311d0d2680SDavid Rientjes  * Wrapper for mpol_rebind_policy() that just requires task
4321d0d2680SDavid Rientjes  * pointer, and updates task mempolicy.
43358568d2aSMiao Xie  *
43458568d2aSMiao Xie  * Called with task's alloc_lock held.
4351d0d2680SDavid Rientjes  */
4361d0d2680SDavid Rientjes 
437708c1bbcSMiao Xie void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new,
438708c1bbcSMiao Xie 			enum mpol_rebind_step step)
4391d0d2680SDavid Rientjes {
440708c1bbcSMiao Xie 	mpol_rebind_policy(tsk->mempolicy, new, step);
4411d0d2680SDavid Rientjes }
4421d0d2680SDavid Rientjes 
4431d0d2680SDavid Rientjes /*
4441d0d2680SDavid Rientjes  * Rebind each vma in mm to new nodemask.
4451d0d2680SDavid Rientjes  *
4461d0d2680SDavid Rientjes  * Call holding a reference to mm.  Takes mm->mmap_sem during call.
4471d0d2680SDavid Rientjes  */
4481d0d2680SDavid Rientjes 
4491d0d2680SDavid Rientjes void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
4501d0d2680SDavid Rientjes {
4511d0d2680SDavid Rientjes 	struct vm_area_struct *vma;
4521d0d2680SDavid Rientjes 
4531d0d2680SDavid Rientjes 	down_write(&mm->mmap_sem);
4541d0d2680SDavid Rientjes 	for (vma = mm->mmap; vma; vma = vma->vm_next)
455708c1bbcSMiao Xie 		mpol_rebind_policy(vma->vm_policy, new, MPOL_REBIND_ONCE);
4561d0d2680SDavid Rientjes 	up_write(&mm->mmap_sem);
4571d0d2680SDavid Rientjes }
4581d0d2680SDavid Rientjes 
45937012946SDavid Rientjes static const struct mempolicy_operations mpol_ops[MPOL_MAX] = {
46037012946SDavid Rientjes 	[MPOL_DEFAULT] = {
46137012946SDavid Rientjes 		.rebind = mpol_rebind_default,
46237012946SDavid Rientjes 	},
46337012946SDavid Rientjes 	[MPOL_INTERLEAVE] = {
46437012946SDavid Rientjes 		.create = mpol_new_interleave,
46537012946SDavid Rientjes 		.rebind = mpol_rebind_nodemask,
46637012946SDavid Rientjes 	},
46737012946SDavid Rientjes 	[MPOL_PREFERRED] = {
46837012946SDavid Rientjes 		.create = mpol_new_preferred,
46937012946SDavid Rientjes 		.rebind = mpol_rebind_preferred,
47037012946SDavid Rientjes 	},
47137012946SDavid Rientjes 	[MPOL_BIND] = {
47237012946SDavid Rientjes 		.create = mpol_new_bind,
47337012946SDavid Rientjes 		.rebind = mpol_rebind_nodemask,
47437012946SDavid Rientjes 	},
47537012946SDavid Rientjes };
47637012946SDavid Rientjes 
477fc301289SChristoph Lameter static void migrate_page_add(struct page *page, struct list_head *pagelist,
478fc301289SChristoph Lameter 				unsigned long flags);
4791a75a6c8SChristoph Lameter 
48098094945SNaoya Horiguchi /*
48198094945SNaoya Horiguchi  * Scan through pages checking if pages follow certain conditions,
48298094945SNaoya Horiguchi  * and move them to the pagelist if they do.
48398094945SNaoya Horiguchi  */
48498094945SNaoya Horiguchi static int queue_pages_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
485dc9aa5b9SChristoph Lameter 		unsigned long addr, unsigned long end,
486dc9aa5b9SChristoph Lameter 		const nodemask_t *nodes, unsigned long flags,
48738e35860SChristoph Lameter 		void *private)
4881da177e4SLinus Torvalds {
48991612e0dSHugh Dickins 	pte_t *orig_pte;
49091612e0dSHugh Dickins 	pte_t *pte;
491705e87c0SHugh Dickins 	spinlock_t *ptl;
492941150a3SHugh Dickins 
493705e87c0SHugh Dickins 	orig_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
49491612e0dSHugh Dickins 	do {
4956aab341eSLinus Torvalds 		struct page *page;
49625ba77c1SAndy Whitcroft 		int nid;
49791612e0dSHugh Dickins 
49891612e0dSHugh Dickins 		if (!pte_present(*pte))
49991612e0dSHugh Dickins 			continue;
5006aab341eSLinus Torvalds 		page = vm_normal_page(vma, addr, *pte);
5016aab341eSLinus Torvalds 		if (!page)
50291612e0dSHugh Dickins 			continue;
503053837fcSNick Piggin 		/*
50462b61f61SHugh Dickins 		 * vm_normal_page() filters out zero pages, but there might
50562b61f61SHugh Dickins 		 * still be PageReserved pages to skip, perhaps in a VDSO.
506053837fcSNick Piggin 		 */
507b79bc0a0SHugh Dickins 		if (PageReserved(page))
508f4598c8bSChristoph Lameter 			continue;
5096aab341eSLinus Torvalds 		nid = page_to_nid(page);
51038e35860SChristoph Lameter 		if (node_isset(nid, *nodes) == !!(flags & MPOL_MF_INVERT))
51138e35860SChristoph Lameter 			continue;
51238e35860SChristoph Lameter 
513b1f72d18SStephen Wilson 		if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
514fc301289SChristoph Lameter 			migrate_page_add(page, private, flags);
515dc9aa5b9SChristoph Lameter 		else
5161da177e4SLinus Torvalds 			break;
51791612e0dSHugh Dickins 	} while (pte++, addr += PAGE_SIZE, addr != end);
518705e87c0SHugh Dickins 	pte_unmap_unlock(orig_pte, ptl);
51991612e0dSHugh Dickins 	return addr != end;
52091612e0dSHugh Dickins }
52191612e0dSHugh Dickins 
52298094945SNaoya Horiguchi static void queue_pages_hugetlb_pmd_range(struct vm_area_struct *vma,
52398094945SNaoya Horiguchi 		pmd_t *pmd, const nodemask_t *nodes, unsigned long flags,
524e2d8cf40SNaoya Horiguchi 				    void *private)
525e2d8cf40SNaoya Horiguchi {
526e2d8cf40SNaoya Horiguchi #ifdef CONFIG_HUGETLB_PAGE
527e2d8cf40SNaoya Horiguchi 	int nid;
528e2d8cf40SNaoya Horiguchi 	struct page *page;
529cb900f41SKirill A. Shutemov 	spinlock_t *ptl;
530d4c54919SNaoya Horiguchi 	pte_t entry;
531e2d8cf40SNaoya Horiguchi 
532cb900f41SKirill A. Shutemov 	ptl = huge_pte_lock(hstate_vma(vma), vma->vm_mm, (pte_t *)pmd);
533d4c54919SNaoya Horiguchi 	entry = huge_ptep_get((pte_t *)pmd);
534d4c54919SNaoya Horiguchi 	if (!pte_present(entry))
535d4c54919SNaoya Horiguchi 		goto unlock;
536d4c54919SNaoya Horiguchi 	page = pte_page(entry);
537e2d8cf40SNaoya Horiguchi 	nid = page_to_nid(page);
538e2d8cf40SNaoya Horiguchi 	if (node_isset(nid, *nodes) == !!(flags & MPOL_MF_INVERT))
539e2d8cf40SNaoya Horiguchi 		goto unlock;
540e2d8cf40SNaoya Horiguchi 	/* With MPOL_MF_MOVE, we migrate only unshared hugepage. */
541e2d8cf40SNaoya Horiguchi 	if (flags & (MPOL_MF_MOVE_ALL) ||
542e2d8cf40SNaoya Horiguchi 	    (flags & MPOL_MF_MOVE && page_mapcount(page) == 1))
543e2d8cf40SNaoya Horiguchi 		isolate_huge_page(page, private);
544e2d8cf40SNaoya Horiguchi unlock:
545cb900f41SKirill A. Shutemov 	spin_unlock(ptl);
546e2d8cf40SNaoya Horiguchi #else
547e2d8cf40SNaoya Horiguchi 	BUG();
548e2d8cf40SNaoya Horiguchi #endif
549e2d8cf40SNaoya Horiguchi }
550e2d8cf40SNaoya Horiguchi 
55198094945SNaoya Horiguchi static inline int queue_pages_pmd_range(struct vm_area_struct *vma, pud_t *pud,
552dc9aa5b9SChristoph Lameter 		unsigned long addr, unsigned long end,
553dc9aa5b9SChristoph Lameter 		const nodemask_t *nodes, unsigned long flags,
55438e35860SChristoph Lameter 		void *private)
55591612e0dSHugh Dickins {
55691612e0dSHugh Dickins 	pmd_t *pmd;
55791612e0dSHugh Dickins 	unsigned long next;
55891612e0dSHugh Dickins 
55991612e0dSHugh Dickins 	pmd = pmd_offset(pud, addr);
56091612e0dSHugh Dickins 	do {
56191612e0dSHugh Dickins 		next = pmd_addr_end(addr, end);
562e2d8cf40SNaoya Horiguchi 		if (!pmd_present(*pmd))
563e2d8cf40SNaoya Horiguchi 			continue;
564e2d8cf40SNaoya Horiguchi 		if (pmd_huge(*pmd) && is_vm_hugetlb_page(vma)) {
56598094945SNaoya Horiguchi 			queue_pages_hugetlb_pmd_range(vma, pmd, nodes,
566e2d8cf40SNaoya Horiguchi 						flags, private);
567e2d8cf40SNaoya Horiguchi 			continue;
568e2d8cf40SNaoya Horiguchi 		}
569e180377fSKirill A. Shutemov 		split_huge_page_pmd(vma, addr, pmd);
5701a5a9906SAndrea Arcangeli 		if (pmd_none_or_trans_huge_or_clear_bad(pmd))
57191612e0dSHugh Dickins 			continue;
57298094945SNaoya Horiguchi 		if (queue_pages_pte_range(vma, pmd, addr, next, nodes,
57338e35860SChristoph Lameter 				    flags, private))
57491612e0dSHugh Dickins 			return -EIO;
57591612e0dSHugh Dickins 	} while (pmd++, addr = next, addr != end);
57691612e0dSHugh Dickins 	return 0;
57791612e0dSHugh Dickins }
57891612e0dSHugh Dickins 
57998094945SNaoya Horiguchi static inline int queue_pages_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
580dc9aa5b9SChristoph Lameter 		unsigned long addr, unsigned long end,
581dc9aa5b9SChristoph Lameter 		const nodemask_t *nodes, unsigned long flags,
58238e35860SChristoph Lameter 		void *private)
58391612e0dSHugh Dickins {
58491612e0dSHugh Dickins 	pud_t *pud;
58591612e0dSHugh Dickins 	unsigned long next;
58691612e0dSHugh Dickins 
58791612e0dSHugh Dickins 	pud = pud_offset(pgd, addr);
58891612e0dSHugh Dickins 	do {
58991612e0dSHugh Dickins 		next = pud_addr_end(addr, end);
590e2d8cf40SNaoya Horiguchi 		if (pud_huge(*pud) && is_vm_hugetlb_page(vma))
591e2d8cf40SNaoya Horiguchi 			continue;
59291612e0dSHugh Dickins 		if (pud_none_or_clear_bad(pud))
59391612e0dSHugh Dickins 			continue;
59498094945SNaoya Horiguchi 		if (queue_pages_pmd_range(vma, pud, addr, next, nodes,
59538e35860SChristoph Lameter 				    flags, private))
59691612e0dSHugh Dickins 			return -EIO;
59791612e0dSHugh Dickins 	} while (pud++, addr = next, addr != end);
59891612e0dSHugh Dickins 	return 0;
59991612e0dSHugh Dickins }
60091612e0dSHugh Dickins 
60198094945SNaoya Horiguchi static inline int queue_pages_pgd_range(struct vm_area_struct *vma,
602dc9aa5b9SChristoph Lameter 		unsigned long addr, unsigned long end,
603dc9aa5b9SChristoph Lameter 		const nodemask_t *nodes, unsigned long flags,
60438e35860SChristoph Lameter 		void *private)
60591612e0dSHugh Dickins {
60691612e0dSHugh Dickins 	pgd_t *pgd;
60791612e0dSHugh Dickins 	unsigned long next;
60891612e0dSHugh Dickins 
609b5810039SNick Piggin 	pgd = pgd_offset(vma->vm_mm, addr);
61091612e0dSHugh Dickins 	do {
61191612e0dSHugh Dickins 		next = pgd_addr_end(addr, end);
61291612e0dSHugh Dickins 		if (pgd_none_or_clear_bad(pgd))
61391612e0dSHugh Dickins 			continue;
61498094945SNaoya Horiguchi 		if (queue_pages_pud_range(vma, pgd, addr, next, nodes,
61538e35860SChristoph Lameter 				    flags, private))
61691612e0dSHugh Dickins 			return -EIO;
61791612e0dSHugh Dickins 	} while (pgd++, addr = next, addr != end);
61891612e0dSHugh Dickins 	return 0;
6191da177e4SLinus Torvalds }
6201da177e4SLinus Torvalds 
6215877231fSAneesh Kumar K.V #ifdef CONFIG_NUMA_BALANCING
622b24f53a0SLee Schermerhorn /*
6234b10e7d5SMel Gorman  * This is used to mark a range of virtual addresses to be inaccessible.
6244b10e7d5SMel Gorman  * These are later cleared by a NUMA hinting fault. Depending on these
6254b10e7d5SMel Gorman  * faults, pages may be migrated for better NUMA placement.
6264b10e7d5SMel Gorman  *
6274b10e7d5SMel Gorman  * This is assuming that NUMA faults are handled using PROT_NONE. If
6284b10e7d5SMel Gorman  * an architecture makes a different choice, it will need further
6294b10e7d5SMel Gorman  * changes to the core.
630b24f53a0SLee Schermerhorn  */
6314b10e7d5SMel Gorman unsigned long change_prot_numa(struct vm_area_struct *vma,
6324b10e7d5SMel Gorman 			unsigned long addr, unsigned long end)
633b24f53a0SLee Schermerhorn {
6344b10e7d5SMel Gorman 	int nr_updated;
635b24f53a0SLee Schermerhorn 
6364b10e7d5SMel Gorman 	nr_updated = change_protection(vma, addr, end, vma->vm_page_prot, 0, 1);
63703c5a6e1SMel Gorman 	if (nr_updated)
63803c5a6e1SMel Gorman 		count_vm_numa_events(NUMA_PTE_UPDATES, nr_updated);
639b24f53a0SLee Schermerhorn 
6404b10e7d5SMel Gorman 	return nr_updated;
641b24f53a0SLee Schermerhorn }
642b24f53a0SLee Schermerhorn #else
643b24f53a0SLee Schermerhorn static unsigned long change_prot_numa(struct vm_area_struct *vma,
644b24f53a0SLee Schermerhorn 			unsigned long addr, unsigned long end)
645b24f53a0SLee Schermerhorn {
646b24f53a0SLee Schermerhorn 	return 0;
647b24f53a0SLee Schermerhorn }
6485877231fSAneesh Kumar K.V #endif /* CONFIG_NUMA_BALANCING */
649b24f53a0SLee Schermerhorn 
650dc9aa5b9SChristoph Lameter /*
65198094945SNaoya Horiguchi  * Walk through page tables and collect pages to be migrated.
65298094945SNaoya Horiguchi  *
65398094945SNaoya Horiguchi  * If pages found in a given range are on a set of nodes (determined by
65498094945SNaoya Horiguchi  * @nodes and @flags,) it's isolated and queued to the pagelist which is
65598094945SNaoya Horiguchi  * passed via @private.)
656dc9aa5b9SChristoph Lameter  */
657d05f0cdcSHugh Dickins static int
65898094945SNaoya Horiguchi queue_pages_range(struct mm_struct *mm, unsigned long start, unsigned long end,
65938e35860SChristoph Lameter 		const nodemask_t *nodes, unsigned long flags, void *private)
6601da177e4SLinus Torvalds {
661d05f0cdcSHugh Dickins 	int err = 0;
662d05f0cdcSHugh Dickins 	struct vm_area_struct *vma, *prev;
6631da177e4SLinus Torvalds 
664d05f0cdcSHugh Dickins 	vma = find_vma(mm, start);
665d05f0cdcSHugh Dickins 	if (!vma)
666d05f0cdcSHugh Dickins 		return -EFAULT;
6671da177e4SLinus Torvalds 	prev = NULL;
668d05f0cdcSHugh Dickins 	for (; vma && vma->vm_start < end; vma = vma->vm_next) {
6695b952b3cSAndi Kleen 		unsigned long endvma = vma->vm_end;
670dc9aa5b9SChristoph Lameter 
6715b952b3cSAndi Kleen 		if (endvma > end)
6725b952b3cSAndi Kleen 			endvma = end;
6735b952b3cSAndi Kleen 		if (vma->vm_start > start)
6745b952b3cSAndi Kleen 			start = vma->vm_start;
675b24f53a0SLee Schermerhorn 
676b24f53a0SLee Schermerhorn 		if (!(flags & MPOL_MF_DISCONTIG_OK)) {
677b24f53a0SLee Schermerhorn 			if (!vma->vm_next && vma->vm_end < end)
678d05f0cdcSHugh Dickins 				return -EFAULT;
679b24f53a0SLee Schermerhorn 			if (prev && prev->vm_end < vma->vm_start)
680d05f0cdcSHugh Dickins 				return -EFAULT;
681b24f53a0SLee Schermerhorn 		}
682b24f53a0SLee Schermerhorn 
683b24f53a0SLee Schermerhorn 		if (flags & MPOL_MF_LAZY) {
684b24f53a0SLee Schermerhorn 			change_prot_numa(vma, start, endvma);
685b24f53a0SLee Schermerhorn 			goto next;
686b24f53a0SLee Schermerhorn 		}
687b24f53a0SLee Schermerhorn 
688b24f53a0SLee Schermerhorn 		if ((flags & MPOL_MF_STRICT) ||
689b24f53a0SLee Schermerhorn 		     ((flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) &&
690b24f53a0SLee Schermerhorn 		      vma_migratable(vma))) {
691b24f53a0SLee Schermerhorn 
69298094945SNaoya Horiguchi 			err = queue_pages_pgd_range(vma, start, endvma, nodes,
69338e35860SChristoph Lameter 						flags, private);
694d05f0cdcSHugh Dickins 			if (err)
6951da177e4SLinus Torvalds 				break;
6961da177e4SLinus Torvalds 		}
697b24f53a0SLee Schermerhorn next:
6981da177e4SLinus Torvalds 		prev = vma;
6991da177e4SLinus Torvalds 	}
700d05f0cdcSHugh Dickins 	return err;
7011da177e4SLinus Torvalds }
7021da177e4SLinus Torvalds 
703869833f2SKOSAKI Motohiro /*
704869833f2SKOSAKI Motohiro  * Apply policy to a single VMA
705869833f2SKOSAKI Motohiro  * This must be called with the mmap_sem held for writing.
706869833f2SKOSAKI Motohiro  */
707869833f2SKOSAKI Motohiro static int vma_replace_policy(struct vm_area_struct *vma,
708869833f2SKOSAKI Motohiro 						struct mempolicy *pol)
7098d34694cSKOSAKI Motohiro {
710869833f2SKOSAKI Motohiro 	int err;
711869833f2SKOSAKI Motohiro 	struct mempolicy *old;
712869833f2SKOSAKI Motohiro 	struct mempolicy *new;
7138d34694cSKOSAKI Motohiro 
7148d34694cSKOSAKI Motohiro 	pr_debug("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n",
7158d34694cSKOSAKI Motohiro 		 vma->vm_start, vma->vm_end, vma->vm_pgoff,
7168d34694cSKOSAKI Motohiro 		 vma->vm_ops, vma->vm_file,
7178d34694cSKOSAKI Motohiro 		 vma->vm_ops ? vma->vm_ops->set_policy : NULL);
7188d34694cSKOSAKI Motohiro 
719869833f2SKOSAKI Motohiro 	new = mpol_dup(pol);
720869833f2SKOSAKI Motohiro 	if (IS_ERR(new))
721869833f2SKOSAKI Motohiro 		return PTR_ERR(new);
722869833f2SKOSAKI Motohiro 
723869833f2SKOSAKI Motohiro 	if (vma->vm_ops && vma->vm_ops->set_policy) {
7248d34694cSKOSAKI Motohiro 		err = vma->vm_ops->set_policy(vma, new);
725869833f2SKOSAKI Motohiro 		if (err)
726869833f2SKOSAKI Motohiro 			goto err_out;
7278d34694cSKOSAKI Motohiro 	}
728869833f2SKOSAKI Motohiro 
729869833f2SKOSAKI Motohiro 	old = vma->vm_policy;
730869833f2SKOSAKI Motohiro 	vma->vm_policy = new; /* protected by mmap_sem */
731869833f2SKOSAKI Motohiro 	mpol_put(old);
732869833f2SKOSAKI Motohiro 
733869833f2SKOSAKI Motohiro 	return 0;
734869833f2SKOSAKI Motohiro  err_out:
735869833f2SKOSAKI Motohiro 	mpol_put(new);
7368d34694cSKOSAKI Motohiro 	return err;
7378d34694cSKOSAKI Motohiro }
7388d34694cSKOSAKI Motohiro 
7391da177e4SLinus Torvalds /* Step 2: apply policy to a range and do splits. */
7409d8cebd4SKOSAKI Motohiro static int mbind_range(struct mm_struct *mm, unsigned long start,
7419d8cebd4SKOSAKI Motohiro 		       unsigned long end, struct mempolicy *new_pol)
7421da177e4SLinus Torvalds {
7431da177e4SLinus Torvalds 	struct vm_area_struct *next;
7449d8cebd4SKOSAKI Motohiro 	struct vm_area_struct *prev;
7459d8cebd4SKOSAKI Motohiro 	struct vm_area_struct *vma;
7469d8cebd4SKOSAKI Motohiro 	int err = 0;
747e26a5114SKOSAKI Motohiro 	pgoff_t pgoff;
7489d8cebd4SKOSAKI Motohiro 	unsigned long vmstart;
7499d8cebd4SKOSAKI Motohiro 	unsigned long vmend;
7501da177e4SLinus Torvalds 
751097d5910SLinus Torvalds 	vma = find_vma(mm, start);
7529d8cebd4SKOSAKI Motohiro 	if (!vma || vma->vm_start > start)
7539d8cebd4SKOSAKI Motohiro 		return -EFAULT;
7549d8cebd4SKOSAKI Motohiro 
755097d5910SLinus Torvalds 	prev = vma->vm_prev;
756e26a5114SKOSAKI Motohiro 	if (start > vma->vm_start)
757e26a5114SKOSAKI Motohiro 		prev = vma;
758e26a5114SKOSAKI Motohiro 
7599d8cebd4SKOSAKI Motohiro 	for (; vma && vma->vm_start < end; prev = vma, vma = next) {
7601da177e4SLinus Torvalds 		next = vma->vm_next;
7619d8cebd4SKOSAKI Motohiro 		vmstart = max(start, vma->vm_start);
7629d8cebd4SKOSAKI Motohiro 		vmend   = min(end, vma->vm_end);
7639d8cebd4SKOSAKI Motohiro 
764e26a5114SKOSAKI Motohiro 		if (mpol_equal(vma_policy(vma), new_pol))
765e26a5114SKOSAKI Motohiro 			continue;
766e26a5114SKOSAKI Motohiro 
767e26a5114SKOSAKI Motohiro 		pgoff = vma->vm_pgoff +
768e26a5114SKOSAKI Motohiro 			((vmstart - vma->vm_start) >> PAGE_SHIFT);
7699d8cebd4SKOSAKI Motohiro 		prev = vma_merge(mm, prev, vmstart, vmend, vma->vm_flags,
770e26a5114SKOSAKI Motohiro 				  vma->anon_vma, vma->vm_file, pgoff,
7718aacc9f5SCaspar Zhang 				  new_pol);
7729d8cebd4SKOSAKI Motohiro 		if (prev) {
7739d8cebd4SKOSAKI Motohiro 			vma = prev;
7749d8cebd4SKOSAKI Motohiro 			next = vma->vm_next;
7753964acd0SOleg Nesterov 			if (mpol_equal(vma_policy(vma), new_pol))
7769d8cebd4SKOSAKI Motohiro 				continue;
7773964acd0SOleg Nesterov 			/* vma_merge() joined vma && vma->next, case 8 */
7783964acd0SOleg Nesterov 			goto replace;
7791da177e4SLinus Torvalds 		}
7809d8cebd4SKOSAKI Motohiro 		if (vma->vm_start != vmstart) {
7819d8cebd4SKOSAKI Motohiro 			err = split_vma(vma->vm_mm, vma, vmstart, 1);
7829d8cebd4SKOSAKI Motohiro 			if (err)
7839d8cebd4SKOSAKI Motohiro 				goto out;
7849d8cebd4SKOSAKI Motohiro 		}
7859d8cebd4SKOSAKI Motohiro 		if (vma->vm_end != vmend) {
7869d8cebd4SKOSAKI Motohiro 			err = split_vma(vma->vm_mm, vma, vmend, 0);
7879d8cebd4SKOSAKI Motohiro 			if (err)
7889d8cebd4SKOSAKI Motohiro 				goto out;
7899d8cebd4SKOSAKI Motohiro 		}
7903964acd0SOleg Nesterov  replace:
791869833f2SKOSAKI Motohiro 		err = vma_replace_policy(vma, new_pol);
7929d8cebd4SKOSAKI Motohiro 		if (err)
7939d8cebd4SKOSAKI Motohiro 			goto out;
7949d8cebd4SKOSAKI Motohiro 	}
7959d8cebd4SKOSAKI Motohiro 
7969d8cebd4SKOSAKI Motohiro  out:
7971da177e4SLinus Torvalds 	return err;
7981da177e4SLinus Torvalds }
7991da177e4SLinus Torvalds 
8001da177e4SLinus Torvalds /* Set the process memory policy */
801028fec41SDavid Rientjes static long do_set_mempolicy(unsigned short mode, unsigned short flags,
802028fec41SDavid Rientjes 			     nodemask_t *nodes)
8031da177e4SLinus Torvalds {
80458568d2aSMiao Xie 	struct mempolicy *new, *old;
805f4e53d91SLee Schermerhorn 	struct mm_struct *mm = current->mm;
8064bfc4495SKAMEZAWA Hiroyuki 	NODEMASK_SCRATCH(scratch);
80758568d2aSMiao Xie 	int ret;
8081da177e4SLinus Torvalds 
8094bfc4495SKAMEZAWA Hiroyuki 	if (!scratch)
8104bfc4495SKAMEZAWA Hiroyuki 		return -ENOMEM;
811f4e53d91SLee Schermerhorn 
8124bfc4495SKAMEZAWA Hiroyuki 	new = mpol_new(mode, flags, nodes);
8134bfc4495SKAMEZAWA Hiroyuki 	if (IS_ERR(new)) {
8144bfc4495SKAMEZAWA Hiroyuki 		ret = PTR_ERR(new);
8154bfc4495SKAMEZAWA Hiroyuki 		goto out;
8164bfc4495SKAMEZAWA Hiroyuki 	}
817f4e53d91SLee Schermerhorn 	/*
818f4e53d91SLee Schermerhorn 	 * prevent changing our mempolicy while show_numa_maps()
819f4e53d91SLee Schermerhorn 	 * is using it.
820f4e53d91SLee Schermerhorn 	 * Note:  do_set_mempolicy() can be called at init time
821f4e53d91SLee Schermerhorn 	 * with no 'mm'.
822f4e53d91SLee Schermerhorn 	 */
823f4e53d91SLee Schermerhorn 	if (mm)
824f4e53d91SLee Schermerhorn 		down_write(&mm->mmap_sem);
82558568d2aSMiao Xie 	task_lock(current);
8264bfc4495SKAMEZAWA Hiroyuki 	ret = mpol_set_nodemask(new, nodes, scratch);
82758568d2aSMiao Xie 	if (ret) {
82858568d2aSMiao Xie 		task_unlock(current);
82958568d2aSMiao Xie 		if (mm)
83058568d2aSMiao Xie 			up_write(&mm->mmap_sem);
83158568d2aSMiao Xie 		mpol_put(new);
8324bfc4495SKAMEZAWA Hiroyuki 		goto out;
83358568d2aSMiao Xie 	}
83458568d2aSMiao Xie 	old = current->mempolicy;
8351da177e4SLinus Torvalds 	current->mempolicy = new;
83645c4745aSLee Schermerhorn 	if (new && new->mode == MPOL_INTERLEAVE &&
837f5b087b5SDavid Rientjes 	    nodes_weight(new->v.nodes))
838dfcd3c0dSAndi Kleen 		current->il_next = first_node(new->v.nodes);
83958568d2aSMiao Xie 	task_unlock(current);
840f4e53d91SLee Schermerhorn 	if (mm)
841f4e53d91SLee Schermerhorn 		up_write(&mm->mmap_sem);
842f4e53d91SLee Schermerhorn 
84358568d2aSMiao Xie 	mpol_put(old);
8444bfc4495SKAMEZAWA Hiroyuki 	ret = 0;
8454bfc4495SKAMEZAWA Hiroyuki out:
8464bfc4495SKAMEZAWA Hiroyuki 	NODEMASK_SCRATCH_FREE(scratch);
8474bfc4495SKAMEZAWA Hiroyuki 	return ret;
8481da177e4SLinus Torvalds }
8491da177e4SLinus Torvalds 
850bea904d5SLee Schermerhorn /*
851bea904d5SLee Schermerhorn  * Return nodemask for policy for get_mempolicy() query
85258568d2aSMiao Xie  *
85358568d2aSMiao Xie  * Called with task's alloc_lock held
854bea904d5SLee Schermerhorn  */
855bea904d5SLee Schermerhorn static void get_policy_nodemask(struct mempolicy *p, nodemask_t *nodes)
8561da177e4SLinus Torvalds {
857dfcd3c0dSAndi Kleen 	nodes_clear(*nodes);
858bea904d5SLee Schermerhorn 	if (p == &default_policy)
859bea904d5SLee Schermerhorn 		return;
860bea904d5SLee Schermerhorn 
86145c4745aSLee Schermerhorn 	switch (p->mode) {
86219770b32SMel Gorman 	case MPOL_BIND:
86319770b32SMel Gorman 		/* Fall through */
8641da177e4SLinus Torvalds 	case MPOL_INTERLEAVE:
865dfcd3c0dSAndi Kleen 		*nodes = p->v.nodes;
8661da177e4SLinus Torvalds 		break;
8671da177e4SLinus Torvalds 	case MPOL_PREFERRED:
868fc36b8d3SLee Schermerhorn 		if (!(p->flags & MPOL_F_LOCAL))
869dfcd3c0dSAndi Kleen 			node_set(p->v.preferred_node, *nodes);
87053f2556bSLee Schermerhorn 		/* else return empty node mask for local allocation */
8711da177e4SLinus Torvalds 		break;
8721da177e4SLinus Torvalds 	default:
8731da177e4SLinus Torvalds 		BUG();
8741da177e4SLinus Torvalds 	}
8751da177e4SLinus Torvalds }
8761da177e4SLinus Torvalds 
8771da177e4SLinus Torvalds static int lookup_node(struct mm_struct *mm, unsigned long addr)
8781da177e4SLinus Torvalds {
8791da177e4SLinus Torvalds 	struct page *p;
8801da177e4SLinus Torvalds 	int err;
8811da177e4SLinus Torvalds 
8821da177e4SLinus Torvalds 	err = get_user_pages(current, mm, addr & PAGE_MASK, 1, 0, 0, &p, NULL);
8831da177e4SLinus Torvalds 	if (err >= 0) {
8841da177e4SLinus Torvalds 		err = page_to_nid(p);
8851da177e4SLinus Torvalds 		put_page(p);
8861da177e4SLinus Torvalds 	}
8871da177e4SLinus Torvalds 	return err;
8881da177e4SLinus Torvalds }
8891da177e4SLinus Torvalds 
8901da177e4SLinus Torvalds /* Retrieve NUMA policy */
891dbcb0f19SAdrian Bunk static long do_get_mempolicy(int *policy, nodemask_t *nmask,
8921da177e4SLinus Torvalds 			     unsigned long addr, unsigned long flags)
8931da177e4SLinus Torvalds {
8948bccd85fSChristoph Lameter 	int err;
8951da177e4SLinus Torvalds 	struct mm_struct *mm = current->mm;
8961da177e4SLinus Torvalds 	struct vm_area_struct *vma = NULL;
8971da177e4SLinus Torvalds 	struct mempolicy *pol = current->mempolicy;
8981da177e4SLinus Torvalds 
899754af6f5SLee Schermerhorn 	if (flags &
900754af6f5SLee Schermerhorn 		~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR|MPOL_F_MEMS_ALLOWED))
9011da177e4SLinus Torvalds 		return -EINVAL;
902754af6f5SLee Schermerhorn 
903754af6f5SLee Schermerhorn 	if (flags & MPOL_F_MEMS_ALLOWED) {
904754af6f5SLee Schermerhorn 		if (flags & (MPOL_F_NODE|MPOL_F_ADDR))
905754af6f5SLee Schermerhorn 			return -EINVAL;
906754af6f5SLee Schermerhorn 		*policy = 0;	/* just so it's initialized */
90758568d2aSMiao Xie 		task_lock(current);
908754af6f5SLee Schermerhorn 		*nmask  = cpuset_current_mems_allowed;
90958568d2aSMiao Xie 		task_unlock(current);
910754af6f5SLee Schermerhorn 		return 0;
911754af6f5SLee Schermerhorn 	}
912754af6f5SLee Schermerhorn 
9131da177e4SLinus Torvalds 	if (flags & MPOL_F_ADDR) {
914bea904d5SLee Schermerhorn 		/*
915bea904d5SLee Schermerhorn 		 * Do NOT fall back to task policy if the
916bea904d5SLee Schermerhorn 		 * vma/shared policy at addr is NULL.  We
917bea904d5SLee Schermerhorn 		 * want to return MPOL_DEFAULT in this case.
918bea904d5SLee Schermerhorn 		 */
9191da177e4SLinus Torvalds 		down_read(&mm->mmap_sem);
9201da177e4SLinus Torvalds 		vma = find_vma_intersection(mm, addr, addr+1);
9211da177e4SLinus Torvalds 		if (!vma) {
9221da177e4SLinus Torvalds 			up_read(&mm->mmap_sem);
9231da177e4SLinus Torvalds 			return -EFAULT;
9241da177e4SLinus Torvalds 		}
9251da177e4SLinus Torvalds 		if (vma->vm_ops && vma->vm_ops->get_policy)
9261da177e4SLinus Torvalds 			pol = vma->vm_ops->get_policy(vma, addr);
9271da177e4SLinus Torvalds 		else
9281da177e4SLinus Torvalds 			pol = vma->vm_policy;
9291da177e4SLinus Torvalds 	} else if (addr)
9301da177e4SLinus Torvalds 		return -EINVAL;
9311da177e4SLinus Torvalds 
9321da177e4SLinus Torvalds 	if (!pol)
933bea904d5SLee Schermerhorn 		pol = &default_policy;	/* indicates default behavior */
9341da177e4SLinus Torvalds 
9351da177e4SLinus Torvalds 	if (flags & MPOL_F_NODE) {
9361da177e4SLinus Torvalds 		if (flags & MPOL_F_ADDR) {
9371da177e4SLinus Torvalds 			err = lookup_node(mm, addr);
9381da177e4SLinus Torvalds 			if (err < 0)
9391da177e4SLinus Torvalds 				goto out;
9408bccd85fSChristoph Lameter 			*policy = err;
9411da177e4SLinus Torvalds 		} else if (pol == current->mempolicy &&
94245c4745aSLee Schermerhorn 				pol->mode == MPOL_INTERLEAVE) {
9438bccd85fSChristoph Lameter 			*policy = current->il_next;
9441da177e4SLinus Torvalds 		} else {
9451da177e4SLinus Torvalds 			err = -EINVAL;
9461da177e4SLinus Torvalds 			goto out;
9471da177e4SLinus Torvalds 		}
948bea904d5SLee Schermerhorn 	} else {
949bea904d5SLee Schermerhorn 		*policy = pol == &default_policy ? MPOL_DEFAULT :
950bea904d5SLee Schermerhorn 						pol->mode;
951d79df630SDavid Rientjes 		/*
952d79df630SDavid Rientjes 		 * Internal mempolicy flags must be masked off before exposing
953d79df630SDavid Rientjes 		 * the policy to userspace.
954d79df630SDavid Rientjes 		 */
955d79df630SDavid Rientjes 		*policy |= (pol->flags & MPOL_MODE_FLAGS);
956bea904d5SLee Schermerhorn 	}
9571da177e4SLinus Torvalds 
9581da177e4SLinus Torvalds 	if (vma) {
9591da177e4SLinus Torvalds 		up_read(&current->mm->mmap_sem);
9601da177e4SLinus Torvalds 		vma = NULL;
9611da177e4SLinus Torvalds 	}
9621da177e4SLinus Torvalds 
9631da177e4SLinus Torvalds 	err = 0;
96458568d2aSMiao Xie 	if (nmask) {
965c6b6ef8bSLee Schermerhorn 		if (mpol_store_user_nodemask(pol)) {
966c6b6ef8bSLee Schermerhorn 			*nmask = pol->w.user_nodemask;
967c6b6ef8bSLee Schermerhorn 		} else {
96858568d2aSMiao Xie 			task_lock(current);
969bea904d5SLee Schermerhorn 			get_policy_nodemask(pol, nmask);
97058568d2aSMiao Xie 			task_unlock(current);
97158568d2aSMiao Xie 		}
972c6b6ef8bSLee Schermerhorn 	}
9731da177e4SLinus Torvalds 
9741da177e4SLinus Torvalds  out:
97552cd3b07SLee Schermerhorn 	mpol_cond_put(pol);
9761da177e4SLinus Torvalds 	if (vma)
9771da177e4SLinus Torvalds 		up_read(&current->mm->mmap_sem);
9781da177e4SLinus Torvalds 	return err;
9791da177e4SLinus Torvalds }
9801da177e4SLinus Torvalds 
981b20a3503SChristoph Lameter #ifdef CONFIG_MIGRATION
9828bccd85fSChristoph Lameter /*
9836ce3c4c0SChristoph Lameter  * page migration
9846ce3c4c0SChristoph Lameter  */
985fc301289SChristoph Lameter static void migrate_page_add(struct page *page, struct list_head *pagelist,
986fc301289SChristoph Lameter 				unsigned long flags)
9876ce3c4c0SChristoph Lameter {
9886ce3c4c0SChristoph Lameter 	/*
989fc301289SChristoph Lameter 	 * Avoid migrating a page that is shared with others.
9906ce3c4c0SChristoph Lameter 	 */
99162695a84SNick Piggin 	if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1) {
99262695a84SNick Piggin 		if (!isolate_lru_page(page)) {
99362695a84SNick Piggin 			list_add_tail(&page->lru, pagelist);
9946d9c285aSKOSAKI Motohiro 			inc_zone_page_state(page, NR_ISOLATED_ANON +
9956d9c285aSKOSAKI Motohiro 					    page_is_file_cache(page));
99662695a84SNick Piggin 		}
99762695a84SNick Piggin 	}
9986ce3c4c0SChristoph Lameter }
9996ce3c4c0SChristoph Lameter 
1000742755a1SChristoph Lameter static struct page *new_node_page(struct page *page, unsigned long node, int **x)
100195a402c3SChristoph Lameter {
1002e2d8cf40SNaoya Horiguchi 	if (PageHuge(page))
1003e2d8cf40SNaoya Horiguchi 		return alloc_huge_page_node(page_hstate(compound_head(page)),
1004e2d8cf40SNaoya Horiguchi 					node);
1005e2d8cf40SNaoya Horiguchi 	else
10066484eb3eSMel Gorman 		return alloc_pages_exact_node(node, GFP_HIGHUSER_MOVABLE, 0);
100795a402c3SChristoph Lameter }
100895a402c3SChristoph Lameter 
10096ce3c4c0SChristoph Lameter /*
10107e2ab150SChristoph Lameter  * Migrate pages from one node to a target node.
10117e2ab150SChristoph Lameter  * Returns error or the number of pages not migrated.
10127e2ab150SChristoph Lameter  */
1013dbcb0f19SAdrian Bunk static int migrate_to_node(struct mm_struct *mm, int source, int dest,
1014dbcb0f19SAdrian Bunk 			   int flags)
10157e2ab150SChristoph Lameter {
10167e2ab150SChristoph Lameter 	nodemask_t nmask;
10177e2ab150SChristoph Lameter 	LIST_HEAD(pagelist);
10187e2ab150SChristoph Lameter 	int err = 0;
10197e2ab150SChristoph Lameter 
10207e2ab150SChristoph Lameter 	nodes_clear(nmask);
10217e2ab150SChristoph Lameter 	node_set(source, nmask);
10227e2ab150SChristoph Lameter 
102308270807SMinchan Kim 	/*
102408270807SMinchan Kim 	 * This does not "check" the range but isolates all pages that
102508270807SMinchan Kim 	 * need migration.  Between passing in the full user address
102608270807SMinchan Kim 	 * space range and MPOL_MF_DISCONTIG_OK, this call can not fail.
102708270807SMinchan Kim 	 */
102808270807SMinchan Kim 	VM_BUG_ON(!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)));
102998094945SNaoya Horiguchi 	queue_pages_range(mm, mm->mmap->vm_start, mm->task_size, &nmask,
10307e2ab150SChristoph Lameter 			flags | MPOL_MF_DISCONTIG_OK, &pagelist);
10317e2ab150SChristoph Lameter 
1032cf608ac1SMinchan Kim 	if (!list_empty(&pagelist)) {
103368711a74SDavid Rientjes 		err = migrate_pages(&pagelist, new_node_page, NULL, dest,
10349c620e2bSHugh Dickins 					MIGRATE_SYNC, MR_SYSCALL);
1035cf608ac1SMinchan Kim 		if (err)
1036e2d8cf40SNaoya Horiguchi 			putback_movable_pages(&pagelist);
1037cf608ac1SMinchan Kim 	}
103895a402c3SChristoph Lameter 
10397e2ab150SChristoph Lameter 	return err;
10407e2ab150SChristoph Lameter }
10417e2ab150SChristoph Lameter 
10427e2ab150SChristoph Lameter /*
10437e2ab150SChristoph Lameter  * Move pages between the two nodesets so as to preserve the physical
10447e2ab150SChristoph Lameter  * layout as much as possible.
104539743889SChristoph Lameter  *
104639743889SChristoph Lameter  * Returns the number of page that could not be moved.
104739743889SChristoph Lameter  */
10480ce72d4fSAndrew Morton int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
10490ce72d4fSAndrew Morton 		     const nodemask_t *to, int flags)
105039743889SChristoph Lameter {
10517e2ab150SChristoph Lameter 	int busy = 0;
10520aedadf9SChristoph Lameter 	int err;
10537e2ab150SChristoph Lameter 	nodemask_t tmp;
105439743889SChristoph Lameter 
10550aedadf9SChristoph Lameter 	err = migrate_prep();
10560aedadf9SChristoph Lameter 	if (err)
10570aedadf9SChristoph Lameter 		return err;
10580aedadf9SChristoph Lameter 
105939743889SChristoph Lameter 	down_read(&mm->mmap_sem);
1060d4984711SChristoph Lameter 
10610ce72d4fSAndrew Morton 	err = migrate_vmas(mm, from, to, flags);
10627b2259b3SChristoph Lameter 	if (err)
10637b2259b3SChristoph Lameter 		goto out;
10647b2259b3SChristoph Lameter 
10657e2ab150SChristoph Lameter 	/*
10667e2ab150SChristoph Lameter 	 * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
10677e2ab150SChristoph Lameter 	 * bit in 'to' is not also set in 'tmp'.  Clear the found 'source'
10687e2ab150SChristoph Lameter 	 * bit in 'tmp', and return that <source, dest> pair for migration.
10697e2ab150SChristoph Lameter 	 * The pair of nodemasks 'to' and 'from' define the map.
10707e2ab150SChristoph Lameter 	 *
10717e2ab150SChristoph Lameter 	 * If no pair of bits is found that way, fallback to picking some
10727e2ab150SChristoph Lameter 	 * pair of 'source' and 'dest' bits that are not the same.  If the
10737e2ab150SChristoph Lameter 	 * 'source' and 'dest' bits are the same, this represents a node
10747e2ab150SChristoph Lameter 	 * that will be migrating to itself, so no pages need move.
10757e2ab150SChristoph Lameter 	 *
10767e2ab150SChristoph Lameter 	 * If no bits are left in 'tmp', or if all remaining bits left
10777e2ab150SChristoph Lameter 	 * in 'tmp' correspond to the same bit in 'to', return false
10787e2ab150SChristoph Lameter 	 * (nothing left to migrate).
10797e2ab150SChristoph Lameter 	 *
10807e2ab150SChristoph Lameter 	 * This lets us pick a pair of nodes to migrate between, such that
10817e2ab150SChristoph Lameter 	 * if possible the dest node is not already occupied by some other
10827e2ab150SChristoph Lameter 	 * source node, minimizing the risk of overloading the memory on a
10837e2ab150SChristoph Lameter 	 * node that would happen if we migrated incoming memory to a node
10847e2ab150SChristoph Lameter 	 * before migrating outgoing memory source that same node.
10857e2ab150SChristoph Lameter 	 *
10867e2ab150SChristoph Lameter 	 * A single scan of tmp is sufficient.  As we go, we remember the
10877e2ab150SChristoph Lameter 	 * most recent <s, d> pair that moved (s != d).  If we find a pair
10887e2ab150SChristoph Lameter 	 * that not only moved, but what's better, moved to an empty slot
10897e2ab150SChristoph Lameter 	 * (d is not set in tmp), then we break out then, with that pair.
1090ae0e47f0SJustin P. Mattock 	 * Otherwise when we finish scanning from_tmp, we at least have the
10917e2ab150SChristoph Lameter 	 * most recent <s, d> pair that moved.  If we get all the way through
10927e2ab150SChristoph Lameter 	 * the scan of tmp without finding any node that moved, much less
10937e2ab150SChristoph Lameter 	 * moved to an empty node, then there is nothing left worth migrating.
10947e2ab150SChristoph Lameter 	 */
10957e2ab150SChristoph Lameter 
10960ce72d4fSAndrew Morton 	tmp = *from;
10977e2ab150SChristoph Lameter 	while (!nodes_empty(tmp)) {
10987e2ab150SChristoph Lameter 		int s,d;
1099b76ac7e7SJianguo Wu 		int source = NUMA_NO_NODE;
11007e2ab150SChristoph Lameter 		int dest = 0;
11017e2ab150SChristoph Lameter 
11027e2ab150SChristoph Lameter 		for_each_node_mask(s, tmp) {
11034a5b18ccSLarry Woodman 
11044a5b18ccSLarry Woodman 			/*
11054a5b18ccSLarry Woodman 			 * do_migrate_pages() tries to maintain the relative
11064a5b18ccSLarry Woodman 			 * node relationship of the pages established between
11074a5b18ccSLarry Woodman 			 * threads and memory areas.
11084a5b18ccSLarry Woodman                          *
11094a5b18ccSLarry Woodman 			 * However if the number of source nodes is not equal to
11104a5b18ccSLarry Woodman 			 * the number of destination nodes we can not preserve
11114a5b18ccSLarry Woodman 			 * this node relative relationship.  In that case, skip
11124a5b18ccSLarry Woodman 			 * copying memory from a node that is in the destination
11134a5b18ccSLarry Woodman 			 * mask.
11144a5b18ccSLarry Woodman 			 *
11154a5b18ccSLarry Woodman 			 * Example: [2,3,4] -> [3,4,5] moves everything.
11164a5b18ccSLarry Woodman 			 *          [0-7] - > [3,4,5] moves only 0,1,2,6,7.
11174a5b18ccSLarry Woodman 			 */
11184a5b18ccSLarry Woodman 
11190ce72d4fSAndrew Morton 			if ((nodes_weight(*from) != nodes_weight(*to)) &&
11200ce72d4fSAndrew Morton 						(node_isset(s, *to)))
11214a5b18ccSLarry Woodman 				continue;
11224a5b18ccSLarry Woodman 
11230ce72d4fSAndrew Morton 			d = node_remap(s, *from, *to);
11247e2ab150SChristoph Lameter 			if (s == d)
11257e2ab150SChristoph Lameter 				continue;
11267e2ab150SChristoph Lameter 
11277e2ab150SChristoph Lameter 			source = s;	/* Node moved. Memorize */
11287e2ab150SChristoph Lameter 			dest = d;
11297e2ab150SChristoph Lameter 
11307e2ab150SChristoph Lameter 			/* dest not in remaining from nodes? */
11317e2ab150SChristoph Lameter 			if (!node_isset(dest, tmp))
11327e2ab150SChristoph Lameter 				break;
11337e2ab150SChristoph Lameter 		}
1134b76ac7e7SJianguo Wu 		if (source == NUMA_NO_NODE)
11357e2ab150SChristoph Lameter 			break;
11367e2ab150SChristoph Lameter 
11377e2ab150SChristoph Lameter 		node_clear(source, tmp);
11387e2ab150SChristoph Lameter 		err = migrate_to_node(mm, source, dest, flags);
11397e2ab150SChristoph Lameter 		if (err > 0)
11407e2ab150SChristoph Lameter 			busy += err;
11417e2ab150SChristoph Lameter 		if (err < 0)
11427e2ab150SChristoph Lameter 			break;
114339743889SChristoph Lameter 	}
11447b2259b3SChristoph Lameter out:
114539743889SChristoph Lameter 	up_read(&mm->mmap_sem);
11467e2ab150SChristoph Lameter 	if (err < 0)
11477e2ab150SChristoph Lameter 		return err;
11487e2ab150SChristoph Lameter 	return busy;
1149b20a3503SChristoph Lameter 
115039743889SChristoph Lameter }
115139743889SChristoph Lameter 
11523ad33b24SLee Schermerhorn /*
11533ad33b24SLee Schermerhorn  * Allocate a new page for page migration based on vma policy.
1154d05f0cdcSHugh Dickins  * Start by assuming the page is mapped by the same vma as contains @start.
11553ad33b24SLee Schermerhorn  * Search forward from there, if not.  N.B., this assumes that the
11563ad33b24SLee Schermerhorn  * list of pages handed to migrate_pages()--which is how we get here--
11573ad33b24SLee Schermerhorn  * is in virtual address order.
11583ad33b24SLee Schermerhorn  */
1159d05f0cdcSHugh Dickins static struct page *new_page(struct page *page, unsigned long start, int **x)
116095a402c3SChristoph Lameter {
1161d05f0cdcSHugh Dickins 	struct vm_area_struct *vma;
11623ad33b24SLee Schermerhorn 	unsigned long uninitialized_var(address);
116395a402c3SChristoph Lameter 
1164d05f0cdcSHugh Dickins 	vma = find_vma(current->mm, start);
11653ad33b24SLee Schermerhorn 	while (vma) {
11663ad33b24SLee Schermerhorn 		address = page_address_in_vma(page, vma);
11673ad33b24SLee Schermerhorn 		if (address != -EFAULT)
11683ad33b24SLee Schermerhorn 			break;
11693ad33b24SLee Schermerhorn 		vma = vma->vm_next;
11703ad33b24SLee Schermerhorn 	}
11713ad33b24SLee Schermerhorn 
117211c731e8SWanpeng Li 	if (PageHuge(page)) {
1173cc81717eSMichal Hocko 		BUG_ON(!vma);
117474060e4dSNaoya Horiguchi 		return alloc_huge_page_noerr(vma, address, 1);
117511c731e8SWanpeng Li 	}
117611c731e8SWanpeng Li 	/*
117711c731e8SWanpeng Li 	 * if !vma, alloc_page_vma() will use task or system default policy
117811c731e8SWanpeng Li 	 */
11793ad33b24SLee Schermerhorn 	return alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
118095a402c3SChristoph Lameter }
1181b20a3503SChristoph Lameter #else
1182b20a3503SChristoph Lameter 
1183b20a3503SChristoph Lameter static void migrate_page_add(struct page *page, struct list_head *pagelist,
1184b20a3503SChristoph Lameter 				unsigned long flags)
1185b20a3503SChristoph Lameter {
1186b20a3503SChristoph Lameter }
1187b20a3503SChristoph Lameter 
11880ce72d4fSAndrew Morton int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
11890ce72d4fSAndrew Morton 		     const nodemask_t *to, int flags)
1190b20a3503SChristoph Lameter {
1191b20a3503SChristoph Lameter 	return -ENOSYS;
1192b20a3503SChristoph Lameter }
119395a402c3SChristoph Lameter 
1194d05f0cdcSHugh Dickins static struct page *new_page(struct page *page, unsigned long start, int **x)
119595a402c3SChristoph Lameter {
119695a402c3SChristoph Lameter 	return NULL;
119795a402c3SChristoph Lameter }
1198b20a3503SChristoph Lameter #endif
1199b20a3503SChristoph Lameter 
1200dbcb0f19SAdrian Bunk static long do_mbind(unsigned long start, unsigned long len,
1201028fec41SDavid Rientjes 		     unsigned short mode, unsigned short mode_flags,
1202028fec41SDavid Rientjes 		     nodemask_t *nmask, unsigned long flags)
12036ce3c4c0SChristoph Lameter {
12046ce3c4c0SChristoph Lameter 	struct mm_struct *mm = current->mm;
12056ce3c4c0SChristoph Lameter 	struct mempolicy *new;
12066ce3c4c0SChristoph Lameter 	unsigned long end;
12076ce3c4c0SChristoph Lameter 	int err;
12086ce3c4c0SChristoph Lameter 	LIST_HEAD(pagelist);
12096ce3c4c0SChristoph Lameter 
1210b24f53a0SLee Schermerhorn 	if (flags & ~(unsigned long)MPOL_MF_VALID)
12116ce3c4c0SChristoph Lameter 		return -EINVAL;
121274c00241SChristoph Lameter 	if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
12136ce3c4c0SChristoph Lameter 		return -EPERM;
12146ce3c4c0SChristoph Lameter 
12156ce3c4c0SChristoph Lameter 	if (start & ~PAGE_MASK)
12166ce3c4c0SChristoph Lameter 		return -EINVAL;
12176ce3c4c0SChristoph Lameter 
12186ce3c4c0SChristoph Lameter 	if (mode == MPOL_DEFAULT)
12196ce3c4c0SChristoph Lameter 		flags &= ~MPOL_MF_STRICT;
12206ce3c4c0SChristoph Lameter 
12216ce3c4c0SChristoph Lameter 	len = (len + PAGE_SIZE - 1) & PAGE_MASK;
12226ce3c4c0SChristoph Lameter 	end = start + len;
12236ce3c4c0SChristoph Lameter 
12246ce3c4c0SChristoph Lameter 	if (end < start)
12256ce3c4c0SChristoph Lameter 		return -EINVAL;
12266ce3c4c0SChristoph Lameter 	if (end == start)
12276ce3c4c0SChristoph Lameter 		return 0;
12286ce3c4c0SChristoph Lameter 
1229028fec41SDavid Rientjes 	new = mpol_new(mode, mode_flags, nmask);
12306ce3c4c0SChristoph Lameter 	if (IS_ERR(new))
12316ce3c4c0SChristoph Lameter 		return PTR_ERR(new);
12326ce3c4c0SChristoph Lameter 
1233b24f53a0SLee Schermerhorn 	if (flags & MPOL_MF_LAZY)
1234b24f53a0SLee Schermerhorn 		new->flags |= MPOL_F_MOF;
1235b24f53a0SLee Schermerhorn 
12366ce3c4c0SChristoph Lameter 	/*
12376ce3c4c0SChristoph Lameter 	 * If we are using the default policy then operation
12386ce3c4c0SChristoph Lameter 	 * on discontinuous address spaces is okay after all
12396ce3c4c0SChristoph Lameter 	 */
12406ce3c4c0SChristoph Lameter 	if (!new)
12416ce3c4c0SChristoph Lameter 		flags |= MPOL_MF_DISCONTIG_OK;
12426ce3c4c0SChristoph Lameter 
1243028fec41SDavid Rientjes 	pr_debug("mbind %lx-%lx mode:%d flags:%d nodes:%lx\n",
1244028fec41SDavid Rientjes 		 start, start + len, mode, mode_flags,
124500ef2d2fSDavid Rientjes 		 nmask ? nodes_addr(*nmask)[0] : NUMA_NO_NODE);
12466ce3c4c0SChristoph Lameter 
12470aedadf9SChristoph Lameter 	if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
12480aedadf9SChristoph Lameter 
12490aedadf9SChristoph Lameter 		err = migrate_prep();
12500aedadf9SChristoph Lameter 		if (err)
1251b05ca738SKOSAKI Motohiro 			goto mpol_out;
12520aedadf9SChristoph Lameter 	}
12534bfc4495SKAMEZAWA Hiroyuki 	{
12544bfc4495SKAMEZAWA Hiroyuki 		NODEMASK_SCRATCH(scratch);
12554bfc4495SKAMEZAWA Hiroyuki 		if (scratch) {
12566ce3c4c0SChristoph Lameter 			down_write(&mm->mmap_sem);
125758568d2aSMiao Xie 			task_lock(current);
12584bfc4495SKAMEZAWA Hiroyuki 			err = mpol_set_nodemask(new, nmask, scratch);
125958568d2aSMiao Xie 			task_unlock(current);
12604bfc4495SKAMEZAWA Hiroyuki 			if (err)
126158568d2aSMiao Xie 				up_write(&mm->mmap_sem);
12624bfc4495SKAMEZAWA Hiroyuki 		} else
12634bfc4495SKAMEZAWA Hiroyuki 			err = -ENOMEM;
12644bfc4495SKAMEZAWA Hiroyuki 		NODEMASK_SCRATCH_FREE(scratch);
12654bfc4495SKAMEZAWA Hiroyuki 	}
1266b05ca738SKOSAKI Motohiro 	if (err)
1267b05ca738SKOSAKI Motohiro 		goto mpol_out;
1268b05ca738SKOSAKI Motohiro 
1269d05f0cdcSHugh Dickins 	err = queue_pages_range(mm, start, end, nmask,
12706ce3c4c0SChristoph Lameter 			  flags | MPOL_MF_INVERT, &pagelist);
1271d05f0cdcSHugh Dickins 	if (!err)
12729d8cebd4SKOSAKI Motohiro 		err = mbind_range(mm, start, end, new);
12737e2ab150SChristoph Lameter 
1274b24f53a0SLee Schermerhorn 	if (!err) {
1275b24f53a0SLee Schermerhorn 		int nr_failed = 0;
1276b24f53a0SLee Schermerhorn 
1277cf608ac1SMinchan Kim 		if (!list_empty(&pagelist)) {
1278b24f53a0SLee Schermerhorn 			WARN_ON_ONCE(flags & MPOL_MF_LAZY);
1279d05f0cdcSHugh Dickins 			nr_failed = migrate_pages(&pagelist, new_page, NULL,
1280d05f0cdcSHugh Dickins 				start, MIGRATE_SYNC, MR_MEMPOLICY_MBIND);
1281cf608ac1SMinchan Kim 			if (nr_failed)
128274060e4dSNaoya Horiguchi 				putback_movable_pages(&pagelist);
1283cf608ac1SMinchan Kim 		}
12846ce3c4c0SChristoph Lameter 
1285b24f53a0SLee Schermerhorn 		if (nr_failed && (flags & MPOL_MF_STRICT))
12866ce3c4c0SChristoph Lameter 			err = -EIO;
1287ab8a3e14SKOSAKI Motohiro 	} else
1288b0e5fd73SJoonsoo Kim 		putback_movable_pages(&pagelist);
1289b20a3503SChristoph Lameter 
12906ce3c4c0SChristoph Lameter 	up_write(&mm->mmap_sem);
1291b05ca738SKOSAKI Motohiro  mpol_out:
1292f0be3d32SLee Schermerhorn 	mpol_put(new);
12936ce3c4c0SChristoph Lameter 	return err;
12946ce3c4c0SChristoph Lameter }
12956ce3c4c0SChristoph Lameter 
129639743889SChristoph Lameter /*
12978bccd85fSChristoph Lameter  * User space interface with variable sized bitmaps for nodelists.
12988bccd85fSChristoph Lameter  */
12998bccd85fSChristoph Lameter 
13008bccd85fSChristoph Lameter /* Copy a node mask from user space. */
130139743889SChristoph Lameter static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask,
13028bccd85fSChristoph Lameter 		     unsigned long maxnode)
13038bccd85fSChristoph Lameter {
13048bccd85fSChristoph Lameter 	unsigned long k;
13058bccd85fSChristoph Lameter 	unsigned long nlongs;
13068bccd85fSChristoph Lameter 	unsigned long endmask;
13078bccd85fSChristoph Lameter 
13088bccd85fSChristoph Lameter 	--maxnode;
13098bccd85fSChristoph Lameter 	nodes_clear(*nodes);
13108bccd85fSChristoph Lameter 	if (maxnode == 0 || !nmask)
13118bccd85fSChristoph Lameter 		return 0;
1312a9c930baSAndi Kleen 	if (maxnode > PAGE_SIZE*BITS_PER_BYTE)
1313636f13c1SChris Wright 		return -EINVAL;
13148bccd85fSChristoph Lameter 
13158bccd85fSChristoph Lameter 	nlongs = BITS_TO_LONGS(maxnode);
13168bccd85fSChristoph Lameter 	if ((maxnode % BITS_PER_LONG) == 0)
13178bccd85fSChristoph Lameter 		endmask = ~0UL;
13188bccd85fSChristoph Lameter 	else
13198bccd85fSChristoph Lameter 		endmask = (1UL << (maxnode % BITS_PER_LONG)) - 1;
13208bccd85fSChristoph Lameter 
13218bccd85fSChristoph Lameter 	/* When the user specified more nodes than supported just check
13228bccd85fSChristoph Lameter 	   if the non supported part is all zero. */
13238bccd85fSChristoph Lameter 	if (nlongs > BITS_TO_LONGS(MAX_NUMNODES)) {
13248bccd85fSChristoph Lameter 		if (nlongs > PAGE_SIZE/sizeof(long))
13258bccd85fSChristoph Lameter 			return -EINVAL;
13268bccd85fSChristoph Lameter 		for (k = BITS_TO_LONGS(MAX_NUMNODES); k < nlongs; k++) {
13278bccd85fSChristoph Lameter 			unsigned long t;
13288bccd85fSChristoph Lameter 			if (get_user(t, nmask + k))
13298bccd85fSChristoph Lameter 				return -EFAULT;
13308bccd85fSChristoph Lameter 			if (k == nlongs - 1) {
13318bccd85fSChristoph Lameter 				if (t & endmask)
13328bccd85fSChristoph Lameter 					return -EINVAL;
13338bccd85fSChristoph Lameter 			} else if (t)
13348bccd85fSChristoph Lameter 				return -EINVAL;
13358bccd85fSChristoph Lameter 		}
13368bccd85fSChristoph Lameter 		nlongs = BITS_TO_LONGS(MAX_NUMNODES);
13378bccd85fSChristoph Lameter 		endmask = ~0UL;
13388bccd85fSChristoph Lameter 	}
13398bccd85fSChristoph Lameter 
13408bccd85fSChristoph Lameter 	if (copy_from_user(nodes_addr(*nodes), nmask, nlongs*sizeof(unsigned long)))
13418bccd85fSChristoph Lameter 		return -EFAULT;
13428bccd85fSChristoph Lameter 	nodes_addr(*nodes)[nlongs-1] &= endmask;
13438bccd85fSChristoph Lameter 	return 0;
13448bccd85fSChristoph Lameter }
13458bccd85fSChristoph Lameter 
13468bccd85fSChristoph Lameter /* Copy a kernel node mask to user space */
13478bccd85fSChristoph Lameter static int copy_nodes_to_user(unsigned long __user *mask, unsigned long maxnode,
13488bccd85fSChristoph Lameter 			      nodemask_t *nodes)
13498bccd85fSChristoph Lameter {
13508bccd85fSChristoph Lameter 	unsigned long copy = ALIGN(maxnode-1, 64) / 8;
13518bccd85fSChristoph Lameter 	const int nbytes = BITS_TO_LONGS(MAX_NUMNODES) * sizeof(long);
13528bccd85fSChristoph Lameter 
13538bccd85fSChristoph Lameter 	if (copy > nbytes) {
13548bccd85fSChristoph Lameter 		if (copy > PAGE_SIZE)
13558bccd85fSChristoph Lameter 			return -EINVAL;
13568bccd85fSChristoph Lameter 		if (clear_user((char __user *)mask + nbytes, copy - nbytes))
13578bccd85fSChristoph Lameter 			return -EFAULT;
13588bccd85fSChristoph Lameter 		copy = nbytes;
13598bccd85fSChristoph Lameter 	}
13608bccd85fSChristoph Lameter 	return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0;
13618bccd85fSChristoph Lameter }
13628bccd85fSChristoph Lameter 
1363938bb9f5SHeiko Carstens SYSCALL_DEFINE6(mbind, unsigned long, start, unsigned long, len,
1364f7f28ca9SRasmus Villemoes 		unsigned long, mode, const unsigned long __user *, nmask,
1365938bb9f5SHeiko Carstens 		unsigned long, maxnode, unsigned, flags)
13668bccd85fSChristoph Lameter {
13678bccd85fSChristoph Lameter 	nodemask_t nodes;
13688bccd85fSChristoph Lameter 	int err;
1369028fec41SDavid Rientjes 	unsigned short mode_flags;
13708bccd85fSChristoph Lameter 
1371028fec41SDavid Rientjes 	mode_flags = mode & MPOL_MODE_FLAGS;
1372028fec41SDavid Rientjes 	mode &= ~MPOL_MODE_FLAGS;
1373a3b51e01SDavid Rientjes 	if (mode >= MPOL_MAX)
1374a3b51e01SDavid Rientjes 		return -EINVAL;
13754c50bc01SDavid Rientjes 	if ((mode_flags & MPOL_F_STATIC_NODES) &&
13764c50bc01SDavid Rientjes 	    (mode_flags & MPOL_F_RELATIVE_NODES))
13774c50bc01SDavid Rientjes 		return -EINVAL;
13788bccd85fSChristoph Lameter 	err = get_nodes(&nodes, nmask, maxnode);
13798bccd85fSChristoph Lameter 	if (err)
13808bccd85fSChristoph Lameter 		return err;
1381028fec41SDavid Rientjes 	return do_mbind(start, len, mode, mode_flags, &nodes, flags);
13828bccd85fSChristoph Lameter }
13838bccd85fSChristoph Lameter 
13848bccd85fSChristoph Lameter /* Set the process memory policy */
138523c8902dSRasmus Villemoes SYSCALL_DEFINE3(set_mempolicy, int, mode, const unsigned long __user *, nmask,
1386938bb9f5SHeiko Carstens 		unsigned long, maxnode)
13878bccd85fSChristoph Lameter {
13888bccd85fSChristoph Lameter 	int err;
13898bccd85fSChristoph Lameter 	nodemask_t nodes;
1390028fec41SDavid Rientjes 	unsigned short flags;
13918bccd85fSChristoph Lameter 
1392028fec41SDavid Rientjes 	flags = mode & MPOL_MODE_FLAGS;
1393028fec41SDavid Rientjes 	mode &= ~MPOL_MODE_FLAGS;
1394028fec41SDavid Rientjes 	if ((unsigned int)mode >= MPOL_MAX)
13958bccd85fSChristoph Lameter 		return -EINVAL;
13964c50bc01SDavid Rientjes 	if ((flags & MPOL_F_STATIC_NODES) && (flags & MPOL_F_RELATIVE_NODES))
13974c50bc01SDavid Rientjes 		return -EINVAL;
13988bccd85fSChristoph Lameter 	err = get_nodes(&nodes, nmask, maxnode);
13998bccd85fSChristoph Lameter 	if (err)
14008bccd85fSChristoph Lameter 		return err;
1401028fec41SDavid Rientjes 	return do_set_mempolicy(mode, flags, &nodes);
14028bccd85fSChristoph Lameter }
14038bccd85fSChristoph Lameter 
1404938bb9f5SHeiko Carstens SYSCALL_DEFINE4(migrate_pages, pid_t, pid, unsigned long, maxnode,
1405938bb9f5SHeiko Carstens 		const unsigned long __user *, old_nodes,
1406938bb9f5SHeiko Carstens 		const unsigned long __user *, new_nodes)
140739743889SChristoph Lameter {
1408c69e8d9cSDavid Howells 	const struct cred *cred = current_cred(), *tcred;
1409596d7cfaSKOSAKI Motohiro 	struct mm_struct *mm = NULL;
141039743889SChristoph Lameter 	struct task_struct *task;
141139743889SChristoph Lameter 	nodemask_t task_nodes;
141239743889SChristoph Lameter 	int err;
1413596d7cfaSKOSAKI Motohiro 	nodemask_t *old;
1414596d7cfaSKOSAKI Motohiro 	nodemask_t *new;
1415596d7cfaSKOSAKI Motohiro 	NODEMASK_SCRATCH(scratch);
141639743889SChristoph Lameter 
1417596d7cfaSKOSAKI Motohiro 	if (!scratch)
1418596d7cfaSKOSAKI Motohiro 		return -ENOMEM;
141939743889SChristoph Lameter 
1420596d7cfaSKOSAKI Motohiro 	old = &scratch->mask1;
1421596d7cfaSKOSAKI Motohiro 	new = &scratch->mask2;
1422596d7cfaSKOSAKI Motohiro 
1423596d7cfaSKOSAKI Motohiro 	err = get_nodes(old, old_nodes, maxnode);
142439743889SChristoph Lameter 	if (err)
1425596d7cfaSKOSAKI Motohiro 		goto out;
1426596d7cfaSKOSAKI Motohiro 
1427596d7cfaSKOSAKI Motohiro 	err = get_nodes(new, new_nodes, maxnode);
1428596d7cfaSKOSAKI Motohiro 	if (err)
1429596d7cfaSKOSAKI Motohiro 		goto out;
143039743889SChristoph Lameter 
143139743889SChristoph Lameter 	/* Find the mm_struct */
143255cfaa3cSZeng Zhaoming 	rcu_read_lock();
1433228ebcbeSPavel Emelyanov 	task = pid ? find_task_by_vpid(pid) : current;
143439743889SChristoph Lameter 	if (!task) {
143555cfaa3cSZeng Zhaoming 		rcu_read_unlock();
1436596d7cfaSKOSAKI Motohiro 		err = -ESRCH;
1437596d7cfaSKOSAKI Motohiro 		goto out;
143839743889SChristoph Lameter 	}
14393268c63eSChristoph Lameter 	get_task_struct(task);
144039743889SChristoph Lameter 
1441596d7cfaSKOSAKI Motohiro 	err = -EINVAL;
144239743889SChristoph Lameter 
144339743889SChristoph Lameter 	/*
144439743889SChristoph Lameter 	 * Check if this process has the right to modify the specified
144539743889SChristoph Lameter 	 * process. The right exists if the process has administrative
14467f927fccSAlexey Dobriyan 	 * capabilities, superuser privileges or the same
144739743889SChristoph Lameter 	 * userid as the target process.
144839743889SChristoph Lameter 	 */
1449c69e8d9cSDavid Howells 	tcred = __task_cred(task);
1450b38a86ebSEric W. Biederman 	if (!uid_eq(cred->euid, tcred->suid) && !uid_eq(cred->euid, tcred->uid) &&
1451b38a86ebSEric W. Biederman 	    !uid_eq(cred->uid,  tcred->suid) && !uid_eq(cred->uid,  tcred->uid) &&
145274c00241SChristoph Lameter 	    !capable(CAP_SYS_NICE)) {
1453c69e8d9cSDavid Howells 		rcu_read_unlock();
145439743889SChristoph Lameter 		err = -EPERM;
14553268c63eSChristoph Lameter 		goto out_put;
145639743889SChristoph Lameter 	}
1457c69e8d9cSDavid Howells 	rcu_read_unlock();
145839743889SChristoph Lameter 
145939743889SChristoph Lameter 	task_nodes = cpuset_mems_allowed(task);
146039743889SChristoph Lameter 	/* Is the user allowed to access the target nodes? */
1461596d7cfaSKOSAKI Motohiro 	if (!nodes_subset(*new, task_nodes) && !capable(CAP_SYS_NICE)) {
146239743889SChristoph Lameter 		err = -EPERM;
14633268c63eSChristoph Lameter 		goto out_put;
146439743889SChristoph Lameter 	}
146539743889SChristoph Lameter 
146601f13bd6SLai Jiangshan 	if (!nodes_subset(*new, node_states[N_MEMORY])) {
14673b42d28bSChristoph Lameter 		err = -EINVAL;
14683268c63eSChristoph Lameter 		goto out_put;
14693b42d28bSChristoph Lameter 	}
14703b42d28bSChristoph Lameter 
147186c3a764SDavid Quigley 	err = security_task_movememory(task);
147286c3a764SDavid Quigley 	if (err)
14733268c63eSChristoph Lameter 		goto out_put;
147486c3a764SDavid Quigley 
14753268c63eSChristoph Lameter 	mm = get_task_mm(task);
14763268c63eSChristoph Lameter 	put_task_struct(task);
1477f2a9ef88SSasha Levin 
1478f2a9ef88SSasha Levin 	if (!mm) {
1479f2a9ef88SSasha Levin 		err = -EINVAL;
1480f2a9ef88SSasha Levin 		goto out;
1481f2a9ef88SSasha Levin 	}
1482f2a9ef88SSasha Levin 
1483596d7cfaSKOSAKI Motohiro 	err = do_migrate_pages(mm, old, new,
148474c00241SChristoph Lameter 		capable(CAP_SYS_NICE) ? MPOL_MF_MOVE_ALL : MPOL_MF_MOVE);
14853268c63eSChristoph Lameter 
148639743889SChristoph Lameter 	mmput(mm);
14873268c63eSChristoph Lameter out:
1488596d7cfaSKOSAKI Motohiro 	NODEMASK_SCRATCH_FREE(scratch);
1489596d7cfaSKOSAKI Motohiro 
149039743889SChristoph Lameter 	return err;
14913268c63eSChristoph Lameter 
14923268c63eSChristoph Lameter out_put:
14933268c63eSChristoph Lameter 	put_task_struct(task);
14943268c63eSChristoph Lameter 	goto out;
14953268c63eSChristoph Lameter 
149639743889SChristoph Lameter }
149739743889SChristoph Lameter 
149839743889SChristoph Lameter 
14998bccd85fSChristoph Lameter /* Retrieve NUMA policy */
1500938bb9f5SHeiko Carstens SYSCALL_DEFINE5(get_mempolicy, int __user *, policy,
1501938bb9f5SHeiko Carstens 		unsigned long __user *, nmask, unsigned long, maxnode,
1502938bb9f5SHeiko Carstens 		unsigned long, addr, unsigned long, flags)
15038bccd85fSChristoph Lameter {
1504dbcb0f19SAdrian Bunk 	int err;
1505dbcb0f19SAdrian Bunk 	int uninitialized_var(pval);
15068bccd85fSChristoph Lameter 	nodemask_t nodes;
15078bccd85fSChristoph Lameter 
15088bccd85fSChristoph Lameter 	if (nmask != NULL && maxnode < MAX_NUMNODES)
15098bccd85fSChristoph Lameter 		return -EINVAL;
15108bccd85fSChristoph Lameter 
15118bccd85fSChristoph Lameter 	err = do_get_mempolicy(&pval, &nodes, addr, flags);
15128bccd85fSChristoph Lameter 
15138bccd85fSChristoph Lameter 	if (err)
15148bccd85fSChristoph Lameter 		return err;
15158bccd85fSChristoph Lameter 
15168bccd85fSChristoph Lameter 	if (policy && put_user(pval, policy))
15178bccd85fSChristoph Lameter 		return -EFAULT;
15188bccd85fSChristoph Lameter 
15198bccd85fSChristoph Lameter 	if (nmask)
15208bccd85fSChristoph Lameter 		err = copy_nodes_to_user(nmask, maxnode, &nodes);
15218bccd85fSChristoph Lameter 
15228bccd85fSChristoph Lameter 	return err;
15238bccd85fSChristoph Lameter }
15248bccd85fSChristoph Lameter 
15251da177e4SLinus Torvalds #ifdef CONFIG_COMPAT
15261da177e4SLinus Torvalds 
1527c93e0f6cSHeiko Carstens COMPAT_SYSCALL_DEFINE5(get_mempolicy, int __user *, policy,
1528c93e0f6cSHeiko Carstens 		       compat_ulong_t __user *, nmask,
1529c93e0f6cSHeiko Carstens 		       compat_ulong_t, maxnode,
1530c93e0f6cSHeiko Carstens 		       compat_ulong_t, addr, compat_ulong_t, flags)
15311da177e4SLinus Torvalds {
15321da177e4SLinus Torvalds 	long err;
15331da177e4SLinus Torvalds 	unsigned long __user *nm = NULL;
15341da177e4SLinus Torvalds 	unsigned long nr_bits, alloc_size;
15351da177e4SLinus Torvalds 	DECLARE_BITMAP(bm, MAX_NUMNODES);
15361da177e4SLinus Torvalds 
15371da177e4SLinus Torvalds 	nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
15381da177e4SLinus Torvalds 	alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
15391da177e4SLinus Torvalds 
15401da177e4SLinus Torvalds 	if (nmask)
15411da177e4SLinus Torvalds 		nm = compat_alloc_user_space(alloc_size);
15421da177e4SLinus Torvalds 
15431da177e4SLinus Torvalds 	err = sys_get_mempolicy(policy, nm, nr_bits+1, addr, flags);
15441da177e4SLinus Torvalds 
15451da177e4SLinus Torvalds 	if (!err && nmask) {
15462bbff6c7SKAMEZAWA Hiroyuki 		unsigned long copy_size;
15472bbff6c7SKAMEZAWA Hiroyuki 		copy_size = min_t(unsigned long, sizeof(bm), alloc_size);
15482bbff6c7SKAMEZAWA Hiroyuki 		err = copy_from_user(bm, nm, copy_size);
15491da177e4SLinus Torvalds 		/* ensure entire bitmap is zeroed */
15501da177e4SLinus Torvalds 		err |= clear_user(nmask, ALIGN(maxnode-1, 8) / 8);
15511da177e4SLinus Torvalds 		err |= compat_put_bitmap(nmask, bm, nr_bits);
15521da177e4SLinus Torvalds 	}
15531da177e4SLinus Torvalds 
15541da177e4SLinus Torvalds 	return err;
15551da177e4SLinus Torvalds }
15561da177e4SLinus Torvalds 
1557c93e0f6cSHeiko Carstens COMPAT_SYSCALL_DEFINE3(set_mempolicy, int, mode, compat_ulong_t __user *, nmask,
1558c93e0f6cSHeiko Carstens 		       compat_ulong_t, maxnode)
15591da177e4SLinus Torvalds {
15601da177e4SLinus Torvalds 	long err = 0;
15611da177e4SLinus Torvalds 	unsigned long __user *nm = NULL;
15621da177e4SLinus Torvalds 	unsigned long nr_bits, alloc_size;
15631da177e4SLinus Torvalds 	DECLARE_BITMAP(bm, MAX_NUMNODES);
15641da177e4SLinus Torvalds 
15651da177e4SLinus Torvalds 	nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
15661da177e4SLinus Torvalds 	alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
15671da177e4SLinus Torvalds 
15681da177e4SLinus Torvalds 	if (nmask) {
15691da177e4SLinus Torvalds 		err = compat_get_bitmap(bm, nmask, nr_bits);
15701da177e4SLinus Torvalds 		nm = compat_alloc_user_space(alloc_size);
15711da177e4SLinus Torvalds 		err |= copy_to_user(nm, bm, alloc_size);
15721da177e4SLinus Torvalds 	}
15731da177e4SLinus Torvalds 
15741da177e4SLinus Torvalds 	if (err)
15751da177e4SLinus Torvalds 		return -EFAULT;
15761da177e4SLinus Torvalds 
15771da177e4SLinus Torvalds 	return sys_set_mempolicy(mode, nm, nr_bits+1);
15781da177e4SLinus Torvalds }
15791da177e4SLinus Torvalds 
1580c93e0f6cSHeiko Carstens COMPAT_SYSCALL_DEFINE6(mbind, compat_ulong_t, start, compat_ulong_t, len,
1581c93e0f6cSHeiko Carstens 		       compat_ulong_t, mode, compat_ulong_t __user *, nmask,
1582c93e0f6cSHeiko Carstens 		       compat_ulong_t, maxnode, compat_ulong_t, flags)
15831da177e4SLinus Torvalds {
15841da177e4SLinus Torvalds 	long err = 0;
15851da177e4SLinus Torvalds 	unsigned long __user *nm = NULL;
15861da177e4SLinus Torvalds 	unsigned long nr_bits, alloc_size;
1587dfcd3c0dSAndi Kleen 	nodemask_t bm;
15881da177e4SLinus Torvalds 
15891da177e4SLinus Torvalds 	nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
15901da177e4SLinus Torvalds 	alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
15911da177e4SLinus Torvalds 
15921da177e4SLinus Torvalds 	if (nmask) {
1593dfcd3c0dSAndi Kleen 		err = compat_get_bitmap(nodes_addr(bm), nmask, nr_bits);
15941da177e4SLinus Torvalds 		nm = compat_alloc_user_space(alloc_size);
1595dfcd3c0dSAndi Kleen 		err |= copy_to_user(nm, nodes_addr(bm), alloc_size);
15961da177e4SLinus Torvalds 	}
15971da177e4SLinus Torvalds 
15981da177e4SLinus Torvalds 	if (err)
15991da177e4SLinus Torvalds 		return -EFAULT;
16001da177e4SLinus Torvalds 
16011da177e4SLinus Torvalds 	return sys_mbind(start, len, mode, nm, nr_bits+1, flags);
16021da177e4SLinus Torvalds }
16031da177e4SLinus Torvalds 
16041da177e4SLinus Torvalds #endif
16051da177e4SLinus Torvalds 
1606480eccf9SLee Schermerhorn /*
1607480eccf9SLee Schermerhorn  * get_vma_policy(@task, @vma, @addr)
1608b46e14acSFabian Frederick  * @task: task for fallback if vma policy == default
1609b46e14acSFabian Frederick  * @vma: virtual memory area whose policy is sought
1610b46e14acSFabian Frederick  * @addr: address in @vma for shared policy lookup
1611480eccf9SLee Schermerhorn  *
1612480eccf9SLee Schermerhorn  * Returns effective policy for a VMA at specified address.
1613480eccf9SLee Schermerhorn  * Falls back to @task or system default policy, as necessary.
161432f8516aSDavid Rientjes  * Current or other task's task mempolicy and non-shared vma policies must be
161532f8516aSDavid Rientjes  * protected by task_lock(task) by the caller.
161652cd3b07SLee Schermerhorn  * Shared policies [those marked as MPOL_F_SHARED] require an extra reference
161752cd3b07SLee Schermerhorn  * count--added by the get_policy() vm_op, as appropriate--to protect against
161852cd3b07SLee Schermerhorn  * freeing by another task.  It is the caller's responsibility to free the
161952cd3b07SLee Schermerhorn  * extra reference for shared policies.
1620480eccf9SLee Schermerhorn  */
1621d98f6cb6SStephen Wilson struct mempolicy *get_vma_policy(struct task_struct *task,
162248fce342SChristoph Lameter 		struct vm_area_struct *vma, unsigned long addr)
16231da177e4SLinus Torvalds {
16245606e387SMel Gorman 	struct mempolicy *pol = get_task_policy(task);
16251da177e4SLinus Torvalds 
16261da177e4SLinus Torvalds 	if (vma) {
1627480eccf9SLee Schermerhorn 		if (vma->vm_ops && vma->vm_ops->get_policy) {
1628ae4d8c16SLee Schermerhorn 			struct mempolicy *vpol = vma->vm_ops->get_policy(vma,
1629ae4d8c16SLee Schermerhorn 									addr);
1630ae4d8c16SLee Schermerhorn 			if (vpol)
1631ae4d8c16SLee Schermerhorn 				pol = vpol;
163200442ad0SMel Gorman 		} else if (vma->vm_policy) {
16331da177e4SLinus Torvalds 			pol = vma->vm_policy;
163400442ad0SMel Gorman 
163500442ad0SMel Gorman 			/*
163600442ad0SMel Gorman 			 * shmem_alloc_page() passes MPOL_F_SHARED policy with
163700442ad0SMel Gorman 			 * a pseudo vma whose vma->vm_ops=NULL. Take a reference
163800442ad0SMel Gorman 			 * count on these policies which will be dropped by
163900442ad0SMel Gorman 			 * mpol_cond_put() later
164000442ad0SMel Gorman 			 */
164100442ad0SMel Gorman 			if (mpol_needs_cond_ref(pol))
164200442ad0SMel Gorman 				mpol_get(pol);
164300442ad0SMel Gorman 		}
16441da177e4SLinus Torvalds 	}
1645*f15ca78eSOleg Nesterov 
16461da177e4SLinus Torvalds 	return pol;
16471da177e4SLinus Torvalds }
16481da177e4SLinus Torvalds 
1649fc314724SMel Gorman bool vma_policy_mof(struct task_struct *task, struct vm_area_struct *vma)
1650fc314724SMel Gorman {
1651fc314724SMel Gorman 	struct mempolicy *pol = get_task_policy(task);
1652*f15ca78eSOleg Nesterov 
1653fc314724SMel Gorman 	if (vma) {
1654fc314724SMel Gorman 		if (vma->vm_ops && vma->vm_ops->get_policy) {
1655fc314724SMel Gorman 			bool ret = false;
1656fc314724SMel Gorman 
1657fc314724SMel Gorman 			pol = vma->vm_ops->get_policy(vma, vma->vm_start);
1658fc314724SMel Gorman 			if (pol && (pol->flags & MPOL_F_MOF))
1659fc314724SMel Gorman 				ret = true;
1660fc314724SMel Gorman 			mpol_cond_put(pol);
1661fc314724SMel Gorman 
1662fc314724SMel Gorman 			return ret;
1663fc314724SMel Gorman 		} else if (vma->vm_policy) {
1664fc314724SMel Gorman 			pol = vma->vm_policy;
1665fc314724SMel Gorman 		}
1666fc314724SMel Gorman 	}
1667fc314724SMel Gorman 
1668fc314724SMel Gorman 	return pol->flags & MPOL_F_MOF;
1669fc314724SMel Gorman }
1670fc314724SMel Gorman 
1671d3eb1570SLai Jiangshan static int apply_policy_zone(struct mempolicy *policy, enum zone_type zone)
1672d3eb1570SLai Jiangshan {
1673d3eb1570SLai Jiangshan 	enum zone_type dynamic_policy_zone = policy_zone;
1674d3eb1570SLai Jiangshan 
1675d3eb1570SLai Jiangshan 	BUG_ON(dynamic_policy_zone == ZONE_MOVABLE);
1676d3eb1570SLai Jiangshan 
1677d3eb1570SLai Jiangshan 	/*
1678d3eb1570SLai Jiangshan 	 * if policy->v.nodes has movable memory only,
1679d3eb1570SLai Jiangshan 	 * we apply policy when gfp_zone(gfp) = ZONE_MOVABLE only.
1680d3eb1570SLai Jiangshan 	 *
1681d3eb1570SLai Jiangshan 	 * policy->v.nodes is intersect with node_states[N_MEMORY].
1682d3eb1570SLai Jiangshan 	 * so if the following test faile, it implies
1683d3eb1570SLai Jiangshan 	 * policy->v.nodes has movable memory only.
1684d3eb1570SLai Jiangshan 	 */
1685d3eb1570SLai Jiangshan 	if (!nodes_intersects(policy->v.nodes, node_states[N_HIGH_MEMORY]))
1686d3eb1570SLai Jiangshan 		dynamic_policy_zone = ZONE_MOVABLE;
1687d3eb1570SLai Jiangshan 
1688d3eb1570SLai Jiangshan 	return zone >= dynamic_policy_zone;
1689d3eb1570SLai Jiangshan }
1690d3eb1570SLai Jiangshan 
169152cd3b07SLee Schermerhorn /*
169252cd3b07SLee Schermerhorn  * Return a nodemask representing a mempolicy for filtering nodes for
169352cd3b07SLee Schermerhorn  * page allocation
169452cd3b07SLee Schermerhorn  */
169552cd3b07SLee Schermerhorn static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *policy)
169619770b32SMel Gorman {
169719770b32SMel Gorman 	/* Lower zones don't get a nodemask applied for MPOL_BIND */
169845c4745aSLee Schermerhorn 	if (unlikely(policy->mode == MPOL_BIND) &&
1699d3eb1570SLai Jiangshan 			apply_policy_zone(policy, gfp_zone(gfp)) &&
170019770b32SMel Gorman 			cpuset_nodemask_valid_mems_allowed(&policy->v.nodes))
170119770b32SMel Gorman 		return &policy->v.nodes;
170219770b32SMel Gorman 
170319770b32SMel Gorman 	return NULL;
170419770b32SMel Gorman }
170519770b32SMel Gorman 
170652cd3b07SLee Schermerhorn /* Return a zonelist indicated by gfp for node representing a mempolicy */
17072f5f9486SAndi Kleen static struct zonelist *policy_zonelist(gfp_t gfp, struct mempolicy *policy,
17082f5f9486SAndi Kleen 	int nd)
17091da177e4SLinus Torvalds {
171045c4745aSLee Schermerhorn 	switch (policy->mode) {
17111da177e4SLinus Torvalds 	case MPOL_PREFERRED:
1712fc36b8d3SLee Schermerhorn 		if (!(policy->flags & MPOL_F_LOCAL))
17131da177e4SLinus Torvalds 			nd = policy->v.preferred_node;
17141da177e4SLinus Torvalds 		break;
17151da177e4SLinus Torvalds 	case MPOL_BIND:
171619770b32SMel Gorman 		/*
171752cd3b07SLee Schermerhorn 		 * Normally, MPOL_BIND allocations are node-local within the
171852cd3b07SLee Schermerhorn 		 * allowed nodemask.  However, if __GFP_THISNODE is set and the
17196eb27e1fSBob Liu 		 * current node isn't part of the mask, we use the zonelist for
172052cd3b07SLee Schermerhorn 		 * the first node in the mask instead.
172119770b32SMel Gorman 		 */
172219770b32SMel Gorman 		if (unlikely(gfp & __GFP_THISNODE) &&
172319770b32SMel Gorman 				unlikely(!node_isset(nd, policy->v.nodes)))
172419770b32SMel Gorman 			nd = first_node(policy->v.nodes);
172519770b32SMel Gorman 		break;
17261da177e4SLinus Torvalds 	default:
17271da177e4SLinus Torvalds 		BUG();
17281da177e4SLinus Torvalds 	}
17290e88460dSMel Gorman 	return node_zonelist(nd, gfp);
17301da177e4SLinus Torvalds }
17311da177e4SLinus Torvalds 
17321da177e4SLinus Torvalds /* Do dynamic interleaving for a process */
17331da177e4SLinus Torvalds static unsigned interleave_nodes(struct mempolicy *policy)
17341da177e4SLinus Torvalds {
17351da177e4SLinus Torvalds 	unsigned nid, next;
17361da177e4SLinus Torvalds 	struct task_struct *me = current;
17371da177e4SLinus Torvalds 
17381da177e4SLinus Torvalds 	nid = me->il_next;
1739dfcd3c0dSAndi Kleen 	next = next_node(nid, policy->v.nodes);
17401da177e4SLinus Torvalds 	if (next >= MAX_NUMNODES)
1741dfcd3c0dSAndi Kleen 		next = first_node(policy->v.nodes);
1742f5b087b5SDavid Rientjes 	if (next < MAX_NUMNODES)
17431da177e4SLinus Torvalds 		me->il_next = next;
17441da177e4SLinus Torvalds 	return nid;
17451da177e4SLinus Torvalds }
17461da177e4SLinus Torvalds 
1747dc85da15SChristoph Lameter /*
1748dc85da15SChristoph Lameter  * Depending on the memory policy provide a node from which to allocate the
1749dc85da15SChristoph Lameter  * next slab entry.
1750dc85da15SChristoph Lameter  */
17512a389610SDavid Rientjes unsigned int mempolicy_slab_node(void)
1752dc85da15SChristoph Lameter {
1753e7b691b0SAndi Kleen 	struct mempolicy *policy;
17542a389610SDavid Rientjes 	int node = numa_mem_id();
1755e7b691b0SAndi Kleen 
1756e7b691b0SAndi Kleen 	if (in_interrupt())
17572a389610SDavid Rientjes 		return node;
1758e7b691b0SAndi Kleen 
1759e7b691b0SAndi Kleen 	policy = current->mempolicy;
1760fc36b8d3SLee Schermerhorn 	if (!policy || policy->flags & MPOL_F_LOCAL)
17612a389610SDavid Rientjes 		return node;
1762765c4507SChristoph Lameter 
1763bea904d5SLee Schermerhorn 	switch (policy->mode) {
1764bea904d5SLee Schermerhorn 	case MPOL_PREFERRED:
1765fc36b8d3SLee Schermerhorn 		/*
1766fc36b8d3SLee Schermerhorn 		 * handled MPOL_F_LOCAL above
1767fc36b8d3SLee Schermerhorn 		 */
1768bea904d5SLee Schermerhorn 		return policy->v.preferred_node;
1769bea904d5SLee Schermerhorn 
1770dc85da15SChristoph Lameter 	case MPOL_INTERLEAVE:
1771dc85da15SChristoph Lameter 		return interleave_nodes(policy);
1772dc85da15SChristoph Lameter 
1773dd1a239fSMel Gorman 	case MPOL_BIND: {
1774dc85da15SChristoph Lameter 		/*
1775dc85da15SChristoph Lameter 		 * Follow bind policy behavior and start allocation at the
1776dc85da15SChristoph Lameter 		 * first node.
1777dc85da15SChristoph Lameter 		 */
177819770b32SMel Gorman 		struct zonelist *zonelist;
177919770b32SMel Gorman 		struct zone *zone;
178019770b32SMel Gorman 		enum zone_type highest_zoneidx = gfp_zone(GFP_KERNEL);
17812a389610SDavid Rientjes 		zonelist = &NODE_DATA(node)->node_zonelists[0];
178219770b32SMel Gorman 		(void)first_zones_zonelist(zonelist, highest_zoneidx,
178319770b32SMel Gorman 							&policy->v.nodes,
178419770b32SMel Gorman 							&zone);
17852a389610SDavid Rientjes 		return zone ? zone->node : node;
1786dd1a239fSMel Gorman 	}
1787dc85da15SChristoph Lameter 
1788dc85da15SChristoph Lameter 	default:
1789bea904d5SLee Schermerhorn 		BUG();
1790dc85da15SChristoph Lameter 	}
1791dc85da15SChristoph Lameter }
1792dc85da15SChristoph Lameter 
17931da177e4SLinus Torvalds /* Do static interleaving for a VMA with known offset. */
17941da177e4SLinus Torvalds static unsigned offset_il_node(struct mempolicy *pol,
17951da177e4SLinus Torvalds 		struct vm_area_struct *vma, unsigned long off)
17961da177e4SLinus Torvalds {
1797dfcd3c0dSAndi Kleen 	unsigned nnodes = nodes_weight(pol->v.nodes);
1798f5b087b5SDavid Rientjes 	unsigned target;
17991da177e4SLinus Torvalds 	int c;
1800b76ac7e7SJianguo Wu 	int nid = NUMA_NO_NODE;
18011da177e4SLinus Torvalds 
1802f5b087b5SDavid Rientjes 	if (!nnodes)
1803f5b087b5SDavid Rientjes 		return numa_node_id();
1804f5b087b5SDavid Rientjes 	target = (unsigned int)off % nnodes;
18051da177e4SLinus Torvalds 	c = 0;
18061da177e4SLinus Torvalds 	do {
1807dfcd3c0dSAndi Kleen 		nid = next_node(nid, pol->v.nodes);
18081da177e4SLinus Torvalds 		c++;
18091da177e4SLinus Torvalds 	} while (c <= target);
18101da177e4SLinus Torvalds 	return nid;
18111da177e4SLinus Torvalds }
18121da177e4SLinus Torvalds 
18135da7ca86SChristoph Lameter /* Determine a node number for interleave */
18145da7ca86SChristoph Lameter static inline unsigned interleave_nid(struct mempolicy *pol,
18155da7ca86SChristoph Lameter 		 struct vm_area_struct *vma, unsigned long addr, int shift)
18165da7ca86SChristoph Lameter {
18175da7ca86SChristoph Lameter 	if (vma) {
18185da7ca86SChristoph Lameter 		unsigned long off;
18195da7ca86SChristoph Lameter 
18203b98b087SNishanth Aravamudan 		/*
18213b98b087SNishanth Aravamudan 		 * for small pages, there is no difference between
18223b98b087SNishanth Aravamudan 		 * shift and PAGE_SHIFT, so the bit-shift is safe.
18233b98b087SNishanth Aravamudan 		 * for huge pages, since vm_pgoff is in units of small
18243b98b087SNishanth Aravamudan 		 * pages, we need to shift off the always 0 bits to get
18253b98b087SNishanth Aravamudan 		 * a useful offset.
18263b98b087SNishanth Aravamudan 		 */
18273b98b087SNishanth Aravamudan 		BUG_ON(shift < PAGE_SHIFT);
18283b98b087SNishanth Aravamudan 		off = vma->vm_pgoff >> (shift - PAGE_SHIFT);
18295da7ca86SChristoph Lameter 		off += (addr - vma->vm_start) >> shift;
18305da7ca86SChristoph Lameter 		return offset_il_node(pol, vma, off);
18315da7ca86SChristoph Lameter 	} else
18325da7ca86SChristoph Lameter 		return interleave_nodes(pol);
18335da7ca86SChristoph Lameter }
18345da7ca86SChristoph Lameter 
1835778d3b0fSMichal Hocko /*
1836778d3b0fSMichal Hocko  * Return the bit number of a random bit set in the nodemask.
1837b76ac7e7SJianguo Wu  * (returns NUMA_NO_NODE if nodemask is empty)
1838778d3b0fSMichal Hocko  */
1839778d3b0fSMichal Hocko int node_random(const nodemask_t *maskp)
1840778d3b0fSMichal Hocko {
1841b76ac7e7SJianguo Wu 	int w, bit = NUMA_NO_NODE;
1842778d3b0fSMichal Hocko 
1843778d3b0fSMichal Hocko 	w = nodes_weight(*maskp);
1844778d3b0fSMichal Hocko 	if (w)
1845778d3b0fSMichal Hocko 		bit = bitmap_ord_to_pos(maskp->bits,
1846778d3b0fSMichal Hocko 			get_random_int() % w, MAX_NUMNODES);
1847778d3b0fSMichal Hocko 	return bit;
1848778d3b0fSMichal Hocko }
1849778d3b0fSMichal Hocko 
185000ac59adSChen, Kenneth W #ifdef CONFIG_HUGETLBFS
1851480eccf9SLee Schermerhorn /*
1852480eccf9SLee Schermerhorn  * huge_zonelist(@vma, @addr, @gfp_flags, @mpol)
1853b46e14acSFabian Frederick  * @vma: virtual memory area whose policy is sought
1854b46e14acSFabian Frederick  * @addr: address in @vma for shared policy lookup and interleave policy
1855b46e14acSFabian Frederick  * @gfp_flags: for requested zone
1856b46e14acSFabian Frederick  * @mpol: pointer to mempolicy pointer for reference counted mempolicy
1857b46e14acSFabian Frederick  * @nodemask: pointer to nodemask pointer for MPOL_BIND nodemask
1858480eccf9SLee Schermerhorn  *
185952cd3b07SLee Schermerhorn  * Returns a zonelist suitable for a huge page allocation and a pointer
186052cd3b07SLee Schermerhorn  * to the struct mempolicy for conditional unref after allocation.
186152cd3b07SLee Schermerhorn  * If the effective policy is 'BIND, returns a pointer to the mempolicy's
186252cd3b07SLee Schermerhorn  * @nodemask for filtering the zonelist.
1863c0ff7453SMiao Xie  *
1864d26914d1SMel Gorman  * Must be protected by read_mems_allowed_begin()
1865480eccf9SLee Schermerhorn  */
1866396faf03SMel Gorman struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr,
186719770b32SMel Gorman 				gfp_t gfp_flags, struct mempolicy **mpol,
186819770b32SMel Gorman 				nodemask_t **nodemask)
18695da7ca86SChristoph Lameter {
1870480eccf9SLee Schermerhorn 	struct zonelist *zl;
18715da7ca86SChristoph Lameter 
187252cd3b07SLee Schermerhorn 	*mpol = get_vma_policy(current, vma, addr);
187319770b32SMel Gorman 	*nodemask = NULL;	/* assume !MPOL_BIND */
18745da7ca86SChristoph Lameter 
187552cd3b07SLee Schermerhorn 	if (unlikely((*mpol)->mode == MPOL_INTERLEAVE)) {
187652cd3b07SLee Schermerhorn 		zl = node_zonelist(interleave_nid(*mpol, vma, addr,
1877a5516438SAndi Kleen 				huge_page_shift(hstate_vma(vma))), gfp_flags);
187852cd3b07SLee Schermerhorn 	} else {
18792f5f9486SAndi Kleen 		zl = policy_zonelist(gfp_flags, *mpol, numa_node_id());
188052cd3b07SLee Schermerhorn 		if ((*mpol)->mode == MPOL_BIND)
188152cd3b07SLee Schermerhorn 			*nodemask = &(*mpol)->v.nodes;
1882480eccf9SLee Schermerhorn 	}
1883480eccf9SLee Schermerhorn 	return zl;
18845da7ca86SChristoph Lameter }
188506808b08SLee Schermerhorn 
188606808b08SLee Schermerhorn /*
188706808b08SLee Schermerhorn  * init_nodemask_of_mempolicy
188806808b08SLee Schermerhorn  *
188906808b08SLee Schermerhorn  * If the current task's mempolicy is "default" [NULL], return 'false'
189006808b08SLee Schermerhorn  * to indicate default policy.  Otherwise, extract the policy nodemask
189106808b08SLee Schermerhorn  * for 'bind' or 'interleave' policy into the argument nodemask, or
189206808b08SLee Schermerhorn  * initialize the argument nodemask to contain the single node for
189306808b08SLee Schermerhorn  * 'preferred' or 'local' policy and return 'true' to indicate presence
189406808b08SLee Schermerhorn  * of non-default mempolicy.
189506808b08SLee Schermerhorn  *
189606808b08SLee Schermerhorn  * We don't bother with reference counting the mempolicy [mpol_get/put]
189706808b08SLee Schermerhorn  * because the current task is examining it's own mempolicy and a task's
189806808b08SLee Schermerhorn  * mempolicy is only ever changed by the task itself.
189906808b08SLee Schermerhorn  *
190006808b08SLee Schermerhorn  * N.B., it is the caller's responsibility to free a returned nodemask.
190106808b08SLee Schermerhorn  */
190206808b08SLee Schermerhorn bool init_nodemask_of_mempolicy(nodemask_t *mask)
190306808b08SLee Schermerhorn {
190406808b08SLee Schermerhorn 	struct mempolicy *mempolicy;
190506808b08SLee Schermerhorn 	int nid;
190606808b08SLee Schermerhorn 
190706808b08SLee Schermerhorn 	if (!(mask && current->mempolicy))
190806808b08SLee Schermerhorn 		return false;
190906808b08SLee Schermerhorn 
1910c0ff7453SMiao Xie 	task_lock(current);
191106808b08SLee Schermerhorn 	mempolicy = current->mempolicy;
191206808b08SLee Schermerhorn 	switch (mempolicy->mode) {
191306808b08SLee Schermerhorn 	case MPOL_PREFERRED:
191406808b08SLee Schermerhorn 		if (mempolicy->flags & MPOL_F_LOCAL)
191506808b08SLee Schermerhorn 			nid = numa_node_id();
191606808b08SLee Schermerhorn 		else
191706808b08SLee Schermerhorn 			nid = mempolicy->v.preferred_node;
191806808b08SLee Schermerhorn 		init_nodemask_of_node(mask, nid);
191906808b08SLee Schermerhorn 		break;
192006808b08SLee Schermerhorn 
192106808b08SLee Schermerhorn 	case MPOL_BIND:
192206808b08SLee Schermerhorn 		/* Fall through */
192306808b08SLee Schermerhorn 	case MPOL_INTERLEAVE:
192406808b08SLee Schermerhorn 		*mask =  mempolicy->v.nodes;
192506808b08SLee Schermerhorn 		break;
192606808b08SLee Schermerhorn 
192706808b08SLee Schermerhorn 	default:
192806808b08SLee Schermerhorn 		BUG();
192906808b08SLee Schermerhorn 	}
1930c0ff7453SMiao Xie 	task_unlock(current);
193106808b08SLee Schermerhorn 
193206808b08SLee Schermerhorn 	return true;
193306808b08SLee Schermerhorn }
193400ac59adSChen, Kenneth W #endif
19355da7ca86SChristoph Lameter 
19366f48d0ebSDavid Rientjes /*
19376f48d0ebSDavid Rientjes  * mempolicy_nodemask_intersects
19386f48d0ebSDavid Rientjes  *
19396f48d0ebSDavid Rientjes  * If tsk's mempolicy is "default" [NULL], return 'true' to indicate default
19406f48d0ebSDavid Rientjes  * policy.  Otherwise, check for intersection between mask and the policy
19416f48d0ebSDavid Rientjes  * nodemask for 'bind' or 'interleave' policy.  For 'perferred' or 'local'
19426f48d0ebSDavid Rientjes  * policy, always return true since it may allocate elsewhere on fallback.
19436f48d0ebSDavid Rientjes  *
19446f48d0ebSDavid Rientjes  * Takes task_lock(tsk) to prevent freeing of its mempolicy.
19456f48d0ebSDavid Rientjes  */
19466f48d0ebSDavid Rientjes bool mempolicy_nodemask_intersects(struct task_struct *tsk,
19476f48d0ebSDavid Rientjes 					const nodemask_t *mask)
19486f48d0ebSDavid Rientjes {
19496f48d0ebSDavid Rientjes 	struct mempolicy *mempolicy;
19506f48d0ebSDavid Rientjes 	bool ret = true;
19516f48d0ebSDavid Rientjes 
19526f48d0ebSDavid Rientjes 	if (!mask)
19536f48d0ebSDavid Rientjes 		return ret;
19546f48d0ebSDavid Rientjes 	task_lock(tsk);
19556f48d0ebSDavid Rientjes 	mempolicy = tsk->mempolicy;
19566f48d0ebSDavid Rientjes 	if (!mempolicy)
19576f48d0ebSDavid Rientjes 		goto out;
19586f48d0ebSDavid Rientjes 
19596f48d0ebSDavid Rientjes 	switch (mempolicy->mode) {
19606f48d0ebSDavid Rientjes 	case MPOL_PREFERRED:
19616f48d0ebSDavid Rientjes 		/*
19626f48d0ebSDavid Rientjes 		 * MPOL_PREFERRED and MPOL_F_LOCAL are only preferred nodes to
19636f48d0ebSDavid Rientjes 		 * allocate from, they may fallback to other nodes when oom.
19646f48d0ebSDavid Rientjes 		 * Thus, it's possible for tsk to have allocated memory from
19656f48d0ebSDavid Rientjes 		 * nodes in mask.
19666f48d0ebSDavid Rientjes 		 */
19676f48d0ebSDavid Rientjes 		break;
19686f48d0ebSDavid Rientjes 	case MPOL_BIND:
19696f48d0ebSDavid Rientjes 	case MPOL_INTERLEAVE:
19706f48d0ebSDavid Rientjes 		ret = nodes_intersects(mempolicy->v.nodes, *mask);
19716f48d0ebSDavid Rientjes 		break;
19726f48d0ebSDavid Rientjes 	default:
19736f48d0ebSDavid Rientjes 		BUG();
19746f48d0ebSDavid Rientjes 	}
19756f48d0ebSDavid Rientjes out:
19766f48d0ebSDavid Rientjes 	task_unlock(tsk);
19776f48d0ebSDavid Rientjes 	return ret;
19786f48d0ebSDavid Rientjes }
19796f48d0ebSDavid Rientjes 
19801da177e4SLinus Torvalds /* Allocate a page in interleaved policy.
19811da177e4SLinus Torvalds    Own path because it needs to do special accounting. */
1982662f3a0bSAndi Kleen static struct page *alloc_page_interleave(gfp_t gfp, unsigned order,
1983662f3a0bSAndi Kleen 					unsigned nid)
19841da177e4SLinus Torvalds {
19851da177e4SLinus Torvalds 	struct zonelist *zl;
19861da177e4SLinus Torvalds 	struct page *page;
19871da177e4SLinus Torvalds 
19880e88460dSMel Gorman 	zl = node_zonelist(nid, gfp);
19891da177e4SLinus Torvalds 	page = __alloc_pages(gfp, order, zl);
1990dd1a239fSMel Gorman 	if (page && page_zone(page) == zonelist_zone(&zl->_zonerefs[0]))
1991ca889e6cSChristoph Lameter 		inc_zone_page_state(page, NUMA_INTERLEAVE_HIT);
19921da177e4SLinus Torvalds 	return page;
19931da177e4SLinus Torvalds }
19941da177e4SLinus Torvalds 
19951da177e4SLinus Torvalds /**
19960bbbc0b3SAndrea Arcangeli  * 	alloc_pages_vma	- Allocate a page for a VMA.
19971da177e4SLinus Torvalds  *
19981da177e4SLinus Torvalds  * 	@gfp:
19991da177e4SLinus Torvalds  *      %GFP_USER    user allocation.
20001da177e4SLinus Torvalds  *      %GFP_KERNEL  kernel allocations,
20011da177e4SLinus Torvalds  *      %GFP_HIGHMEM highmem/user allocations,
20021da177e4SLinus Torvalds  *      %GFP_FS      allocation should not call back into a file system.
20031da177e4SLinus Torvalds  *      %GFP_ATOMIC  don't sleep.
20041da177e4SLinus Torvalds  *
20050bbbc0b3SAndrea Arcangeli  *	@order:Order of the GFP allocation.
20061da177e4SLinus Torvalds  * 	@vma:  Pointer to VMA or NULL if not available.
20071da177e4SLinus Torvalds  *	@addr: Virtual Address of the allocation. Must be inside the VMA.
20081da177e4SLinus Torvalds  *
20091da177e4SLinus Torvalds  * 	This function allocates a page from the kernel page pool and applies
20101da177e4SLinus Torvalds  *	a NUMA policy associated with the VMA or the current process.
20111da177e4SLinus Torvalds  *	When VMA is not NULL caller must hold down_read on the mmap_sem of the
20121da177e4SLinus Torvalds  *	mm_struct of the VMA to prevent it from going away. Should be used for
20131da177e4SLinus Torvalds  *	all allocations for pages that will be mapped into
20141da177e4SLinus Torvalds  * 	user space. Returns NULL when no page can be allocated.
20151da177e4SLinus Torvalds  *
20161da177e4SLinus Torvalds  *	Should be called with the mm_sem of the vma hold.
20171da177e4SLinus Torvalds  */
20181da177e4SLinus Torvalds struct page *
20190bbbc0b3SAndrea Arcangeli alloc_pages_vma(gfp_t gfp, int order, struct vm_area_struct *vma,
20202f5f9486SAndi Kleen 		unsigned long addr, int node)
20211da177e4SLinus Torvalds {
2022cc9a6c87SMel Gorman 	struct mempolicy *pol;
2023c0ff7453SMiao Xie 	struct page *page;
2024cc9a6c87SMel Gorman 	unsigned int cpuset_mems_cookie;
20251da177e4SLinus Torvalds 
2026cc9a6c87SMel Gorman retry_cpuset:
2027cc9a6c87SMel Gorman 	pol = get_vma_policy(current, vma, addr);
2028d26914d1SMel Gorman 	cpuset_mems_cookie = read_mems_allowed_begin();
2029cc9a6c87SMel Gorman 
203045c4745aSLee Schermerhorn 	if (unlikely(pol->mode == MPOL_INTERLEAVE)) {
20311da177e4SLinus Torvalds 		unsigned nid;
20325da7ca86SChristoph Lameter 
20338eac563cSAndi Kleen 		nid = interleave_nid(pol, vma, addr, PAGE_SHIFT + order);
203452cd3b07SLee Schermerhorn 		mpol_cond_put(pol);
20350bbbc0b3SAndrea Arcangeli 		page = alloc_page_interleave(gfp, order, nid);
2036d26914d1SMel Gorman 		if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
2037cc9a6c87SMel Gorman 			goto retry_cpuset;
2038cc9a6c87SMel Gorman 
2039c0ff7453SMiao Xie 		return page;
20401da177e4SLinus Torvalds 	}
2041212a0a6fSDavid Rientjes 	page = __alloc_pages_nodemask(gfp, order,
2042212a0a6fSDavid Rientjes 				      policy_zonelist(gfp, pol, node),
20430bbbc0b3SAndrea Arcangeli 				      policy_nodemask(gfp, pol));
20442386740dSOleg Nesterov 	mpol_cond_put(pol);
2045d26914d1SMel Gorman 	if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
2046cc9a6c87SMel Gorman 		goto retry_cpuset;
2047c0ff7453SMiao Xie 	return page;
20481da177e4SLinus Torvalds }
20491da177e4SLinus Torvalds 
20501da177e4SLinus Torvalds /**
20511da177e4SLinus Torvalds  * 	alloc_pages_current - Allocate pages.
20521da177e4SLinus Torvalds  *
20531da177e4SLinus Torvalds  *	@gfp:
20541da177e4SLinus Torvalds  *		%GFP_USER   user allocation,
20551da177e4SLinus Torvalds  *      	%GFP_KERNEL kernel allocation,
20561da177e4SLinus Torvalds  *      	%GFP_HIGHMEM highmem allocation,
20571da177e4SLinus Torvalds  *      	%GFP_FS     don't call back into a file system.
20581da177e4SLinus Torvalds  *      	%GFP_ATOMIC don't sleep.
20591da177e4SLinus Torvalds  *	@order: Power of two of allocation size in pages. 0 is a single page.
20601da177e4SLinus Torvalds  *
20611da177e4SLinus Torvalds  *	Allocate a page from the kernel page pool.  When not in
20621da177e4SLinus Torvalds  *	interrupt context and apply the current process NUMA policy.
20631da177e4SLinus Torvalds  *	Returns NULL when no page can be allocated.
20641da177e4SLinus Torvalds  *
2065cf2a473cSPaul Jackson  *	Don't call cpuset_update_task_memory_state() unless
20661da177e4SLinus Torvalds  *	1) it's ok to take cpuset_sem (can WAIT), and
20671da177e4SLinus Torvalds  *	2) allocating for current task (not interrupt).
20681da177e4SLinus Torvalds  */
2069dd0fc66fSAl Viro struct page *alloc_pages_current(gfp_t gfp, unsigned order)
20701da177e4SLinus Torvalds {
20715606e387SMel Gorman 	struct mempolicy *pol = get_task_policy(current);
2072c0ff7453SMiao Xie 	struct page *page;
2073cc9a6c87SMel Gorman 	unsigned int cpuset_mems_cookie;
20741da177e4SLinus Torvalds 
2075*f15ca78eSOleg Nesterov 	if (in_interrupt() || (gfp & __GFP_THISNODE))
20761da177e4SLinus Torvalds 		pol = &default_policy;
207752cd3b07SLee Schermerhorn 
2078cc9a6c87SMel Gorman retry_cpuset:
2079d26914d1SMel Gorman 	cpuset_mems_cookie = read_mems_allowed_begin();
2080cc9a6c87SMel Gorman 
208152cd3b07SLee Schermerhorn 	/*
208252cd3b07SLee Schermerhorn 	 * No reference counting needed for current->mempolicy
208352cd3b07SLee Schermerhorn 	 * nor system default_policy
208452cd3b07SLee Schermerhorn 	 */
208545c4745aSLee Schermerhorn 	if (pol->mode == MPOL_INTERLEAVE)
2086c0ff7453SMiao Xie 		page = alloc_page_interleave(gfp, order, interleave_nodes(pol));
2087c0ff7453SMiao Xie 	else
2088c0ff7453SMiao Xie 		page = __alloc_pages_nodemask(gfp, order,
20895c4b4be3SAndi Kleen 				policy_zonelist(gfp, pol, numa_node_id()),
20905c4b4be3SAndi Kleen 				policy_nodemask(gfp, pol));
2091cc9a6c87SMel Gorman 
2092d26914d1SMel Gorman 	if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
2093cc9a6c87SMel Gorman 		goto retry_cpuset;
2094cc9a6c87SMel Gorman 
2095c0ff7453SMiao Xie 	return page;
20961da177e4SLinus Torvalds }
20971da177e4SLinus Torvalds EXPORT_SYMBOL(alloc_pages_current);
20981da177e4SLinus Torvalds 
2099ef0855d3SOleg Nesterov int vma_dup_policy(struct vm_area_struct *src, struct vm_area_struct *dst)
2100ef0855d3SOleg Nesterov {
2101ef0855d3SOleg Nesterov 	struct mempolicy *pol = mpol_dup(vma_policy(src));
2102ef0855d3SOleg Nesterov 
2103ef0855d3SOleg Nesterov 	if (IS_ERR(pol))
2104ef0855d3SOleg Nesterov 		return PTR_ERR(pol);
2105ef0855d3SOleg Nesterov 	dst->vm_policy = pol;
2106ef0855d3SOleg Nesterov 	return 0;
2107ef0855d3SOleg Nesterov }
2108ef0855d3SOleg Nesterov 
21094225399aSPaul Jackson /*
2110846a16bfSLee Schermerhorn  * If mpol_dup() sees current->cpuset == cpuset_being_rebound, then it
21114225399aSPaul Jackson  * rebinds the mempolicy its copying by calling mpol_rebind_policy()
21124225399aSPaul Jackson  * with the mems_allowed returned by cpuset_mems_allowed().  This
21134225399aSPaul Jackson  * keeps mempolicies cpuset relative after its cpuset moves.  See
21144225399aSPaul Jackson  * further kernel/cpuset.c update_nodemask().
2115708c1bbcSMiao Xie  *
2116708c1bbcSMiao Xie  * current's mempolicy may be rebinded by the other task(the task that changes
2117708c1bbcSMiao Xie  * cpuset's mems), so we needn't do rebind work for current task.
21184225399aSPaul Jackson  */
21194225399aSPaul Jackson 
2120846a16bfSLee Schermerhorn /* Slow path of a mempolicy duplicate */
2121846a16bfSLee Schermerhorn struct mempolicy *__mpol_dup(struct mempolicy *old)
21221da177e4SLinus Torvalds {
21231da177e4SLinus Torvalds 	struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
21241da177e4SLinus Torvalds 
21251da177e4SLinus Torvalds 	if (!new)
21261da177e4SLinus Torvalds 		return ERR_PTR(-ENOMEM);
2127708c1bbcSMiao Xie 
2128708c1bbcSMiao Xie 	/* task's mempolicy is protected by alloc_lock */
2129708c1bbcSMiao Xie 	if (old == current->mempolicy) {
2130708c1bbcSMiao Xie 		task_lock(current);
2131708c1bbcSMiao Xie 		*new = *old;
2132708c1bbcSMiao Xie 		task_unlock(current);
2133708c1bbcSMiao Xie 	} else
2134708c1bbcSMiao Xie 		*new = *old;
2135708c1bbcSMiao Xie 
21364225399aSPaul Jackson 	if (current_cpuset_is_being_rebound()) {
21374225399aSPaul Jackson 		nodemask_t mems = cpuset_mems_allowed(current);
2138708c1bbcSMiao Xie 		if (new->flags & MPOL_F_REBINDING)
2139708c1bbcSMiao Xie 			mpol_rebind_policy(new, &mems, MPOL_REBIND_STEP2);
2140708c1bbcSMiao Xie 		else
2141708c1bbcSMiao Xie 			mpol_rebind_policy(new, &mems, MPOL_REBIND_ONCE);
21424225399aSPaul Jackson 	}
21431da177e4SLinus Torvalds 	atomic_set(&new->refcnt, 1);
21441da177e4SLinus Torvalds 	return new;
21451da177e4SLinus Torvalds }
21461da177e4SLinus Torvalds 
21471da177e4SLinus Torvalds /* Slow path of a mempolicy comparison */
2148fcfb4dccSKOSAKI Motohiro bool __mpol_equal(struct mempolicy *a, struct mempolicy *b)
21491da177e4SLinus Torvalds {
21501da177e4SLinus Torvalds 	if (!a || !b)
2151fcfb4dccSKOSAKI Motohiro 		return false;
215245c4745aSLee Schermerhorn 	if (a->mode != b->mode)
2153fcfb4dccSKOSAKI Motohiro 		return false;
215419800502SBob Liu 	if (a->flags != b->flags)
2155fcfb4dccSKOSAKI Motohiro 		return false;
215619800502SBob Liu 	if (mpol_store_user_nodemask(a))
215719800502SBob Liu 		if (!nodes_equal(a->w.user_nodemask, b->w.user_nodemask))
2158fcfb4dccSKOSAKI Motohiro 			return false;
215919800502SBob Liu 
216045c4745aSLee Schermerhorn 	switch (a->mode) {
216119770b32SMel Gorman 	case MPOL_BIND:
216219770b32SMel Gorman 		/* Fall through */
21631da177e4SLinus Torvalds 	case MPOL_INTERLEAVE:
2164fcfb4dccSKOSAKI Motohiro 		return !!nodes_equal(a->v.nodes, b->v.nodes);
21651da177e4SLinus Torvalds 	case MPOL_PREFERRED:
216675719661SNamhyung Kim 		return a->v.preferred_node == b->v.preferred_node;
21671da177e4SLinus Torvalds 	default:
21681da177e4SLinus Torvalds 		BUG();
2169fcfb4dccSKOSAKI Motohiro 		return false;
21701da177e4SLinus Torvalds 	}
21711da177e4SLinus Torvalds }
21721da177e4SLinus Torvalds 
21731da177e4SLinus Torvalds /*
21741da177e4SLinus Torvalds  * Shared memory backing store policy support.
21751da177e4SLinus Torvalds  *
21761da177e4SLinus Torvalds  * Remember policies even when nobody has shared memory mapped.
21771da177e4SLinus Torvalds  * The policies are kept in Red-Black tree linked from the inode.
21781da177e4SLinus Torvalds  * They are protected by the sp->lock spinlock, which should be held
21791da177e4SLinus Torvalds  * for any accesses to the tree.
21801da177e4SLinus Torvalds  */
21811da177e4SLinus Torvalds 
21821da177e4SLinus Torvalds /* lookup first element intersecting start-end */
218342288fe3SMel Gorman /* Caller holds sp->lock */
21841da177e4SLinus Torvalds static struct sp_node *
21851da177e4SLinus Torvalds sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end)
21861da177e4SLinus Torvalds {
21871da177e4SLinus Torvalds 	struct rb_node *n = sp->root.rb_node;
21881da177e4SLinus Torvalds 
21891da177e4SLinus Torvalds 	while (n) {
21901da177e4SLinus Torvalds 		struct sp_node *p = rb_entry(n, struct sp_node, nd);
21911da177e4SLinus Torvalds 
21921da177e4SLinus Torvalds 		if (start >= p->end)
21931da177e4SLinus Torvalds 			n = n->rb_right;
21941da177e4SLinus Torvalds 		else if (end <= p->start)
21951da177e4SLinus Torvalds 			n = n->rb_left;
21961da177e4SLinus Torvalds 		else
21971da177e4SLinus Torvalds 			break;
21981da177e4SLinus Torvalds 	}
21991da177e4SLinus Torvalds 	if (!n)
22001da177e4SLinus Torvalds 		return NULL;
22011da177e4SLinus Torvalds 	for (;;) {
22021da177e4SLinus Torvalds 		struct sp_node *w = NULL;
22031da177e4SLinus Torvalds 		struct rb_node *prev = rb_prev(n);
22041da177e4SLinus Torvalds 		if (!prev)
22051da177e4SLinus Torvalds 			break;
22061da177e4SLinus Torvalds 		w = rb_entry(prev, struct sp_node, nd);
22071da177e4SLinus Torvalds 		if (w->end <= start)
22081da177e4SLinus Torvalds 			break;
22091da177e4SLinus Torvalds 		n = prev;
22101da177e4SLinus Torvalds 	}
22111da177e4SLinus Torvalds 	return rb_entry(n, struct sp_node, nd);
22121da177e4SLinus Torvalds }
22131da177e4SLinus Torvalds 
22141da177e4SLinus Torvalds /* Insert a new shared policy into the list. */
22151da177e4SLinus Torvalds /* Caller holds sp->lock */
22161da177e4SLinus Torvalds static void sp_insert(struct shared_policy *sp, struct sp_node *new)
22171da177e4SLinus Torvalds {
22181da177e4SLinus Torvalds 	struct rb_node **p = &sp->root.rb_node;
22191da177e4SLinus Torvalds 	struct rb_node *parent = NULL;
22201da177e4SLinus Torvalds 	struct sp_node *nd;
22211da177e4SLinus Torvalds 
22221da177e4SLinus Torvalds 	while (*p) {
22231da177e4SLinus Torvalds 		parent = *p;
22241da177e4SLinus Torvalds 		nd = rb_entry(parent, struct sp_node, nd);
22251da177e4SLinus Torvalds 		if (new->start < nd->start)
22261da177e4SLinus Torvalds 			p = &(*p)->rb_left;
22271da177e4SLinus Torvalds 		else if (new->end > nd->end)
22281da177e4SLinus Torvalds 			p = &(*p)->rb_right;
22291da177e4SLinus Torvalds 		else
22301da177e4SLinus Torvalds 			BUG();
22311da177e4SLinus Torvalds 	}
22321da177e4SLinus Torvalds 	rb_link_node(&new->nd, parent, p);
22331da177e4SLinus Torvalds 	rb_insert_color(&new->nd, &sp->root);
2234140d5a49SPaul Mundt 	pr_debug("inserting %lx-%lx: %d\n", new->start, new->end,
223545c4745aSLee Schermerhorn 		 new->policy ? new->policy->mode : 0);
22361da177e4SLinus Torvalds }
22371da177e4SLinus Torvalds 
22381da177e4SLinus Torvalds /* Find shared policy intersecting idx */
22391da177e4SLinus Torvalds struct mempolicy *
22401da177e4SLinus Torvalds mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx)
22411da177e4SLinus Torvalds {
22421da177e4SLinus Torvalds 	struct mempolicy *pol = NULL;
22431da177e4SLinus Torvalds 	struct sp_node *sn;
22441da177e4SLinus Torvalds 
22451da177e4SLinus Torvalds 	if (!sp->root.rb_node)
22461da177e4SLinus Torvalds 		return NULL;
224742288fe3SMel Gorman 	spin_lock(&sp->lock);
22481da177e4SLinus Torvalds 	sn = sp_lookup(sp, idx, idx+1);
22491da177e4SLinus Torvalds 	if (sn) {
22501da177e4SLinus Torvalds 		mpol_get(sn->policy);
22511da177e4SLinus Torvalds 		pol = sn->policy;
22521da177e4SLinus Torvalds 	}
225342288fe3SMel Gorman 	spin_unlock(&sp->lock);
22541da177e4SLinus Torvalds 	return pol;
22551da177e4SLinus Torvalds }
22561da177e4SLinus Torvalds 
225763f74ca2SKOSAKI Motohiro static void sp_free(struct sp_node *n)
225863f74ca2SKOSAKI Motohiro {
225963f74ca2SKOSAKI Motohiro 	mpol_put(n->policy);
226063f74ca2SKOSAKI Motohiro 	kmem_cache_free(sn_cache, n);
226163f74ca2SKOSAKI Motohiro }
226263f74ca2SKOSAKI Motohiro 
2263771fb4d8SLee Schermerhorn /**
2264771fb4d8SLee Schermerhorn  * mpol_misplaced - check whether current page node is valid in policy
2265771fb4d8SLee Schermerhorn  *
2266b46e14acSFabian Frederick  * @page: page to be checked
2267b46e14acSFabian Frederick  * @vma: vm area where page mapped
2268b46e14acSFabian Frederick  * @addr: virtual address where page mapped
2269771fb4d8SLee Schermerhorn  *
2270771fb4d8SLee Schermerhorn  * Lookup current policy node id for vma,addr and "compare to" page's
2271771fb4d8SLee Schermerhorn  * node id.
2272771fb4d8SLee Schermerhorn  *
2273771fb4d8SLee Schermerhorn  * Returns:
2274771fb4d8SLee Schermerhorn  *	-1	- not misplaced, page is in the right node
2275771fb4d8SLee Schermerhorn  *	node	- node id where the page should be
2276771fb4d8SLee Schermerhorn  *
2277771fb4d8SLee Schermerhorn  * Policy determination "mimics" alloc_page_vma().
2278771fb4d8SLee Schermerhorn  * Called from fault path where we know the vma and faulting address.
2279771fb4d8SLee Schermerhorn  */
2280771fb4d8SLee Schermerhorn int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long addr)
2281771fb4d8SLee Schermerhorn {
2282771fb4d8SLee Schermerhorn 	struct mempolicy *pol;
2283771fb4d8SLee Schermerhorn 	struct zone *zone;
2284771fb4d8SLee Schermerhorn 	int curnid = page_to_nid(page);
2285771fb4d8SLee Schermerhorn 	unsigned long pgoff;
228690572890SPeter Zijlstra 	int thiscpu = raw_smp_processor_id();
228790572890SPeter Zijlstra 	int thisnid = cpu_to_node(thiscpu);
2288771fb4d8SLee Schermerhorn 	int polnid = -1;
2289771fb4d8SLee Schermerhorn 	int ret = -1;
2290771fb4d8SLee Schermerhorn 
2291771fb4d8SLee Schermerhorn 	BUG_ON(!vma);
2292771fb4d8SLee Schermerhorn 
2293771fb4d8SLee Schermerhorn 	pol = get_vma_policy(current, vma, addr);
2294771fb4d8SLee Schermerhorn 	if (!(pol->flags & MPOL_F_MOF))
2295771fb4d8SLee Schermerhorn 		goto out;
2296771fb4d8SLee Schermerhorn 
2297771fb4d8SLee Schermerhorn 	switch (pol->mode) {
2298771fb4d8SLee Schermerhorn 	case MPOL_INTERLEAVE:
2299771fb4d8SLee Schermerhorn 		BUG_ON(addr >= vma->vm_end);
2300771fb4d8SLee Schermerhorn 		BUG_ON(addr < vma->vm_start);
2301771fb4d8SLee Schermerhorn 
2302771fb4d8SLee Schermerhorn 		pgoff = vma->vm_pgoff;
2303771fb4d8SLee Schermerhorn 		pgoff += (addr - vma->vm_start) >> PAGE_SHIFT;
2304771fb4d8SLee Schermerhorn 		polnid = offset_il_node(pol, vma, pgoff);
2305771fb4d8SLee Schermerhorn 		break;
2306771fb4d8SLee Schermerhorn 
2307771fb4d8SLee Schermerhorn 	case MPOL_PREFERRED:
2308771fb4d8SLee Schermerhorn 		if (pol->flags & MPOL_F_LOCAL)
2309771fb4d8SLee Schermerhorn 			polnid = numa_node_id();
2310771fb4d8SLee Schermerhorn 		else
2311771fb4d8SLee Schermerhorn 			polnid = pol->v.preferred_node;
2312771fb4d8SLee Schermerhorn 		break;
2313771fb4d8SLee Schermerhorn 
2314771fb4d8SLee Schermerhorn 	case MPOL_BIND:
2315771fb4d8SLee Schermerhorn 		/*
2316771fb4d8SLee Schermerhorn 		 * allows binding to multiple nodes.
2317771fb4d8SLee Schermerhorn 		 * use current page if in policy nodemask,
2318771fb4d8SLee Schermerhorn 		 * else select nearest allowed node, if any.
2319771fb4d8SLee Schermerhorn 		 * If no allowed nodes, use current [!misplaced].
2320771fb4d8SLee Schermerhorn 		 */
2321771fb4d8SLee Schermerhorn 		if (node_isset(curnid, pol->v.nodes))
2322771fb4d8SLee Schermerhorn 			goto out;
2323771fb4d8SLee Schermerhorn 		(void)first_zones_zonelist(
2324771fb4d8SLee Schermerhorn 				node_zonelist(numa_node_id(), GFP_HIGHUSER),
2325771fb4d8SLee Schermerhorn 				gfp_zone(GFP_HIGHUSER),
2326771fb4d8SLee Schermerhorn 				&pol->v.nodes, &zone);
2327771fb4d8SLee Schermerhorn 		polnid = zone->node;
2328771fb4d8SLee Schermerhorn 		break;
2329771fb4d8SLee Schermerhorn 
2330771fb4d8SLee Schermerhorn 	default:
2331771fb4d8SLee Schermerhorn 		BUG();
2332771fb4d8SLee Schermerhorn 	}
23335606e387SMel Gorman 
23345606e387SMel Gorman 	/* Migrate the page towards the node whose CPU is referencing it */
2335e42c8ff2SMel Gorman 	if (pol->flags & MPOL_F_MORON) {
233690572890SPeter Zijlstra 		polnid = thisnid;
23375606e387SMel Gorman 
233810f39042SRik van Riel 		if (!should_numa_migrate_memory(current, page, curnid, thiscpu))
2339de1c9ce6SRik van Riel 			goto out;
2340de1c9ce6SRik van Riel 	}
2341e42c8ff2SMel Gorman 
2342771fb4d8SLee Schermerhorn 	if (curnid != polnid)
2343771fb4d8SLee Schermerhorn 		ret = polnid;
2344771fb4d8SLee Schermerhorn out:
2345771fb4d8SLee Schermerhorn 	mpol_cond_put(pol);
2346771fb4d8SLee Schermerhorn 
2347771fb4d8SLee Schermerhorn 	return ret;
2348771fb4d8SLee Schermerhorn }
2349771fb4d8SLee Schermerhorn 
23501da177e4SLinus Torvalds static void sp_delete(struct shared_policy *sp, struct sp_node *n)
23511da177e4SLinus Torvalds {
2352140d5a49SPaul Mundt 	pr_debug("deleting %lx-l%lx\n", n->start, n->end);
23531da177e4SLinus Torvalds 	rb_erase(&n->nd, &sp->root);
235463f74ca2SKOSAKI Motohiro 	sp_free(n);
23551da177e4SLinus Torvalds }
23561da177e4SLinus Torvalds 
235742288fe3SMel Gorman static void sp_node_init(struct sp_node *node, unsigned long start,
235842288fe3SMel Gorman 			unsigned long end, struct mempolicy *pol)
235942288fe3SMel Gorman {
236042288fe3SMel Gorman 	node->start = start;
236142288fe3SMel Gorman 	node->end = end;
236242288fe3SMel Gorman 	node->policy = pol;
236342288fe3SMel Gorman }
236442288fe3SMel Gorman 
2365dbcb0f19SAdrian Bunk static struct sp_node *sp_alloc(unsigned long start, unsigned long end,
2366dbcb0f19SAdrian Bunk 				struct mempolicy *pol)
23671da177e4SLinus Torvalds {
2368869833f2SKOSAKI Motohiro 	struct sp_node *n;
2369869833f2SKOSAKI Motohiro 	struct mempolicy *newpol;
23701da177e4SLinus Torvalds 
2371869833f2SKOSAKI Motohiro 	n = kmem_cache_alloc(sn_cache, GFP_KERNEL);
23721da177e4SLinus Torvalds 	if (!n)
23731da177e4SLinus Torvalds 		return NULL;
2374869833f2SKOSAKI Motohiro 
2375869833f2SKOSAKI Motohiro 	newpol = mpol_dup(pol);
2376869833f2SKOSAKI Motohiro 	if (IS_ERR(newpol)) {
2377869833f2SKOSAKI Motohiro 		kmem_cache_free(sn_cache, n);
2378869833f2SKOSAKI Motohiro 		return NULL;
2379869833f2SKOSAKI Motohiro 	}
2380869833f2SKOSAKI Motohiro 	newpol->flags |= MPOL_F_SHARED;
238142288fe3SMel Gorman 	sp_node_init(n, start, end, newpol);
2382869833f2SKOSAKI Motohiro 
23831da177e4SLinus Torvalds 	return n;
23841da177e4SLinus Torvalds }
23851da177e4SLinus Torvalds 
23861da177e4SLinus Torvalds /* Replace a policy range. */
23871da177e4SLinus Torvalds static int shared_policy_replace(struct shared_policy *sp, unsigned long start,
23881da177e4SLinus Torvalds 				 unsigned long end, struct sp_node *new)
23891da177e4SLinus Torvalds {
2390b22d127aSMel Gorman 	struct sp_node *n;
239142288fe3SMel Gorman 	struct sp_node *n_new = NULL;
239242288fe3SMel Gorman 	struct mempolicy *mpol_new = NULL;
2393b22d127aSMel Gorman 	int ret = 0;
23941da177e4SLinus Torvalds 
239542288fe3SMel Gorman restart:
239642288fe3SMel Gorman 	spin_lock(&sp->lock);
23971da177e4SLinus Torvalds 	n = sp_lookup(sp, start, end);
23981da177e4SLinus Torvalds 	/* Take care of old policies in the same range. */
23991da177e4SLinus Torvalds 	while (n && n->start < end) {
24001da177e4SLinus Torvalds 		struct rb_node *next = rb_next(&n->nd);
24011da177e4SLinus Torvalds 		if (n->start >= start) {
24021da177e4SLinus Torvalds 			if (n->end <= end)
24031da177e4SLinus Torvalds 				sp_delete(sp, n);
24041da177e4SLinus Torvalds 			else
24051da177e4SLinus Torvalds 				n->start = end;
24061da177e4SLinus Torvalds 		} else {
24071da177e4SLinus Torvalds 			/* Old policy spanning whole new range. */
24081da177e4SLinus Torvalds 			if (n->end > end) {
240942288fe3SMel Gorman 				if (!n_new)
241042288fe3SMel Gorman 					goto alloc_new;
241142288fe3SMel Gorman 
241242288fe3SMel Gorman 				*mpol_new = *n->policy;
241342288fe3SMel Gorman 				atomic_set(&mpol_new->refcnt, 1);
24147880639cSKOSAKI Motohiro 				sp_node_init(n_new, end, n->end, mpol_new);
24151da177e4SLinus Torvalds 				n->end = start;
24165ca39575SHillf Danton 				sp_insert(sp, n_new);
241742288fe3SMel Gorman 				n_new = NULL;
241842288fe3SMel Gorman 				mpol_new = NULL;
24191da177e4SLinus Torvalds 				break;
24201da177e4SLinus Torvalds 			} else
24211da177e4SLinus Torvalds 				n->end = start;
24221da177e4SLinus Torvalds 		}
24231da177e4SLinus Torvalds 		if (!next)
24241da177e4SLinus Torvalds 			break;
24251da177e4SLinus Torvalds 		n = rb_entry(next, struct sp_node, nd);
24261da177e4SLinus Torvalds 	}
24271da177e4SLinus Torvalds 	if (new)
24281da177e4SLinus Torvalds 		sp_insert(sp, new);
242942288fe3SMel Gorman 	spin_unlock(&sp->lock);
243042288fe3SMel Gorman 	ret = 0;
243142288fe3SMel Gorman 
243242288fe3SMel Gorman err_out:
243342288fe3SMel Gorman 	if (mpol_new)
243442288fe3SMel Gorman 		mpol_put(mpol_new);
243542288fe3SMel Gorman 	if (n_new)
243642288fe3SMel Gorman 		kmem_cache_free(sn_cache, n_new);
243742288fe3SMel Gorman 
2438b22d127aSMel Gorman 	return ret;
243942288fe3SMel Gorman 
244042288fe3SMel Gorman alloc_new:
244142288fe3SMel Gorman 	spin_unlock(&sp->lock);
244242288fe3SMel Gorman 	ret = -ENOMEM;
244342288fe3SMel Gorman 	n_new = kmem_cache_alloc(sn_cache, GFP_KERNEL);
244442288fe3SMel Gorman 	if (!n_new)
244542288fe3SMel Gorman 		goto err_out;
244642288fe3SMel Gorman 	mpol_new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
244742288fe3SMel Gorman 	if (!mpol_new)
244842288fe3SMel Gorman 		goto err_out;
244942288fe3SMel Gorman 	goto restart;
24501da177e4SLinus Torvalds }
24511da177e4SLinus Torvalds 
245271fe804bSLee Schermerhorn /**
245371fe804bSLee Schermerhorn  * mpol_shared_policy_init - initialize shared policy for inode
245471fe804bSLee Schermerhorn  * @sp: pointer to inode shared policy
245571fe804bSLee Schermerhorn  * @mpol:  struct mempolicy to install
245671fe804bSLee Schermerhorn  *
245771fe804bSLee Schermerhorn  * Install non-NULL @mpol in inode's shared policy rb-tree.
245871fe804bSLee Schermerhorn  * On entry, the current task has a reference on a non-NULL @mpol.
245971fe804bSLee Schermerhorn  * This must be released on exit.
24604bfc4495SKAMEZAWA Hiroyuki  * This is called at get_inode() calls and we can use GFP_KERNEL.
246171fe804bSLee Schermerhorn  */
246271fe804bSLee Schermerhorn void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol)
24637339ff83SRobin Holt {
246458568d2aSMiao Xie 	int ret;
246558568d2aSMiao Xie 
246671fe804bSLee Schermerhorn 	sp->root = RB_ROOT;		/* empty tree == default mempolicy */
246742288fe3SMel Gorman 	spin_lock_init(&sp->lock);
24687339ff83SRobin Holt 
246971fe804bSLee Schermerhorn 	if (mpol) {
24707339ff83SRobin Holt 		struct vm_area_struct pvma;
247171fe804bSLee Schermerhorn 		struct mempolicy *new;
24724bfc4495SKAMEZAWA Hiroyuki 		NODEMASK_SCRATCH(scratch);
24737339ff83SRobin Holt 
24744bfc4495SKAMEZAWA Hiroyuki 		if (!scratch)
24755c0c1654SLee Schermerhorn 			goto put_mpol;
247671fe804bSLee Schermerhorn 		/* contextualize the tmpfs mount point mempolicy */
247771fe804bSLee Schermerhorn 		new = mpol_new(mpol->mode, mpol->flags, &mpol->w.user_nodemask);
247815d77835SLee Schermerhorn 		if (IS_ERR(new))
24790cae3457SDan Carpenter 			goto free_scratch; /* no valid nodemask intersection */
248058568d2aSMiao Xie 
248158568d2aSMiao Xie 		task_lock(current);
24824bfc4495SKAMEZAWA Hiroyuki 		ret = mpol_set_nodemask(new, &mpol->w.user_nodemask, scratch);
248358568d2aSMiao Xie 		task_unlock(current);
248415d77835SLee Schermerhorn 		if (ret)
24855c0c1654SLee Schermerhorn 			goto put_new;
248671fe804bSLee Schermerhorn 
248771fe804bSLee Schermerhorn 		/* Create pseudo-vma that contains just the policy */
24887339ff83SRobin Holt 		memset(&pvma, 0, sizeof(struct vm_area_struct));
248971fe804bSLee Schermerhorn 		pvma.vm_end = TASK_SIZE;	/* policy covers entire file */
249071fe804bSLee Schermerhorn 		mpol_set_shared_policy(sp, &pvma, new); /* adds ref */
249115d77835SLee Schermerhorn 
24925c0c1654SLee Schermerhorn put_new:
249371fe804bSLee Schermerhorn 		mpol_put(new);			/* drop initial ref */
24940cae3457SDan Carpenter free_scratch:
24954bfc4495SKAMEZAWA Hiroyuki 		NODEMASK_SCRATCH_FREE(scratch);
24965c0c1654SLee Schermerhorn put_mpol:
24975c0c1654SLee Schermerhorn 		mpol_put(mpol);	/* drop our incoming ref on sb mpol */
24987339ff83SRobin Holt 	}
24997339ff83SRobin Holt }
25007339ff83SRobin Holt 
25011da177e4SLinus Torvalds int mpol_set_shared_policy(struct shared_policy *info,
25021da177e4SLinus Torvalds 			struct vm_area_struct *vma, struct mempolicy *npol)
25031da177e4SLinus Torvalds {
25041da177e4SLinus Torvalds 	int err;
25051da177e4SLinus Torvalds 	struct sp_node *new = NULL;
25061da177e4SLinus Torvalds 	unsigned long sz = vma_pages(vma);
25071da177e4SLinus Torvalds 
2508028fec41SDavid Rientjes 	pr_debug("set_shared_policy %lx sz %lu %d %d %lx\n",
25091da177e4SLinus Torvalds 		 vma->vm_pgoff,
251045c4745aSLee Schermerhorn 		 sz, npol ? npol->mode : -1,
2511028fec41SDavid Rientjes 		 npol ? npol->flags : -1,
251200ef2d2fSDavid Rientjes 		 npol ? nodes_addr(npol->v.nodes)[0] : NUMA_NO_NODE);
25131da177e4SLinus Torvalds 
25141da177e4SLinus Torvalds 	if (npol) {
25151da177e4SLinus Torvalds 		new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, npol);
25161da177e4SLinus Torvalds 		if (!new)
25171da177e4SLinus Torvalds 			return -ENOMEM;
25181da177e4SLinus Torvalds 	}
25191da177e4SLinus Torvalds 	err = shared_policy_replace(info, vma->vm_pgoff, vma->vm_pgoff+sz, new);
25201da177e4SLinus Torvalds 	if (err && new)
252163f74ca2SKOSAKI Motohiro 		sp_free(new);
25221da177e4SLinus Torvalds 	return err;
25231da177e4SLinus Torvalds }
25241da177e4SLinus Torvalds 
25251da177e4SLinus Torvalds /* Free a backing policy store on inode delete. */
25261da177e4SLinus Torvalds void mpol_free_shared_policy(struct shared_policy *p)
25271da177e4SLinus Torvalds {
25281da177e4SLinus Torvalds 	struct sp_node *n;
25291da177e4SLinus Torvalds 	struct rb_node *next;
25301da177e4SLinus Torvalds 
25311da177e4SLinus Torvalds 	if (!p->root.rb_node)
25321da177e4SLinus Torvalds 		return;
253342288fe3SMel Gorman 	spin_lock(&p->lock);
25341da177e4SLinus Torvalds 	next = rb_first(&p->root);
25351da177e4SLinus Torvalds 	while (next) {
25361da177e4SLinus Torvalds 		n = rb_entry(next, struct sp_node, nd);
25371da177e4SLinus Torvalds 		next = rb_next(&n->nd);
253863f74ca2SKOSAKI Motohiro 		sp_delete(p, n);
25391da177e4SLinus Torvalds 	}
254042288fe3SMel Gorman 	spin_unlock(&p->lock);
25411da177e4SLinus Torvalds }
25421da177e4SLinus Torvalds 
25431a687c2eSMel Gorman #ifdef CONFIG_NUMA_BALANCING
2544c297663cSMel Gorman static int __initdata numabalancing_override;
25451a687c2eSMel Gorman 
25461a687c2eSMel Gorman static void __init check_numabalancing_enable(void)
25471a687c2eSMel Gorman {
25481a687c2eSMel Gorman 	bool numabalancing_default = false;
25491a687c2eSMel Gorman 
25501a687c2eSMel Gorman 	if (IS_ENABLED(CONFIG_NUMA_BALANCING_DEFAULT_ENABLED))
25511a687c2eSMel Gorman 		numabalancing_default = true;
25521a687c2eSMel Gorman 
2553c297663cSMel Gorman 	/* Parsed by setup_numabalancing. override == 1 enables, -1 disables */
2554c297663cSMel Gorman 	if (numabalancing_override)
2555c297663cSMel Gorman 		set_numabalancing_state(numabalancing_override == 1);
2556c297663cSMel Gorman 
25571a687c2eSMel Gorman 	if (nr_node_ids > 1 && !numabalancing_override) {
25584a404beaSAndrew Morton 		pr_info("%s automatic NUMA balancing. "
2559c297663cSMel Gorman 			"Configure with numa_balancing= or the "
2560c297663cSMel Gorman 			"kernel.numa_balancing sysctl",
2561c297663cSMel Gorman 			numabalancing_default ? "Enabling" : "Disabling");
25621a687c2eSMel Gorman 		set_numabalancing_state(numabalancing_default);
25631a687c2eSMel Gorman 	}
25641a687c2eSMel Gorman }
25651a687c2eSMel Gorman 
25661a687c2eSMel Gorman static int __init setup_numabalancing(char *str)
25671a687c2eSMel Gorman {
25681a687c2eSMel Gorman 	int ret = 0;
25691a687c2eSMel Gorman 	if (!str)
25701a687c2eSMel Gorman 		goto out;
25711a687c2eSMel Gorman 
25721a687c2eSMel Gorman 	if (!strcmp(str, "enable")) {
2573c297663cSMel Gorman 		numabalancing_override = 1;
25741a687c2eSMel Gorman 		ret = 1;
25751a687c2eSMel Gorman 	} else if (!strcmp(str, "disable")) {
2576c297663cSMel Gorman 		numabalancing_override = -1;
25771a687c2eSMel Gorman 		ret = 1;
25781a687c2eSMel Gorman 	}
25791a687c2eSMel Gorman out:
25801a687c2eSMel Gorman 	if (!ret)
25814a404beaSAndrew Morton 		pr_warn("Unable to parse numa_balancing=\n");
25821a687c2eSMel Gorman 
25831a687c2eSMel Gorman 	return ret;
25841a687c2eSMel Gorman }
25851a687c2eSMel Gorman __setup("numa_balancing=", setup_numabalancing);
25861a687c2eSMel Gorman #else
25871a687c2eSMel Gorman static inline void __init check_numabalancing_enable(void)
25881a687c2eSMel Gorman {
25891a687c2eSMel Gorman }
25901a687c2eSMel Gorman #endif /* CONFIG_NUMA_BALANCING */
25911a687c2eSMel Gorman 
25921da177e4SLinus Torvalds /* assumes fs == KERNEL_DS */
25931da177e4SLinus Torvalds void __init numa_policy_init(void)
25941da177e4SLinus Torvalds {
2595b71636e2SPaul Mundt 	nodemask_t interleave_nodes;
2596b71636e2SPaul Mundt 	unsigned long largest = 0;
2597b71636e2SPaul Mundt 	int nid, prefer = 0;
2598b71636e2SPaul Mundt 
25991da177e4SLinus Torvalds 	policy_cache = kmem_cache_create("numa_policy",
26001da177e4SLinus Torvalds 					 sizeof(struct mempolicy),
260120c2df83SPaul Mundt 					 0, SLAB_PANIC, NULL);
26021da177e4SLinus Torvalds 
26031da177e4SLinus Torvalds 	sn_cache = kmem_cache_create("shared_policy_node",
26041da177e4SLinus Torvalds 				     sizeof(struct sp_node),
260520c2df83SPaul Mundt 				     0, SLAB_PANIC, NULL);
26061da177e4SLinus Torvalds 
26075606e387SMel Gorman 	for_each_node(nid) {
26085606e387SMel Gorman 		preferred_node_policy[nid] = (struct mempolicy) {
26095606e387SMel Gorman 			.refcnt = ATOMIC_INIT(1),
26105606e387SMel Gorman 			.mode = MPOL_PREFERRED,
26115606e387SMel Gorman 			.flags = MPOL_F_MOF | MPOL_F_MORON,
26125606e387SMel Gorman 			.v = { .preferred_node = nid, },
26135606e387SMel Gorman 		};
26145606e387SMel Gorman 	}
26155606e387SMel Gorman 
2616b71636e2SPaul Mundt 	/*
2617b71636e2SPaul Mundt 	 * Set interleaving policy for system init. Interleaving is only
2618b71636e2SPaul Mundt 	 * enabled across suitably sized nodes (default is >= 16MB), or
2619b71636e2SPaul Mundt 	 * fall back to the largest node if they're all smaller.
2620b71636e2SPaul Mundt 	 */
2621b71636e2SPaul Mundt 	nodes_clear(interleave_nodes);
262201f13bd6SLai Jiangshan 	for_each_node_state(nid, N_MEMORY) {
2623b71636e2SPaul Mundt 		unsigned long total_pages = node_present_pages(nid);
26241da177e4SLinus Torvalds 
2625b71636e2SPaul Mundt 		/* Preserve the largest node */
2626b71636e2SPaul Mundt 		if (largest < total_pages) {
2627b71636e2SPaul Mundt 			largest = total_pages;
2628b71636e2SPaul Mundt 			prefer = nid;
2629b71636e2SPaul Mundt 		}
2630b71636e2SPaul Mundt 
2631b71636e2SPaul Mundt 		/* Interleave this node? */
2632b71636e2SPaul Mundt 		if ((total_pages << PAGE_SHIFT) >= (16 << 20))
2633b71636e2SPaul Mundt 			node_set(nid, interleave_nodes);
2634b71636e2SPaul Mundt 	}
2635b71636e2SPaul Mundt 
2636b71636e2SPaul Mundt 	/* All too small, use the largest */
2637b71636e2SPaul Mundt 	if (unlikely(nodes_empty(interleave_nodes)))
2638b71636e2SPaul Mundt 		node_set(prefer, interleave_nodes);
2639b71636e2SPaul Mundt 
2640028fec41SDavid Rientjes 	if (do_set_mempolicy(MPOL_INTERLEAVE, 0, &interleave_nodes))
2641b1de0d13SMitchel Humpherys 		pr_err("%s: interleaving failed\n", __func__);
26421a687c2eSMel Gorman 
26431a687c2eSMel Gorman 	check_numabalancing_enable();
26441da177e4SLinus Torvalds }
26451da177e4SLinus Torvalds 
26468bccd85fSChristoph Lameter /* Reset policy of current process to default */
26471da177e4SLinus Torvalds void numa_default_policy(void)
26481da177e4SLinus Torvalds {
2649028fec41SDavid Rientjes 	do_set_mempolicy(MPOL_DEFAULT, 0, NULL);
26501da177e4SLinus Torvalds }
265168860ec1SPaul Jackson 
26524225399aSPaul Jackson /*
2653095f1fc4SLee Schermerhorn  * Parse and format mempolicy from/to strings
2654095f1fc4SLee Schermerhorn  */
2655095f1fc4SLee Schermerhorn 
2656095f1fc4SLee Schermerhorn /*
2657f2a07f40SHugh Dickins  * "local" is implemented internally by MPOL_PREFERRED with MPOL_F_LOCAL flag.
26581a75a6c8SChristoph Lameter  */
2659345ace9cSLee Schermerhorn static const char * const policy_modes[] =
2660345ace9cSLee Schermerhorn {
2661345ace9cSLee Schermerhorn 	[MPOL_DEFAULT]    = "default",
2662345ace9cSLee Schermerhorn 	[MPOL_PREFERRED]  = "prefer",
2663345ace9cSLee Schermerhorn 	[MPOL_BIND]       = "bind",
2664345ace9cSLee Schermerhorn 	[MPOL_INTERLEAVE] = "interleave",
2665d3a71033SLee Schermerhorn 	[MPOL_LOCAL]      = "local",
2666345ace9cSLee Schermerhorn };
26671a75a6c8SChristoph Lameter 
2668095f1fc4SLee Schermerhorn 
2669095f1fc4SLee Schermerhorn #ifdef CONFIG_TMPFS
2670095f1fc4SLee Schermerhorn /**
2671f2a07f40SHugh Dickins  * mpol_parse_str - parse string to mempolicy, for tmpfs mpol mount option.
2672095f1fc4SLee Schermerhorn  * @str:  string containing mempolicy to parse
267371fe804bSLee Schermerhorn  * @mpol:  pointer to struct mempolicy pointer, returned on success.
2674095f1fc4SLee Schermerhorn  *
2675095f1fc4SLee Schermerhorn  * Format of input:
2676095f1fc4SLee Schermerhorn  *	<mode>[=<flags>][:<nodelist>]
2677095f1fc4SLee Schermerhorn  *
267871fe804bSLee Schermerhorn  * On success, returns 0, else 1
2679095f1fc4SLee Schermerhorn  */
2680a7a88b23SHugh Dickins int mpol_parse_str(char *str, struct mempolicy **mpol)
2681095f1fc4SLee Schermerhorn {
268271fe804bSLee Schermerhorn 	struct mempolicy *new = NULL;
2683b4652e84SLee Schermerhorn 	unsigned short mode;
2684f2a07f40SHugh Dickins 	unsigned short mode_flags;
268571fe804bSLee Schermerhorn 	nodemask_t nodes;
2686095f1fc4SLee Schermerhorn 	char *nodelist = strchr(str, ':');
2687095f1fc4SLee Schermerhorn 	char *flags = strchr(str, '=');
2688095f1fc4SLee Schermerhorn 	int err = 1;
2689095f1fc4SLee Schermerhorn 
2690095f1fc4SLee Schermerhorn 	if (nodelist) {
2691095f1fc4SLee Schermerhorn 		/* NUL-terminate mode or flags string */
2692095f1fc4SLee Schermerhorn 		*nodelist++ = '\0';
269371fe804bSLee Schermerhorn 		if (nodelist_parse(nodelist, nodes))
2694095f1fc4SLee Schermerhorn 			goto out;
269501f13bd6SLai Jiangshan 		if (!nodes_subset(nodes, node_states[N_MEMORY]))
2696095f1fc4SLee Schermerhorn 			goto out;
269771fe804bSLee Schermerhorn 	} else
269871fe804bSLee Schermerhorn 		nodes_clear(nodes);
269971fe804bSLee Schermerhorn 
2700095f1fc4SLee Schermerhorn 	if (flags)
2701095f1fc4SLee Schermerhorn 		*flags++ = '\0';	/* terminate mode string */
2702095f1fc4SLee Schermerhorn 
2703479e2802SPeter Zijlstra 	for (mode = 0; mode < MPOL_MAX; mode++) {
2704345ace9cSLee Schermerhorn 		if (!strcmp(str, policy_modes[mode])) {
2705095f1fc4SLee Schermerhorn 			break;
2706095f1fc4SLee Schermerhorn 		}
2707095f1fc4SLee Schermerhorn 	}
2708a720094dSMel Gorman 	if (mode >= MPOL_MAX)
2709095f1fc4SLee Schermerhorn 		goto out;
2710095f1fc4SLee Schermerhorn 
271171fe804bSLee Schermerhorn 	switch (mode) {
2712095f1fc4SLee Schermerhorn 	case MPOL_PREFERRED:
271371fe804bSLee Schermerhorn 		/*
271471fe804bSLee Schermerhorn 		 * Insist on a nodelist of one node only
271571fe804bSLee Schermerhorn 		 */
2716095f1fc4SLee Schermerhorn 		if (nodelist) {
2717095f1fc4SLee Schermerhorn 			char *rest = nodelist;
2718095f1fc4SLee Schermerhorn 			while (isdigit(*rest))
2719095f1fc4SLee Schermerhorn 				rest++;
2720926f2ae0SKOSAKI Motohiro 			if (*rest)
2721926f2ae0SKOSAKI Motohiro 				goto out;
2722095f1fc4SLee Schermerhorn 		}
2723095f1fc4SLee Schermerhorn 		break;
2724095f1fc4SLee Schermerhorn 	case MPOL_INTERLEAVE:
2725095f1fc4SLee Schermerhorn 		/*
2726095f1fc4SLee Schermerhorn 		 * Default to online nodes with memory if no nodelist
2727095f1fc4SLee Schermerhorn 		 */
2728095f1fc4SLee Schermerhorn 		if (!nodelist)
272901f13bd6SLai Jiangshan 			nodes = node_states[N_MEMORY];
27303f226aa1SLee Schermerhorn 		break;
273171fe804bSLee Schermerhorn 	case MPOL_LOCAL:
27323f226aa1SLee Schermerhorn 		/*
273371fe804bSLee Schermerhorn 		 * Don't allow a nodelist;  mpol_new() checks flags
27343f226aa1SLee Schermerhorn 		 */
273571fe804bSLee Schermerhorn 		if (nodelist)
27363f226aa1SLee Schermerhorn 			goto out;
273771fe804bSLee Schermerhorn 		mode = MPOL_PREFERRED;
27383f226aa1SLee Schermerhorn 		break;
2739413b43deSRavikiran G Thirumalai 	case MPOL_DEFAULT:
2740413b43deSRavikiran G Thirumalai 		/*
2741413b43deSRavikiran G Thirumalai 		 * Insist on a empty nodelist
2742413b43deSRavikiran G Thirumalai 		 */
2743413b43deSRavikiran G Thirumalai 		if (!nodelist)
2744413b43deSRavikiran G Thirumalai 			err = 0;
2745413b43deSRavikiran G Thirumalai 		goto out;
2746d69b2e63SKOSAKI Motohiro 	case MPOL_BIND:
274771fe804bSLee Schermerhorn 		/*
2748d69b2e63SKOSAKI Motohiro 		 * Insist on a nodelist
274971fe804bSLee Schermerhorn 		 */
2750d69b2e63SKOSAKI Motohiro 		if (!nodelist)
2751d69b2e63SKOSAKI Motohiro 			goto out;
2752095f1fc4SLee Schermerhorn 	}
2753095f1fc4SLee Schermerhorn 
275471fe804bSLee Schermerhorn 	mode_flags = 0;
2755095f1fc4SLee Schermerhorn 	if (flags) {
2756095f1fc4SLee Schermerhorn 		/*
2757095f1fc4SLee Schermerhorn 		 * Currently, we only support two mutually exclusive
2758095f1fc4SLee Schermerhorn 		 * mode flags.
2759095f1fc4SLee Schermerhorn 		 */
2760095f1fc4SLee Schermerhorn 		if (!strcmp(flags, "static"))
276171fe804bSLee Schermerhorn 			mode_flags |= MPOL_F_STATIC_NODES;
2762095f1fc4SLee Schermerhorn 		else if (!strcmp(flags, "relative"))
276371fe804bSLee Schermerhorn 			mode_flags |= MPOL_F_RELATIVE_NODES;
2764095f1fc4SLee Schermerhorn 		else
2765926f2ae0SKOSAKI Motohiro 			goto out;
2766095f1fc4SLee Schermerhorn 	}
276771fe804bSLee Schermerhorn 
276871fe804bSLee Schermerhorn 	new = mpol_new(mode, mode_flags, &nodes);
276971fe804bSLee Schermerhorn 	if (IS_ERR(new))
2770926f2ae0SKOSAKI Motohiro 		goto out;
2771926f2ae0SKOSAKI Motohiro 
2772f2a07f40SHugh Dickins 	/*
2773f2a07f40SHugh Dickins 	 * Save nodes for mpol_to_str() to show the tmpfs mount options
2774f2a07f40SHugh Dickins 	 * for /proc/mounts, /proc/pid/mounts and /proc/pid/mountinfo.
2775f2a07f40SHugh Dickins 	 */
2776f2a07f40SHugh Dickins 	if (mode != MPOL_PREFERRED)
2777f2a07f40SHugh Dickins 		new->v.nodes = nodes;
2778f2a07f40SHugh Dickins 	else if (nodelist)
2779f2a07f40SHugh Dickins 		new->v.preferred_node = first_node(nodes);
2780f2a07f40SHugh Dickins 	else
2781f2a07f40SHugh Dickins 		new->flags |= MPOL_F_LOCAL;
2782f2a07f40SHugh Dickins 
2783f2a07f40SHugh Dickins 	/*
2784f2a07f40SHugh Dickins 	 * Save nodes for contextualization: this will be used to "clone"
2785f2a07f40SHugh Dickins 	 * the mempolicy in a specific context [cpuset] at a later time.
2786f2a07f40SHugh Dickins 	 */
2787e17f74afSLee Schermerhorn 	new->w.user_nodemask = nodes;
2788f2a07f40SHugh Dickins 
2789926f2ae0SKOSAKI Motohiro 	err = 0;
279071fe804bSLee Schermerhorn 
2791095f1fc4SLee Schermerhorn out:
2792095f1fc4SLee Schermerhorn 	/* Restore string for error message */
2793095f1fc4SLee Schermerhorn 	if (nodelist)
2794095f1fc4SLee Schermerhorn 		*--nodelist = ':';
2795095f1fc4SLee Schermerhorn 	if (flags)
2796095f1fc4SLee Schermerhorn 		*--flags = '=';
279771fe804bSLee Schermerhorn 	if (!err)
279871fe804bSLee Schermerhorn 		*mpol = new;
2799095f1fc4SLee Schermerhorn 	return err;
2800095f1fc4SLee Schermerhorn }
2801095f1fc4SLee Schermerhorn #endif /* CONFIG_TMPFS */
2802095f1fc4SLee Schermerhorn 
280371fe804bSLee Schermerhorn /**
280471fe804bSLee Schermerhorn  * mpol_to_str - format a mempolicy structure for printing
280571fe804bSLee Schermerhorn  * @buffer:  to contain formatted mempolicy string
280671fe804bSLee Schermerhorn  * @maxlen:  length of @buffer
280771fe804bSLee Schermerhorn  * @pol:  pointer to mempolicy to be formatted
280871fe804bSLee Schermerhorn  *
2809948927eeSDavid Rientjes  * Convert @pol into a string.  If @buffer is too short, truncate the string.
2810948927eeSDavid Rientjes  * Recommend a @maxlen of at least 32 for the longest mode, "interleave", the
2811948927eeSDavid Rientjes  * longest flag, "relative", and to display at least a few node ids.
28121a75a6c8SChristoph Lameter  */
2813948927eeSDavid Rientjes void mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)
28141a75a6c8SChristoph Lameter {
28151a75a6c8SChristoph Lameter 	char *p = buffer;
2816948927eeSDavid Rientjes 	nodemask_t nodes = NODE_MASK_NONE;
2817948927eeSDavid Rientjes 	unsigned short mode = MPOL_DEFAULT;
2818948927eeSDavid Rientjes 	unsigned short flags = 0;
28191a75a6c8SChristoph Lameter 
28208790c71aSDavid Rientjes 	if (pol && pol != &default_policy && !(pol->flags & MPOL_F_MORON)) {
2821bea904d5SLee Schermerhorn 		mode = pol->mode;
2822948927eeSDavid Rientjes 		flags = pol->flags;
2823948927eeSDavid Rientjes 	}
2824bea904d5SLee Schermerhorn 
28251a75a6c8SChristoph Lameter 	switch (mode) {
28261a75a6c8SChristoph Lameter 	case MPOL_DEFAULT:
28271a75a6c8SChristoph Lameter 		break;
28281a75a6c8SChristoph Lameter 	case MPOL_PREFERRED:
2829fc36b8d3SLee Schermerhorn 		if (flags & MPOL_F_LOCAL)
2830f2a07f40SHugh Dickins 			mode = MPOL_LOCAL;
283153f2556bSLee Schermerhorn 		else
2832fc36b8d3SLee Schermerhorn 			node_set(pol->v.preferred_node, nodes);
28331a75a6c8SChristoph Lameter 		break;
28341a75a6c8SChristoph Lameter 	case MPOL_BIND:
28351a75a6c8SChristoph Lameter 	case MPOL_INTERLEAVE:
28361a75a6c8SChristoph Lameter 		nodes = pol->v.nodes;
28371a75a6c8SChristoph Lameter 		break;
28381a75a6c8SChristoph Lameter 	default:
2839948927eeSDavid Rientjes 		WARN_ON_ONCE(1);
2840948927eeSDavid Rientjes 		snprintf(p, maxlen, "unknown");
2841948927eeSDavid Rientjes 		return;
28421a75a6c8SChristoph Lameter 	}
28431a75a6c8SChristoph Lameter 
2844b7a9f420SDavid Rientjes 	p += snprintf(p, maxlen, "%s", policy_modes[mode]);
28451a75a6c8SChristoph Lameter 
2846fc36b8d3SLee Schermerhorn 	if (flags & MPOL_MODE_FLAGS) {
2847948927eeSDavid Rientjes 		p += snprintf(p, buffer + maxlen - p, "=");
2848f5b087b5SDavid Rientjes 
28492291990aSLee Schermerhorn 		/*
28502291990aSLee Schermerhorn 		 * Currently, the only defined flags are mutually exclusive
28512291990aSLee Schermerhorn 		 */
2852f5b087b5SDavid Rientjes 		if (flags & MPOL_F_STATIC_NODES)
28532291990aSLee Schermerhorn 			p += snprintf(p, buffer + maxlen - p, "static");
28542291990aSLee Schermerhorn 		else if (flags & MPOL_F_RELATIVE_NODES)
28552291990aSLee Schermerhorn 			p += snprintf(p, buffer + maxlen - p, "relative");
2856f5b087b5SDavid Rientjes 	}
2857f5b087b5SDavid Rientjes 
28581a75a6c8SChristoph Lameter 	if (!nodes_empty(nodes)) {
2859948927eeSDavid Rientjes 		p += snprintf(p, buffer + maxlen - p, ":");
28601a75a6c8SChristoph Lameter 	 	p += nodelist_scnprintf(p, buffer + maxlen - p, nodes);
28611a75a6c8SChristoph Lameter 	}
28621a75a6c8SChristoph Lameter }
2863