xref: /openbmc/linux/mm/mempolicy.c (revision 2cf1c348)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Simple NUMA memory policy for the Linux kernel.
4  *
5  * Copyright 2003,2004 Andi Kleen, SuSE Labs.
6  * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
7  *
8  * NUMA policy allows the user to give hints in which node(s) memory should
9  * be allocated.
10  *
11  * Support four policies per VMA and per process:
12  *
13  * The VMA policy has priority over the process policy for a page fault.
14  *
15  * interleave     Allocate memory interleaved over a set of nodes,
16  *                with normal fallback if it fails.
17  *                For VMA based allocations this interleaves based on the
18  *                offset into the backing object or offset into the mapping
19  *                for anonymous memory. For process policy an process counter
20  *                is used.
21  *
22  * bind           Only allocate memory on a specific set of nodes,
23  *                no fallback.
24  *                FIXME: memory is allocated starting with the first node
25  *                to the last. It would be better if bind would truly restrict
26  *                the allocation to memory nodes instead
27  *
28  * preferred       Try a specific node first before normal fallback.
29  *                As a special case NUMA_NO_NODE here means do the allocation
30  *                on the local CPU. This is normally identical to default,
31  *                but useful to set in a VMA when you have a non default
32  *                process policy.
33  *
34  * preferred many Try a set of nodes first before normal fallback. This is
35  *                similar to preferred without the special case.
36  *
37  * default        Allocate on the local node first, or when on a VMA
38  *                use the process policy. This is what Linux always did
39  *		  in a NUMA aware kernel and still does by, ahem, default.
40  *
41  * The process policy is applied for most non interrupt memory allocations
42  * in that process' context. Interrupts ignore the policies and always
43  * try to allocate on the local CPU. The VMA policy is only applied for memory
44  * allocations for a VMA in the VM.
45  *
46  * Currently there are a few corner cases in swapping where the policy
47  * is not applied, but the majority should be handled. When process policy
48  * is used it is not remembered over swap outs/swap ins.
49  *
50  * Only the highest zone in the zone hierarchy gets policied. Allocations
51  * requesting a lower zone just use default policy. This implies that
52  * on systems with highmem kernel lowmem allocation don't get policied.
53  * Same with GFP_DMA allocations.
54  *
55  * For shmfs/tmpfs/hugetlbfs shared memory the policy is shared between
56  * all users and remembered even when nobody has memory mapped.
57  */
58 
59 /* Notebook:
60    fix mmap readahead to honour policy and enable policy for any page cache
61    object
62    statistics for bigpages
63    global policy for page cache? currently it uses process policy. Requires
64    first item above.
65    handle mremap for shared memory (currently ignored for the policy)
66    grows down?
67    make bind policy root only? It can trigger oom much faster and the
68    kernel is not always grateful with that.
69 */
70 
71 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
72 
73 #include <linux/mempolicy.h>
74 #include <linux/pagewalk.h>
75 #include <linux/highmem.h>
76 #include <linux/hugetlb.h>
77 #include <linux/kernel.h>
78 #include <linux/sched.h>
79 #include <linux/sched/mm.h>
80 #include <linux/sched/numa_balancing.h>
81 #include <linux/sched/task.h>
82 #include <linux/nodemask.h>
83 #include <linux/cpuset.h>
84 #include <linux/slab.h>
85 #include <linux/string.h>
86 #include <linux/export.h>
87 #include <linux/nsproxy.h>
88 #include <linux/interrupt.h>
89 #include <linux/init.h>
90 #include <linux/compat.h>
91 #include <linux/ptrace.h>
92 #include <linux/swap.h>
93 #include <linux/seq_file.h>
94 #include <linux/proc_fs.h>
95 #include <linux/migrate.h>
96 #include <linux/ksm.h>
97 #include <linux/rmap.h>
98 #include <linux/security.h>
99 #include <linux/syscalls.h>
100 #include <linux/ctype.h>
101 #include <linux/mm_inline.h>
102 #include <linux/mmu_notifier.h>
103 #include <linux/printk.h>
104 #include <linux/swapops.h>
105 
106 #include <asm/tlbflush.h>
107 #include <linux/uaccess.h>
108 
109 #include "internal.h"
110 
111 /* Internal flags */
112 #define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0)	/* Skip checks for continuous vmas */
113 #define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1)		/* Invert check for nodemask */
114 
115 static struct kmem_cache *policy_cache;
116 static struct kmem_cache *sn_cache;
117 
118 /* Highest zone. An specific allocation for a zone below that is not
119    policied. */
120 enum zone_type policy_zone = 0;
121 
122 /*
123  * run-time system-wide default policy => local allocation
124  */
125 static struct mempolicy default_policy = {
126 	.refcnt = ATOMIC_INIT(1), /* never free it */
127 	.mode = MPOL_LOCAL,
128 };
129 
130 static struct mempolicy preferred_node_policy[MAX_NUMNODES];
131 
132 /**
133  * numa_map_to_online_node - Find closest online node
134  * @node: Node id to start the search
135  *
136  * Lookup the next closest node by distance if @nid is not online.
137  *
138  * Return: this @node if it is online, otherwise the closest node by distance
139  */
140 int numa_map_to_online_node(int node)
141 {
142 	int min_dist = INT_MAX, dist, n, min_node;
143 
144 	if (node == NUMA_NO_NODE || node_online(node))
145 		return node;
146 
147 	min_node = node;
148 	for_each_online_node(n) {
149 		dist = node_distance(node, n);
150 		if (dist < min_dist) {
151 			min_dist = dist;
152 			min_node = n;
153 		}
154 	}
155 
156 	return min_node;
157 }
158 EXPORT_SYMBOL_GPL(numa_map_to_online_node);
159 
160 struct mempolicy *get_task_policy(struct task_struct *p)
161 {
162 	struct mempolicy *pol = p->mempolicy;
163 	int node;
164 
165 	if (pol)
166 		return pol;
167 
168 	node = numa_node_id();
169 	if (node != NUMA_NO_NODE) {
170 		pol = &preferred_node_policy[node];
171 		/* preferred_node_policy is not initialised early in boot */
172 		if (pol->mode)
173 			return pol;
174 	}
175 
176 	return &default_policy;
177 }
178 
179 static const struct mempolicy_operations {
180 	int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
181 	void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes);
182 } mpol_ops[MPOL_MAX];
183 
184 static inline int mpol_store_user_nodemask(const struct mempolicy *pol)
185 {
186 	return pol->flags & MPOL_MODE_FLAGS;
187 }
188 
189 static void mpol_relative_nodemask(nodemask_t *ret, const nodemask_t *orig,
190 				   const nodemask_t *rel)
191 {
192 	nodemask_t tmp;
193 	nodes_fold(tmp, *orig, nodes_weight(*rel));
194 	nodes_onto(*ret, tmp, *rel);
195 }
196 
197 static int mpol_new_nodemask(struct mempolicy *pol, const nodemask_t *nodes)
198 {
199 	if (nodes_empty(*nodes))
200 		return -EINVAL;
201 	pol->nodes = *nodes;
202 	return 0;
203 }
204 
205 static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes)
206 {
207 	if (nodes_empty(*nodes))
208 		return -EINVAL;
209 
210 	nodes_clear(pol->nodes);
211 	node_set(first_node(*nodes), pol->nodes);
212 	return 0;
213 }
214 
215 /*
216  * mpol_set_nodemask is called after mpol_new() to set up the nodemask, if
217  * any, for the new policy.  mpol_new() has already validated the nodes
218  * parameter with respect to the policy mode and flags.
219  *
220  * Must be called holding task's alloc_lock to protect task's mems_allowed
221  * and mempolicy.  May also be called holding the mmap_lock for write.
222  */
223 static int mpol_set_nodemask(struct mempolicy *pol,
224 		     const nodemask_t *nodes, struct nodemask_scratch *nsc)
225 {
226 	int ret;
227 
228 	/*
229 	 * Default (pol==NULL) resp. local memory policies are not a
230 	 * subject of any remapping. They also do not need any special
231 	 * constructor.
232 	 */
233 	if (!pol || pol->mode == MPOL_LOCAL)
234 		return 0;
235 
236 	/* Check N_MEMORY */
237 	nodes_and(nsc->mask1,
238 		  cpuset_current_mems_allowed, node_states[N_MEMORY]);
239 
240 	VM_BUG_ON(!nodes);
241 
242 	if (pol->flags & MPOL_F_RELATIVE_NODES)
243 		mpol_relative_nodemask(&nsc->mask2, nodes, &nsc->mask1);
244 	else
245 		nodes_and(nsc->mask2, *nodes, nsc->mask1);
246 
247 	if (mpol_store_user_nodemask(pol))
248 		pol->w.user_nodemask = *nodes;
249 	else
250 		pol->w.cpuset_mems_allowed = cpuset_current_mems_allowed;
251 
252 	ret = mpol_ops[pol->mode].create(pol, &nsc->mask2);
253 	return ret;
254 }
255 
256 /*
257  * This function just creates a new policy, does some check and simple
258  * initialization. You must invoke mpol_set_nodemask() to set nodes.
259  */
260 static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags,
261 				  nodemask_t *nodes)
262 {
263 	struct mempolicy *policy;
264 
265 	pr_debug("setting mode %d flags %d nodes[0] %lx\n",
266 		 mode, flags, nodes ? nodes_addr(*nodes)[0] : NUMA_NO_NODE);
267 
268 	if (mode == MPOL_DEFAULT) {
269 		if (nodes && !nodes_empty(*nodes))
270 			return ERR_PTR(-EINVAL);
271 		return NULL;
272 	}
273 	VM_BUG_ON(!nodes);
274 
275 	/*
276 	 * MPOL_PREFERRED cannot be used with MPOL_F_STATIC_NODES or
277 	 * MPOL_F_RELATIVE_NODES if the nodemask is empty (local allocation).
278 	 * All other modes require a valid pointer to a non-empty nodemask.
279 	 */
280 	if (mode == MPOL_PREFERRED) {
281 		if (nodes_empty(*nodes)) {
282 			if (((flags & MPOL_F_STATIC_NODES) ||
283 			     (flags & MPOL_F_RELATIVE_NODES)))
284 				return ERR_PTR(-EINVAL);
285 
286 			mode = MPOL_LOCAL;
287 		}
288 	} else if (mode == MPOL_LOCAL) {
289 		if (!nodes_empty(*nodes) ||
290 		    (flags & MPOL_F_STATIC_NODES) ||
291 		    (flags & MPOL_F_RELATIVE_NODES))
292 			return ERR_PTR(-EINVAL);
293 	} else if (nodes_empty(*nodes))
294 		return ERR_PTR(-EINVAL);
295 	policy = kmem_cache_alloc(policy_cache, GFP_KERNEL);
296 	if (!policy)
297 		return ERR_PTR(-ENOMEM);
298 	atomic_set(&policy->refcnt, 1);
299 	policy->mode = mode;
300 	policy->flags = flags;
301 	policy->home_node = NUMA_NO_NODE;
302 
303 	return policy;
304 }
305 
306 /* Slow path of a mpol destructor. */
307 void __mpol_put(struct mempolicy *p)
308 {
309 	if (!atomic_dec_and_test(&p->refcnt))
310 		return;
311 	kmem_cache_free(policy_cache, p);
312 }
313 
314 static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes)
315 {
316 }
317 
318 static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes)
319 {
320 	nodemask_t tmp;
321 
322 	if (pol->flags & MPOL_F_STATIC_NODES)
323 		nodes_and(tmp, pol->w.user_nodemask, *nodes);
324 	else if (pol->flags & MPOL_F_RELATIVE_NODES)
325 		mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
326 	else {
327 		nodes_remap(tmp, pol->nodes, pol->w.cpuset_mems_allowed,
328 								*nodes);
329 		pol->w.cpuset_mems_allowed = *nodes;
330 	}
331 
332 	if (nodes_empty(tmp))
333 		tmp = *nodes;
334 
335 	pol->nodes = tmp;
336 }
337 
338 static void mpol_rebind_preferred(struct mempolicy *pol,
339 						const nodemask_t *nodes)
340 {
341 	pol->w.cpuset_mems_allowed = *nodes;
342 }
343 
344 /*
345  * mpol_rebind_policy - Migrate a policy to a different set of nodes
346  *
347  * Per-vma policies are protected by mmap_lock. Allocations using per-task
348  * policies are protected by task->mems_allowed_seq to prevent a premature
349  * OOM/allocation failure due to parallel nodemask modification.
350  */
351 static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask)
352 {
353 	if (!pol)
354 		return;
355 	if (!mpol_store_user_nodemask(pol) &&
356 	    nodes_equal(pol->w.cpuset_mems_allowed, *newmask))
357 		return;
358 
359 	mpol_ops[pol->mode].rebind(pol, newmask);
360 }
361 
362 /*
363  * Wrapper for mpol_rebind_policy() that just requires task
364  * pointer, and updates task mempolicy.
365  *
366  * Called with task's alloc_lock held.
367  */
368 
369 void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new)
370 {
371 	mpol_rebind_policy(tsk->mempolicy, new);
372 }
373 
374 /*
375  * Rebind each vma in mm to new nodemask.
376  *
377  * Call holding a reference to mm.  Takes mm->mmap_lock during call.
378  */
379 
380 void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
381 {
382 	struct vm_area_struct *vma;
383 
384 	mmap_write_lock(mm);
385 	for (vma = mm->mmap; vma; vma = vma->vm_next)
386 		mpol_rebind_policy(vma->vm_policy, new);
387 	mmap_write_unlock(mm);
388 }
389 
390 static const struct mempolicy_operations mpol_ops[MPOL_MAX] = {
391 	[MPOL_DEFAULT] = {
392 		.rebind = mpol_rebind_default,
393 	},
394 	[MPOL_INTERLEAVE] = {
395 		.create = mpol_new_nodemask,
396 		.rebind = mpol_rebind_nodemask,
397 	},
398 	[MPOL_PREFERRED] = {
399 		.create = mpol_new_preferred,
400 		.rebind = mpol_rebind_preferred,
401 	},
402 	[MPOL_BIND] = {
403 		.create = mpol_new_nodemask,
404 		.rebind = mpol_rebind_nodemask,
405 	},
406 	[MPOL_LOCAL] = {
407 		.rebind = mpol_rebind_default,
408 	},
409 	[MPOL_PREFERRED_MANY] = {
410 		.create = mpol_new_nodemask,
411 		.rebind = mpol_rebind_preferred,
412 	},
413 };
414 
415 static int migrate_page_add(struct page *page, struct list_head *pagelist,
416 				unsigned long flags);
417 
418 struct queue_pages {
419 	struct list_head *pagelist;
420 	unsigned long flags;
421 	nodemask_t *nmask;
422 	unsigned long start;
423 	unsigned long end;
424 	struct vm_area_struct *first;
425 };
426 
427 /*
428  * Check if the page's nid is in qp->nmask.
429  *
430  * If MPOL_MF_INVERT is set in qp->flags, check if the nid is
431  * in the invert of qp->nmask.
432  */
433 static inline bool queue_pages_required(struct page *page,
434 					struct queue_pages *qp)
435 {
436 	int nid = page_to_nid(page);
437 	unsigned long flags = qp->flags;
438 
439 	return node_isset(nid, *qp->nmask) == !(flags & MPOL_MF_INVERT);
440 }
441 
442 /*
443  * queue_pages_pmd() has four possible return values:
444  * 0 - pages are placed on the right node or queued successfully, or
445  *     special page is met, i.e. huge zero page.
446  * 1 - there is unmovable page, and MPOL_MF_MOVE* & MPOL_MF_STRICT were
447  *     specified.
448  * 2 - THP was split.
449  * -EIO - is migration entry or only MPOL_MF_STRICT was specified and an
450  *        existing page was already on a node that does not follow the
451  *        policy.
452  */
453 static int queue_pages_pmd(pmd_t *pmd, spinlock_t *ptl, unsigned long addr,
454 				unsigned long end, struct mm_walk *walk)
455 	__releases(ptl)
456 {
457 	int ret = 0;
458 	struct page *page;
459 	struct queue_pages *qp = walk->private;
460 	unsigned long flags;
461 
462 	if (unlikely(is_pmd_migration_entry(*pmd))) {
463 		ret = -EIO;
464 		goto unlock;
465 	}
466 	page = pmd_page(*pmd);
467 	if (is_huge_zero_page(page)) {
468 		spin_unlock(ptl);
469 		walk->action = ACTION_CONTINUE;
470 		goto out;
471 	}
472 	if (!queue_pages_required(page, qp))
473 		goto unlock;
474 
475 	flags = qp->flags;
476 	/* go to thp migration */
477 	if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
478 		if (!vma_migratable(walk->vma) ||
479 		    migrate_page_add(page, qp->pagelist, flags)) {
480 			ret = 1;
481 			goto unlock;
482 		}
483 	} else
484 		ret = -EIO;
485 unlock:
486 	spin_unlock(ptl);
487 out:
488 	return ret;
489 }
490 
491 /*
492  * Scan through pages checking if pages follow certain conditions,
493  * and move them to the pagelist if they do.
494  *
495  * queue_pages_pte_range() has three possible return values:
496  * 0 - pages are placed on the right node or queued successfully, or
497  *     special page is met, i.e. zero page.
498  * 1 - there is unmovable page, and MPOL_MF_MOVE* & MPOL_MF_STRICT were
499  *     specified.
500  * -EIO - only MPOL_MF_STRICT was specified and an existing page was already
501  *        on a node that does not follow the policy.
502  */
503 static int queue_pages_pte_range(pmd_t *pmd, unsigned long addr,
504 			unsigned long end, struct mm_walk *walk)
505 {
506 	struct vm_area_struct *vma = walk->vma;
507 	struct page *page;
508 	struct queue_pages *qp = walk->private;
509 	unsigned long flags = qp->flags;
510 	int ret;
511 	bool has_unmovable = false;
512 	pte_t *pte, *mapped_pte;
513 	spinlock_t *ptl;
514 
515 	ptl = pmd_trans_huge_lock(pmd, vma);
516 	if (ptl) {
517 		ret = queue_pages_pmd(pmd, ptl, addr, end, walk);
518 		if (ret != 2)
519 			return ret;
520 	}
521 	/* THP was split, fall through to pte walk */
522 
523 	if (pmd_trans_unstable(pmd))
524 		return 0;
525 
526 	mapped_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
527 	for (; addr != end; pte++, addr += PAGE_SIZE) {
528 		if (!pte_present(*pte))
529 			continue;
530 		page = vm_normal_page(vma, addr, *pte);
531 		if (!page)
532 			continue;
533 		/*
534 		 * vm_normal_page() filters out zero pages, but there might
535 		 * still be PageReserved pages to skip, perhaps in a VDSO.
536 		 */
537 		if (PageReserved(page))
538 			continue;
539 		if (!queue_pages_required(page, qp))
540 			continue;
541 		if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
542 			/* MPOL_MF_STRICT must be specified if we get here */
543 			if (!vma_migratable(vma)) {
544 				has_unmovable = true;
545 				break;
546 			}
547 
548 			/*
549 			 * Do not abort immediately since there may be
550 			 * temporary off LRU pages in the range.  Still
551 			 * need migrate other LRU pages.
552 			 */
553 			if (migrate_page_add(page, qp->pagelist, flags))
554 				has_unmovable = true;
555 		} else
556 			break;
557 	}
558 	pte_unmap_unlock(mapped_pte, ptl);
559 	cond_resched();
560 
561 	if (has_unmovable)
562 		return 1;
563 
564 	return addr != end ? -EIO : 0;
565 }
566 
567 static int queue_pages_hugetlb(pte_t *pte, unsigned long hmask,
568 			       unsigned long addr, unsigned long end,
569 			       struct mm_walk *walk)
570 {
571 	int ret = 0;
572 #ifdef CONFIG_HUGETLB_PAGE
573 	struct queue_pages *qp = walk->private;
574 	unsigned long flags = (qp->flags & MPOL_MF_VALID);
575 	struct page *page;
576 	spinlock_t *ptl;
577 	pte_t entry;
578 
579 	ptl = huge_pte_lock(hstate_vma(walk->vma), walk->mm, pte);
580 	entry = huge_ptep_get(pte);
581 	if (!pte_present(entry))
582 		goto unlock;
583 	page = pte_page(entry);
584 	if (!queue_pages_required(page, qp))
585 		goto unlock;
586 
587 	if (flags == MPOL_MF_STRICT) {
588 		/*
589 		 * STRICT alone means only detecting misplaced page and no
590 		 * need to further check other vma.
591 		 */
592 		ret = -EIO;
593 		goto unlock;
594 	}
595 
596 	if (!vma_migratable(walk->vma)) {
597 		/*
598 		 * Must be STRICT with MOVE*, otherwise .test_walk() have
599 		 * stopped walking current vma.
600 		 * Detecting misplaced page but allow migrating pages which
601 		 * have been queued.
602 		 */
603 		ret = 1;
604 		goto unlock;
605 	}
606 
607 	/* With MPOL_MF_MOVE, we migrate only unshared hugepage. */
608 	if (flags & (MPOL_MF_MOVE_ALL) ||
609 	    (flags & MPOL_MF_MOVE && page_mapcount(page) == 1)) {
610 		if (!isolate_huge_page(page, qp->pagelist) &&
611 			(flags & MPOL_MF_STRICT))
612 			/*
613 			 * Failed to isolate page but allow migrating pages
614 			 * which have been queued.
615 			 */
616 			ret = 1;
617 	}
618 unlock:
619 	spin_unlock(ptl);
620 #else
621 	BUG();
622 #endif
623 	return ret;
624 }
625 
626 #ifdef CONFIG_NUMA_BALANCING
627 /*
628  * This is used to mark a range of virtual addresses to be inaccessible.
629  * These are later cleared by a NUMA hinting fault. Depending on these
630  * faults, pages may be migrated for better NUMA placement.
631  *
632  * This is assuming that NUMA faults are handled using PROT_NONE. If
633  * an architecture makes a different choice, it will need further
634  * changes to the core.
635  */
636 unsigned long change_prot_numa(struct vm_area_struct *vma,
637 			unsigned long addr, unsigned long end)
638 {
639 	int nr_updated;
640 
641 	nr_updated = change_protection(vma, addr, end, PAGE_NONE, MM_CP_PROT_NUMA);
642 	if (nr_updated)
643 		count_vm_numa_events(NUMA_PTE_UPDATES, nr_updated);
644 
645 	return nr_updated;
646 }
647 #else
648 static unsigned long change_prot_numa(struct vm_area_struct *vma,
649 			unsigned long addr, unsigned long end)
650 {
651 	return 0;
652 }
653 #endif /* CONFIG_NUMA_BALANCING */
654 
655 static int queue_pages_test_walk(unsigned long start, unsigned long end,
656 				struct mm_walk *walk)
657 {
658 	struct vm_area_struct *vma = walk->vma;
659 	struct queue_pages *qp = walk->private;
660 	unsigned long endvma = vma->vm_end;
661 	unsigned long flags = qp->flags;
662 
663 	/* range check first */
664 	VM_BUG_ON_VMA(!range_in_vma(vma, start, end), vma);
665 
666 	if (!qp->first) {
667 		qp->first = vma;
668 		if (!(flags & MPOL_MF_DISCONTIG_OK) &&
669 			(qp->start < vma->vm_start))
670 			/* hole at head side of range */
671 			return -EFAULT;
672 	}
673 	if (!(flags & MPOL_MF_DISCONTIG_OK) &&
674 		((vma->vm_end < qp->end) &&
675 		(!vma->vm_next || vma->vm_end < vma->vm_next->vm_start)))
676 		/* hole at middle or tail of range */
677 		return -EFAULT;
678 
679 	/*
680 	 * Need check MPOL_MF_STRICT to return -EIO if possible
681 	 * regardless of vma_migratable
682 	 */
683 	if (!vma_migratable(vma) &&
684 	    !(flags & MPOL_MF_STRICT))
685 		return 1;
686 
687 	if (endvma > end)
688 		endvma = end;
689 
690 	if (flags & MPOL_MF_LAZY) {
691 		/* Similar to task_numa_work, skip inaccessible VMAs */
692 		if (!is_vm_hugetlb_page(vma) && vma_is_accessible(vma) &&
693 			!(vma->vm_flags & VM_MIXEDMAP))
694 			change_prot_numa(vma, start, endvma);
695 		return 1;
696 	}
697 
698 	/* queue pages from current vma */
699 	if (flags & MPOL_MF_VALID)
700 		return 0;
701 	return 1;
702 }
703 
704 static const struct mm_walk_ops queue_pages_walk_ops = {
705 	.hugetlb_entry		= queue_pages_hugetlb,
706 	.pmd_entry		= queue_pages_pte_range,
707 	.test_walk		= queue_pages_test_walk,
708 };
709 
710 /*
711  * Walk through page tables and collect pages to be migrated.
712  *
713  * If pages found in a given range are on a set of nodes (determined by
714  * @nodes and @flags,) it's isolated and queued to the pagelist which is
715  * passed via @private.
716  *
717  * queue_pages_range() has three possible return values:
718  * 1 - there is unmovable page, but MPOL_MF_MOVE* & MPOL_MF_STRICT were
719  *     specified.
720  * 0 - queue pages successfully or no misplaced page.
721  * errno - i.e. misplaced pages with MPOL_MF_STRICT specified (-EIO) or
722  *         memory range specified by nodemask and maxnode points outside
723  *         your accessible address space (-EFAULT)
724  */
725 static int
726 queue_pages_range(struct mm_struct *mm, unsigned long start, unsigned long end,
727 		nodemask_t *nodes, unsigned long flags,
728 		struct list_head *pagelist)
729 {
730 	int err;
731 	struct queue_pages qp = {
732 		.pagelist = pagelist,
733 		.flags = flags,
734 		.nmask = nodes,
735 		.start = start,
736 		.end = end,
737 		.first = NULL,
738 	};
739 
740 	err = walk_page_range(mm, start, end, &queue_pages_walk_ops, &qp);
741 
742 	if (!qp.first)
743 		/* whole range in hole */
744 		err = -EFAULT;
745 
746 	return err;
747 }
748 
749 /*
750  * Apply policy to a single VMA
751  * This must be called with the mmap_lock held for writing.
752  */
753 static int vma_replace_policy(struct vm_area_struct *vma,
754 						struct mempolicy *pol)
755 {
756 	int err;
757 	struct mempolicy *old;
758 	struct mempolicy *new;
759 
760 	pr_debug("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n",
761 		 vma->vm_start, vma->vm_end, vma->vm_pgoff,
762 		 vma->vm_ops, vma->vm_file,
763 		 vma->vm_ops ? vma->vm_ops->set_policy : NULL);
764 
765 	new = mpol_dup(pol);
766 	if (IS_ERR(new))
767 		return PTR_ERR(new);
768 
769 	if (vma->vm_ops && vma->vm_ops->set_policy) {
770 		err = vma->vm_ops->set_policy(vma, new);
771 		if (err)
772 			goto err_out;
773 	}
774 
775 	old = vma->vm_policy;
776 	vma->vm_policy = new; /* protected by mmap_lock */
777 	mpol_put(old);
778 
779 	return 0;
780  err_out:
781 	mpol_put(new);
782 	return err;
783 }
784 
785 /* Step 2: apply policy to a range and do splits. */
786 static int mbind_range(struct mm_struct *mm, unsigned long start,
787 		       unsigned long end, struct mempolicy *new_pol)
788 {
789 	struct vm_area_struct *next;
790 	struct vm_area_struct *prev;
791 	struct vm_area_struct *vma;
792 	int err = 0;
793 	pgoff_t pgoff;
794 	unsigned long vmstart;
795 	unsigned long vmend;
796 
797 	vma = find_vma(mm, start);
798 	VM_BUG_ON(!vma);
799 
800 	prev = vma->vm_prev;
801 	if (start > vma->vm_start)
802 		prev = vma;
803 
804 	for (; vma && vma->vm_start < end; prev = vma, vma = next) {
805 		next = vma->vm_next;
806 		vmstart = max(start, vma->vm_start);
807 		vmend   = min(end, vma->vm_end);
808 
809 		if (mpol_equal(vma_policy(vma), new_pol))
810 			continue;
811 
812 		pgoff = vma->vm_pgoff +
813 			((vmstart - vma->vm_start) >> PAGE_SHIFT);
814 		prev = vma_merge(mm, prev, vmstart, vmend, vma->vm_flags,
815 				 vma->anon_vma, vma->vm_file, pgoff,
816 				 new_pol, vma->vm_userfaultfd_ctx,
817 				 vma_anon_name(vma));
818 		if (prev) {
819 			vma = prev;
820 			next = vma->vm_next;
821 			if (mpol_equal(vma_policy(vma), new_pol))
822 				continue;
823 			/* vma_merge() joined vma && vma->next, case 8 */
824 			goto replace;
825 		}
826 		if (vma->vm_start != vmstart) {
827 			err = split_vma(vma->vm_mm, vma, vmstart, 1);
828 			if (err)
829 				goto out;
830 		}
831 		if (vma->vm_end != vmend) {
832 			err = split_vma(vma->vm_mm, vma, vmend, 0);
833 			if (err)
834 				goto out;
835 		}
836  replace:
837 		err = vma_replace_policy(vma, new_pol);
838 		if (err)
839 			goto out;
840 	}
841 
842  out:
843 	return err;
844 }
845 
846 /* Set the process memory policy */
847 static long do_set_mempolicy(unsigned short mode, unsigned short flags,
848 			     nodemask_t *nodes)
849 {
850 	struct mempolicy *new, *old;
851 	NODEMASK_SCRATCH(scratch);
852 	int ret;
853 
854 	if (!scratch)
855 		return -ENOMEM;
856 
857 	new = mpol_new(mode, flags, nodes);
858 	if (IS_ERR(new)) {
859 		ret = PTR_ERR(new);
860 		goto out;
861 	}
862 
863 	ret = mpol_set_nodemask(new, nodes, scratch);
864 	if (ret) {
865 		mpol_put(new);
866 		goto out;
867 	}
868 	task_lock(current);
869 	old = current->mempolicy;
870 	current->mempolicy = new;
871 	if (new && new->mode == MPOL_INTERLEAVE)
872 		current->il_prev = MAX_NUMNODES-1;
873 	task_unlock(current);
874 	mpol_put(old);
875 	ret = 0;
876 out:
877 	NODEMASK_SCRATCH_FREE(scratch);
878 	return ret;
879 }
880 
881 /*
882  * Return nodemask for policy for get_mempolicy() query
883  *
884  * Called with task's alloc_lock held
885  */
886 static void get_policy_nodemask(struct mempolicy *p, nodemask_t *nodes)
887 {
888 	nodes_clear(*nodes);
889 	if (p == &default_policy)
890 		return;
891 
892 	switch (p->mode) {
893 	case MPOL_BIND:
894 	case MPOL_INTERLEAVE:
895 	case MPOL_PREFERRED:
896 	case MPOL_PREFERRED_MANY:
897 		*nodes = p->nodes;
898 		break;
899 	case MPOL_LOCAL:
900 		/* return empty node mask for local allocation */
901 		break;
902 	default:
903 		BUG();
904 	}
905 }
906 
907 static int lookup_node(struct mm_struct *mm, unsigned long addr)
908 {
909 	struct page *p = NULL;
910 	int err;
911 
912 	int locked = 1;
913 	err = get_user_pages_locked(addr & PAGE_MASK, 1, 0, &p, &locked);
914 	if (err > 0) {
915 		err = page_to_nid(p);
916 		put_page(p);
917 	}
918 	if (locked)
919 		mmap_read_unlock(mm);
920 	return err;
921 }
922 
923 /* Retrieve NUMA policy */
924 static long do_get_mempolicy(int *policy, nodemask_t *nmask,
925 			     unsigned long addr, unsigned long flags)
926 {
927 	int err;
928 	struct mm_struct *mm = current->mm;
929 	struct vm_area_struct *vma = NULL;
930 	struct mempolicy *pol = current->mempolicy, *pol_refcount = NULL;
931 
932 	if (flags &
933 		~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR|MPOL_F_MEMS_ALLOWED))
934 		return -EINVAL;
935 
936 	if (flags & MPOL_F_MEMS_ALLOWED) {
937 		if (flags & (MPOL_F_NODE|MPOL_F_ADDR))
938 			return -EINVAL;
939 		*policy = 0;	/* just so it's initialized */
940 		task_lock(current);
941 		*nmask  = cpuset_current_mems_allowed;
942 		task_unlock(current);
943 		return 0;
944 	}
945 
946 	if (flags & MPOL_F_ADDR) {
947 		/*
948 		 * Do NOT fall back to task policy if the
949 		 * vma/shared policy at addr is NULL.  We
950 		 * want to return MPOL_DEFAULT in this case.
951 		 */
952 		mmap_read_lock(mm);
953 		vma = vma_lookup(mm, addr);
954 		if (!vma) {
955 			mmap_read_unlock(mm);
956 			return -EFAULT;
957 		}
958 		if (vma->vm_ops && vma->vm_ops->get_policy)
959 			pol = vma->vm_ops->get_policy(vma, addr);
960 		else
961 			pol = vma->vm_policy;
962 	} else if (addr)
963 		return -EINVAL;
964 
965 	if (!pol)
966 		pol = &default_policy;	/* indicates default behavior */
967 
968 	if (flags & MPOL_F_NODE) {
969 		if (flags & MPOL_F_ADDR) {
970 			/*
971 			 * Take a refcount on the mpol, lookup_node()
972 			 * will drop the mmap_lock, so after calling
973 			 * lookup_node() only "pol" remains valid, "vma"
974 			 * is stale.
975 			 */
976 			pol_refcount = pol;
977 			vma = NULL;
978 			mpol_get(pol);
979 			err = lookup_node(mm, addr);
980 			if (err < 0)
981 				goto out;
982 			*policy = err;
983 		} else if (pol == current->mempolicy &&
984 				pol->mode == MPOL_INTERLEAVE) {
985 			*policy = next_node_in(current->il_prev, pol->nodes);
986 		} else {
987 			err = -EINVAL;
988 			goto out;
989 		}
990 	} else {
991 		*policy = pol == &default_policy ? MPOL_DEFAULT :
992 						pol->mode;
993 		/*
994 		 * Internal mempolicy flags must be masked off before exposing
995 		 * the policy to userspace.
996 		 */
997 		*policy |= (pol->flags & MPOL_MODE_FLAGS);
998 	}
999 
1000 	err = 0;
1001 	if (nmask) {
1002 		if (mpol_store_user_nodemask(pol)) {
1003 			*nmask = pol->w.user_nodemask;
1004 		} else {
1005 			task_lock(current);
1006 			get_policy_nodemask(pol, nmask);
1007 			task_unlock(current);
1008 		}
1009 	}
1010 
1011  out:
1012 	mpol_cond_put(pol);
1013 	if (vma)
1014 		mmap_read_unlock(mm);
1015 	if (pol_refcount)
1016 		mpol_put(pol_refcount);
1017 	return err;
1018 }
1019 
1020 #ifdef CONFIG_MIGRATION
1021 /*
1022  * page migration, thp tail pages can be passed.
1023  */
1024 static int migrate_page_add(struct page *page, struct list_head *pagelist,
1025 				unsigned long flags)
1026 {
1027 	struct page *head = compound_head(page);
1028 	/*
1029 	 * Avoid migrating a page that is shared with others.
1030 	 */
1031 	if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(head) == 1) {
1032 		if (!isolate_lru_page(head)) {
1033 			list_add_tail(&head->lru, pagelist);
1034 			mod_node_page_state(page_pgdat(head),
1035 				NR_ISOLATED_ANON + page_is_file_lru(head),
1036 				thp_nr_pages(head));
1037 		} else if (flags & MPOL_MF_STRICT) {
1038 			/*
1039 			 * Non-movable page may reach here.  And, there may be
1040 			 * temporary off LRU pages or non-LRU movable pages.
1041 			 * Treat them as unmovable pages since they can't be
1042 			 * isolated, so they can't be moved at the moment.  It
1043 			 * should return -EIO for this case too.
1044 			 */
1045 			return -EIO;
1046 		}
1047 	}
1048 
1049 	return 0;
1050 }
1051 
1052 /*
1053  * Migrate pages from one node to a target node.
1054  * Returns error or the number of pages not migrated.
1055  */
1056 static int migrate_to_node(struct mm_struct *mm, int source, int dest,
1057 			   int flags)
1058 {
1059 	nodemask_t nmask;
1060 	LIST_HEAD(pagelist);
1061 	int err = 0;
1062 	struct migration_target_control mtc = {
1063 		.nid = dest,
1064 		.gfp_mask = GFP_HIGHUSER_MOVABLE | __GFP_THISNODE,
1065 	};
1066 
1067 	nodes_clear(nmask);
1068 	node_set(source, nmask);
1069 
1070 	/*
1071 	 * This does not "check" the range but isolates all pages that
1072 	 * need migration.  Between passing in the full user address
1073 	 * space range and MPOL_MF_DISCONTIG_OK, this call can not fail.
1074 	 */
1075 	VM_BUG_ON(!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)));
1076 	queue_pages_range(mm, mm->mmap->vm_start, mm->task_size, &nmask,
1077 			flags | MPOL_MF_DISCONTIG_OK, &pagelist);
1078 
1079 	if (!list_empty(&pagelist)) {
1080 		err = migrate_pages(&pagelist, alloc_migration_target, NULL,
1081 				(unsigned long)&mtc, MIGRATE_SYNC, MR_SYSCALL, NULL);
1082 		if (err)
1083 			putback_movable_pages(&pagelist);
1084 	}
1085 
1086 	return err;
1087 }
1088 
1089 /*
1090  * Move pages between the two nodesets so as to preserve the physical
1091  * layout as much as possible.
1092  *
1093  * Returns the number of page that could not be moved.
1094  */
1095 int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
1096 		     const nodemask_t *to, int flags)
1097 {
1098 	int busy = 0;
1099 	int err = 0;
1100 	nodemask_t tmp;
1101 
1102 	lru_cache_disable();
1103 
1104 	mmap_read_lock(mm);
1105 
1106 	/*
1107 	 * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
1108 	 * bit in 'to' is not also set in 'tmp'.  Clear the found 'source'
1109 	 * bit in 'tmp', and return that <source, dest> pair for migration.
1110 	 * The pair of nodemasks 'to' and 'from' define the map.
1111 	 *
1112 	 * If no pair of bits is found that way, fallback to picking some
1113 	 * pair of 'source' and 'dest' bits that are not the same.  If the
1114 	 * 'source' and 'dest' bits are the same, this represents a node
1115 	 * that will be migrating to itself, so no pages need move.
1116 	 *
1117 	 * If no bits are left in 'tmp', or if all remaining bits left
1118 	 * in 'tmp' correspond to the same bit in 'to', return false
1119 	 * (nothing left to migrate).
1120 	 *
1121 	 * This lets us pick a pair of nodes to migrate between, such that
1122 	 * if possible the dest node is not already occupied by some other
1123 	 * source node, minimizing the risk of overloading the memory on a
1124 	 * node that would happen if we migrated incoming memory to a node
1125 	 * before migrating outgoing memory source that same node.
1126 	 *
1127 	 * A single scan of tmp is sufficient.  As we go, we remember the
1128 	 * most recent <s, d> pair that moved (s != d).  If we find a pair
1129 	 * that not only moved, but what's better, moved to an empty slot
1130 	 * (d is not set in tmp), then we break out then, with that pair.
1131 	 * Otherwise when we finish scanning from_tmp, we at least have the
1132 	 * most recent <s, d> pair that moved.  If we get all the way through
1133 	 * the scan of tmp without finding any node that moved, much less
1134 	 * moved to an empty node, then there is nothing left worth migrating.
1135 	 */
1136 
1137 	tmp = *from;
1138 	while (!nodes_empty(tmp)) {
1139 		int s, d;
1140 		int source = NUMA_NO_NODE;
1141 		int dest = 0;
1142 
1143 		for_each_node_mask(s, tmp) {
1144 
1145 			/*
1146 			 * do_migrate_pages() tries to maintain the relative
1147 			 * node relationship of the pages established between
1148 			 * threads and memory areas.
1149                          *
1150 			 * However if the number of source nodes is not equal to
1151 			 * the number of destination nodes we can not preserve
1152 			 * this node relative relationship.  In that case, skip
1153 			 * copying memory from a node that is in the destination
1154 			 * mask.
1155 			 *
1156 			 * Example: [2,3,4] -> [3,4,5] moves everything.
1157 			 *          [0-7] - > [3,4,5] moves only 0,1,2,6,7.
1158 			 */
1159 
1160 			if ((nodes_weight(*from) != nodes_weight(*to)) &&
1161 						(node_isset(s, *to)))
1162 				continue;
1163 
1164 			d = node_remap(s, *from, *to);
1165 			if (s == d)
1166 				continue;
1167 
1168 			source = s;	/* Node moved. Memorize */
1169 			dest = d;
1170 
1171 			/* dest not in remaining from nodes? */
1172 			if (!node_isset(dest, tmp))
1173 				break;
1174 		}
1175 		if (source == NUMA_NO_NODE)
1176 			break;
1177 
1178 		node_clear(source, tmp);
1179 		err = migrate_to_node(mm, source, dest, flags);
1180 		if (err > 0)
1181 			busy += err;
1182 		if (err < 0)
1183 			break;
1184 	}
1185 	mmap_read_unlock(mm);
1186 
1187 	lru_cache_enable();
1188 	if (err < 0)
1189 		return err;
1190 	return busy;
1191 
1192 }
1193 
1194 /*
1195  * Allocate a new page for page migration based on vma policy.
1196  * Start by assuming the page is mapped by the same vma as contains @start.
1197  * Search forward from there, if not.  N.B., this assumes that the
1198  * list of pages handed to migrate_pages()--which is how we get here--
1199  * is in virtual address order.
1200  */
1201 static struct page *new_page(struct page *page, unsigned long start)
1202 {
1203 	struct vm_area_struct *vma;
1204 	unsigned long address;
1205 
1206 	vma = find_vma(current->mm, start);
1207 	while (vma) {
1208 		address = page_address_in_vma(page, vma);
1209 		if (address != -EFAULT)
1210 			break;
1211 		vma = vma->vm_next;
1212 	}
1213 
1214 	if (PageHuge(page)) {
1215 		return alloc_huge_page_vma(page_hstate(compound_head(page)),
1216 				vma, address);
1217 	} else if (PageTransHuge(page)) {
1218 		struct page *thp;
1219 
1220 		thp = alloc_hugepage_vma(GFP_TRANSHUGE, vma, address,
1221 					 HPAGE_PMD_ORDER);
1222 		if (!thp)
1223 			return NULL;
1224 		prep_transhuge_page(thp);
1225 		return thp;
1226 	}
1227 	/*
1228 	 * if !vma, alloc_page_vma() will use task or system default policy
1229 	 */
1230 	return alloc_page_vma(GFP_HIGHUSER_MOVABLE | __GFP_RETRY_MAYFAIL,
1231 			vma, address);
1232 }
1233 #else
1234 
1235 static int migrate_page_add(struct page *page, struct list_head *pagelist,
1236 				unsigned long flags)
1237 {
1238 	return -EIO;
1239 }
1240 
1241 int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
1242 		     const nodemask_t *to, int flags)
1243 {
1244 	return -ENOSYS;
1245 }
1246 
1247 static struct page *new_page(struct page *page, unsigned long start)
1248 {
1249 	return NULL;
1250 }
1251 #endif
1252 
1253 static long do_mbind(unsigned long start, unsigned long len,
1254 		     unsigned short mode, unsigned short mode_flags,
1255 		     nodemask_t *nmask, unsigned long flags)
1256 {
1257 	struct mm_struct *mm = current->mm;
1258 	struct mempolicy *new;
1259 	unsigned long end;
1260 	int err;
1261 	int ret;
1262 	LIST_HEAD(pagelist);
1263 
1264 	if (flags & ~(unsigned long)MPOL_MF_VALID)
1265 		return -EINVAL;
1266 	if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
1267 		return -EPERM;
1268 
1269 	if (start & ~PAGE_MASK)
1270 		return -EINVAL;
1271 
1272 	if (mode == MPOL_DEFAULT)
1273 		flags &= ~MPOL_MF_STRICT;
1274 
1275 	len = (len + PAGE_SIZE - 1) & PAGE_MASK;
1276 	end = start + len;
1277 
1278 	if (end < start)
1279 		return -EINVAL;
1280 	if (end == start)
1281 		return 0;
1282 
1283 	new = mpol_new(mode, mode_flags, nmask);
1284 	if (IS_ERR(new))
1285 		return PTR_ERR(new);
1286 
1287 	if (flags & MPOL_MF_LAZY)
1288 		new->flags |= MPOL_F_MOF;
1289 
1290 	/*
1291 	 * If we are using the default policy then operation
1292 	 * on discontinuous address spaces is okay after all
1293 	 */
1294 	if (!new)
1295 		flags |= MPOL_MF_DISCONTIG_OK;
1296 
1297 	pr_debug("mbind %lx-%lx mode:%d flags:%d nodes:%lx\n",
1298 		 start, start + len, mode, mode_flags,
1299 		 nmask ? nodes_addr(*nmask)[0] : NUMA_NO_NODE);
1300 
1301 	if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
1302 
1303 		lru_cache_disable();
1304 	}
1305 	{
1306 		NODEMASK_SCRATCH(scratch);
1307 		if (scratch) {
1308 			mmap_write_lock(mm);
1309 			err = mpol_set_nodemask(new, nmask, scratch);
1310 			if (err)
1311 				mmap_write_unlock(mm);
1312 		} else
1313 			err = -ENOMEM;
1314 		NODEMASK_SCRATCH_FREE(scratch);
1315 	}
1316 	if (err)
1317 		goto mpol_out;
1318 
1319 	ret = queue_pages_range(mm, start, end, nmask,
1320 			  flags | MPOL_MF_INVERT, &pagelist);
1321 
1322 	if (ret < 0) {
1323 		err = ret;
1324 		goto up_out;
1325 	}
1326 
1327 	err = mbind_range(mm, start, end, new);
1328 
1329 	if (!err) {
1330 		int nr_failed = 0;
1331 
1332 		if (!list_empty(&pagelist)) {
1333 			WARN_ON_ONCE(flags & MPOL_MF_LAZY);
1334 			nr_failed = migrate_pages(&pagelist, new_page, NULL,
1335 				start, MIGRATE_SYNC, MR_MEMPOLICY_MBIND, NULL);
1336 			if (nr_failed)
1337 				putback_movable_pages(&pagelist);
1338 		}
1339 
1340 		if ((ret > 0) || (nr_failed && (flags & MPOL_MF_STRICT)))
1341 			err = -EIO;
1342 	} else {
1343 up_out:
1344 		if (!list_empty(&pagelist))
1345 			putback_movable_pages(&pagelist);
1346 	}
1347 
1348 	mmap_write_unlock(mm);
1349 mpol_out:
1350 	mpol_put(new);
1351 	if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
1352 		lru_cache_enable();
1353 	return err;
1354 }
1355 
1356 /*
1357  * User space interface with variable sized bitmaps for nodelists.
1358  */
1359 static int get_bitmap(unsigned long *mask, const unsigned long __user *nmask,
1360 		      unsigned long maxnode)
1361 {
1362 	unsigned long nlongs = BITS_TO_LONGS(maxnode);
1363 	int ret;
1364 
1365 	if (in_compat_syscall())
1366 		ret = compat_get_bitmap(mask,
1367 					(const compat_ulong_t __user *)nmask,
1368 					maxnode);
1369 	else
1370 		ret = copy_from_user(mask, nmask,
1371 				     nlongs * sizeof(unsigned long));
1372 
1373 	if (ret)
1374 		return -EFAULT;
1375 
1376 	if (maxnode % BITS_PER_LONG)
1377 		mask[nlongs - 1] &= (1UL << (maxnode % BITS_PER_LONG)) - 1;
1378 
1379 	return 0;
1380 }
1381 
1382 /* Copy a node mask from user space. */
1383 static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask,
1384 		     unsigned long maxnode)
1385 {
1386 	--maxnode;
1387 	nodes_clear(*nodes);
1388 	if (maxnode == 0 || !nmask)
1389 		return 0;
1390 	if (maxnode > PAGE_SIZE*BITS_PER_BYTE)
1391 		return -EINVAL;
1392 
1393 	/*
1394 	 * When the user specified more nodes than supported just check
1395 	 * if the non supported part is all zero, one word at a time,
1396 	 * starting at the end.
1397 	 */
1398 	while (maxnode > MAX_NUMNODES) {
1399 		unsigned long bits = min_t(unsigned long, maxnode, BITS_PER_LONG);
1400 		unsigned long t;
1401 
1402 		if (get_bitmap(&t, &nmask[maxnode / BITS_PER_LONG], bits))
1403 			return -EFAULT;
1404 
1405 		if (maxnode - bits >= MAX_NUMNODES) {
1406 			maxnode -= bits;
1407 		} else {
1408 			maxnode = MAX_NUMNODES;
1409 			t &= ~((1UL << (MAX_NUMNODES % BITS_PER_LONG)) - 1);
1410 		}
1411 		if (t)
1412 			return -EINVAL;
1413 	}
1414 
1415 	return get_bitmap(nodes_addr(*nodes), nmask, maxnode);
1416 }
1417 
1418 /* Copy a kernel node mask to user space */
1419 static int copy_nodes_to_user(unsigned long __user *mask, unsigned long maxnode,
1420 			      nodemask_t *nodes)
1421 {
1422 	unsigned long copy = ALIGN(maxnode-1, 64) / 8;
1423 	unsigned int nbytes = BITS_TO_LONGS(nr_node_ids) * sizeof(long);
1424 	bool compat = in_compat_syscall();
1425 
1426 	if (compat)
1427 		nbytes = BITS_TO_COMPAT_LONGS(nr_node_ids) * sizeof(compat_long_t);
1428 
1429 	if (copy > nbytes) {
1430 		if (copy > PAGE_SIZE)
1431 			return -EINVAL;
1432 		if (clear_user((char __user *)mask + nbytes, copy - nbytes))
1433 			return -EFAULT;
1434 		copy = nbytes;
1435 		maxnode = nr_node_ids;
1436 	}
1437 
1438 	if (compat)
1439 		return compat_put_bitmap((compat_ulong_t __user *)mask,
1440 					 nodes_addr(*nodes), maxnode);
1441 
1442 	return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0;
1443 }
1444 
1445 /* Basic parameter sanity check used by both mbind() and set_mempolicy() */
1446 static inline int sanitize_mpol_flags(int *mode, unsigned short *flags)
1447 {
1448 	*flags = *mode & MPOL_MODE_FLAGS;
1449 	*mode &= ~MPOL_MODE_FLAGS;
1450 
1451 	if ((unsigned int)(*mode) >=  MPOL_MAX)
1452 		return -EINVAL;
1453 	if ((*flags & MPOL_F_STATIC_NODES) && (*flags & MPOL_F_RELATIVE_NODES))
1454 		return -EINVAL;
1455 	if (*flags & MPOL_F_NUMA_BALANCING) {
1456 		if (*mode != MPOL_BIND)
1457 			return -EINVAL;
1458 		*flags |= (MPOL_F_MOF | MPOL_F_MORON);
1459 	}
1460 	return 0;
1461 }
1462 
1463 static long kernel_mbind(unsigned long start, unsigned long len,
1464 			 unsigned long mode, const unsigned long __user *nmask,
1465 			 unsigned long maxnode, unsigned int flags)
1466 {
1467 	unsigned short mode_flags;
1468 	nodemask_t nodes;
1469 	int lmode = mode;
1470 	int err;
1471 
1472 	start = untagged_addr(start);
1473 	err = sanitize_mpol_flags(&lmode, &mode_flags);
1474 	if (err)
1475 		return err;
1476 
1477 	err = get_nodes(&nodes, nmask, maxnode);
1478 	if (err)
1479 		return err;
1480 
1481 	return do_mbind(start, len, lmode, mode_flags, &nodes, flags);
1482 }
1483 
1484 SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, len,
1485 		unsigned long, home_node, unsigned long, flags)
1486 {
1487 	struct mm_struct *mm = current->mm;
1488 	struct vm_area_struct *vma;
1489 	struct mempolicy *new;
1490 	unsigned long vmstart;
1491 	unsigned long vmend;
1492 	unsigned long end;
1493 	int err = -ENOENT;
1494 
1495 	start = untagged_addr(start);
1496 	if (start & ~PAGE_MASK)
1497 		return -EINVAL;
1498 	/*
1499 	 * flags is used for future extension if any.
1500 	 */
1501 	if (flags != 0)
1502 		return -EINVAL;
1503 
1504 	/*
1505 	 * Check home_node is online to avoid accessing uninitialized
1506 	 * NODE_DATA.
1507 	 */
1508 	if (home_node >= MAX_NUMNODES || !node_online(home_node))
1509 		return -EINVAL;
1510 
1511 	len = (len + PAGE_SIZE - 1) & PAGE_MASK;
1512 	end = start + len;
1513 
1514 	if (end < start)
1515 		return -EINVAL;
1516 	if (end == start)
1517 		return 0;
1518 	mmap_write_lock(mm);
1519 	vma = find_vma(mm, start);
1520 	for (; vma && vma->vm_start < end;  vma = vma->vm_next) {
1521 
1522 		vmstart = max(start, vma->vm_start);
1523 		vmend   = min(end, vma->vm_end);
1524 		new = mpol_dup(vma_policy(vma));
1525 		if (IS_ERR(new)) {
1526 			err = PTR_ERR(new);
1527 			break;
1528 		}
1529 		/*
1530 		 * Only update home node if there is an existing vma policy
1531 		 */
1532 		if (!new)
1533 			continue;
1534 
1535 		/*
1536 		 * If any vma in the range got policy other than MPOL_BIND
1537 		 * or MPOL_PREFERRED_MANY we return error. We don't reset
1538 		 * the home node for vmas we already updated before.
1539 		 */
1540 		if (new->mode != MPOL_BIND && new->mode != MPOL_PREFERRED_MANY) {
1541 			err = -EOPNOTSUPP;
1542 			break;
1543 		}
1544 
1545 		new->home_node = home_node;
1546 		err = mbind_range(mm, vmstart, vmend, new);
1547 		mpol_put(new);
1548 		if (err)
1549 			break;
1550 	}
1551 	mmap_write_unlock(mm);
1552 	return err;
1553 }
1554 
1555 SYSCALL_DEFINE6(mbind, unsigned long, start, unsigned long, len,
1556 		unsigned long, mode, const unsigned long __user *, nmask,
1557 		unsigned long, maxnode, unsigned int, flags)
1558 {
1559 	return kernel_mbind(start, len, mode, nmask, maxnode, flags);
1560 }
1561 
1562 /* Set the process memory policy */
1563 static long kernel_set_mempolicy(int mode, const unsigned long __user *nmask,
1564 				 unsigned long maxnode)
1565 {
1566 	unsigned short mode_flags;
1567 	nodemask_t nodes;
1568 	int lmode = mode;
1569 	int err;
1570 
1571 	err = sanitize_mpol_flags(&lmode, &mode_flags);
1572 	if (err)
1573 		return err;
1574 
1575 	err = get_nodes(&nodes, nmask, maxnode);
1576 	if (err)
1577 		return err;
1578 
1579 	return do_set_mempolicy(lmode, mode_flags, &nodes);
1580 }
1581 
1582 SYSCALL_DEFINE3(set_mempolicy, int, mode, const unsigned long __user *, nmask,
1583 		unsigned long, maxnode)
1584 {
1585 	return kernel_set_mempolicy(mode, nmask, maxnode);
1586 }
1587 
1588 static int kernel_migrate_pages(pid_t pid, unsigned long maxnode,
1589 				const unsigned long __user *old_nodes,
1590 				const unsigned long __user *new_nodes)
1591 {
1592 	struct mm_struct *mm = NULL;
1593 	struct task_struct *task;
1594 	nodemask_t task_nodes;
1595 	int err;
1596 	nodemask_t *old;
1597 	nodemask_t *new;
1598 	NODEMASK_SCRATCH(scratch);
1599 
1600 	if (!scratch)
1601 		return -ENOMEM;
1602 
1603 	old = &scratch->mask1;
1604 	new = &scratch->mask2;
1605 
1606 	err = get_nodes(old, old_nodes, maxnode);
1607 	if (err)
1608 		goto out;
1609 
1610 	err = get_nodes(new, new_nodes, maxnode);
1611 	if (err)
1612 		goto out;
1613 
1614 	/* Find the mm_struct */
1615 	rcu_read_lock();
1616 	task = pid ? find_task_by_vpid(pid) : current;
1617 	if (!task) {
1618 		rcu_read_unlock();
1619 		err = -ESRCH;
1620 		goto out;
1621 	}
1622 	get_task_struct(task);
1623 
1624 	err = -EINVAL;
1625 
1626 	/*
1627 	 * Check if this process has the right to modify the specified process.
1628 	 * Use the regular "ptrace_may_access()" checks.
1629 	 */
1630 	if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) {
1631 		rcu_read_unlock();
1632 		err = -EPERM;
1633 		goto out_put;
1634 	}
1635 	rcu_read_unlock();
1636 
1637 	task_nodes = cpuset_mems_allowed(task);
1638 	/* Is the user allowed to access the target nodes? */
1639 	if (!nodes_subset(*new, task_nodes) && !capable(CAP_SYS_NICE)) {
1640 		err = -EPERM;
1641 		goto out_put;
1642 	}
1643 
1644 	task_nodes = cpuset_mems_allowed(current);
1645 	nodes_and(*new, *new, task_nodes);
1646 	if (nodes_empty(*new))
1647 		goto out_put;
1648 
1649 	err = security_task_movememory(task);
1650 	if (err)
1651 		goto out_put;
1652 
1653 	mm = get_task_mm(task);
1654 	put_task_struct(task);
1655 
1656 	if (!mm) {
1657 		err = -EINVAL;
1658 		goto out;
1659 	}
1660 
1661 	err = do_migrate_pages(mm, old, new,
1662 		capable(CAP_SYS_NICE) ? MPOL_MF_MOVE_ALL : MPOL_MF_MOVE);
1663 
1664 	mmput(mm);
1665 out:
1666 	NODEMASK_SCRATCH_FREE(scratch);
1667 
1668 	return err;
1669 
1670 out_put:
1671 	put_task_struct(task);
1672 	goto out;
1673 
1674 }
1675 
1676 SYSCALL_DEFINE4(migrate_pages, pid_t, pid, unsigned long, maxnode,
1677 		const unsigned long __user *, old_nodes,
1678 		const unsigned long __user *, new_nodes)
1679 {
1680 	return kernel_migrate_pages(pid, maxnode, old_nodes, new_nodes);
1681 }
1682 
1683 
1684 /* Retrieve NUMA policy */
1685 static int kernel_get_mempolicy(int __user *policy,
1686 				unsigned long __user *nmask,
1687 				unsigned long maxnode,
1688 				unsigned long addr,
1689 				unsigned long flags)
1690 {
1691 	int err;
1692 	int pval;
1693 	nodemask_t nodes;
1694 
1695 	if (nmask != NULL && maxnode < nr_node_ids)
1696 		return -EINVAL;
1697 
1698 	addr = untagged_addr(addr);
1699 
1700 	err = do_get_mempolicy(&pval, &nodes, addr, flags);
1701 
1702 	if (err)
1703 		return err;
1704 
1705 	if (policy && put_user(pval, policy))
1706 		return -EFAULT;
1707 
1708 	if (nmask)
1709 		err = copy_nodes_to_user(nmask, maxnode, &nodes);
1710 
1711 	return err;
1712 }
1713 
1714 SYSCALL_DEFINE5(get_mempolicy, int __user *, policy,
1715 		unsigned long __user *, nmask, unsigned long, maxnode,
1716 		unsigned long, addr, unsigned long, flags)
1717 {
1718 	return kernel_get_mempolicy(policy, nmask, maxnode, addr, flags);
1719 }
1720 
1721 bool vma_migratable(struct vm_area_struct *vma)
1722 {
1723 	if (vma->vm_flags & (VM_IO | VM_PFNMAP))
1724 		return false;
1725 
1726 	/*
1727 	 * DAX device mappings require predictable access latency, so avoid
1728 	 * incurring periodic faults.
1729 	 */
1730 	if (vma_is_dax(vma))
1731 		return false;
1732 
1733 	if (is_vm_hugetlb_page(vma) &&
1734 		!hugepage_migration_supported(hstate_vma(vma)))
1735 		return false;
1736 
1737 	/*
1738 	 * Migration allocates pages in the highest zone. If we cannot
1739 	 * do so then migration (at least from node to node) is not
1740 	 * possible.
1741 	 */
1742 	if (vma->vm_file &&
1743 		gfp_zone(mapping_gfp_mask(vma->vm_file->f_mapping))
1744 			< policy_zone)
1745 		return false;
1746 	return true;
1747 }
1748 
1749 struct mempolicy *__get_vma_policy(struct vm_area_struct *vma,
1750 						unsigned long addr)
1751 {
1752 	struct mempolicy *pol = NULL;
1753 
1754 	if (vma) {
1755 		if (vma->vm_ops && vma->vm_ops->get_policy) {
1756 			pol = vma->vm_ops->get_policy(vma, addr);
1757 		} else if (vma->vm_policy) {
1758 			pol = vma->vm_policy;
1759 
1760 			/*
1761 			 * shmem_alloc_page() passes MPOL_F_SHARED policy with
1762 			 * a pseudo vma whose vma->vm_ops=NULL. Take a reference
1763 			 * count on these policies which will be dropped by
1764 			 * mpol_cond_put() later
1765 			 */
1766 			if (mpol_needs_cond_ref(pol))
1767 				mpol_get(pol);
1768 		}
1769 	}
1770 
1771 	return pol;
1772 }
1773 
1774 /*
1775  * get_vma_policy(@vma, @addr)
1776  * @vma: virtual memory area whose policy is sought
1777  * @addr: address in @vma for shared policy lookup
1778  *
1779  * Returns effective policy for a VMA at specified address.
1780  * Falls back to current->mempolicy or system default policy, as necessary.
1781  * Shared policies [those marked as MPOL_F_SHARED] require an extra reference
1782  * count--added by the get_policy() vm_op, as appropriate--to protect against
1783  * freeing by another task.  It is the caller's responsibility to free the
1784  * extra reference for shared policies.
1785  */
1786 static struct mempolicy *get_vma_policy(struct vm_area_struct *vma,
1787 						unsigned long addr)
1788 {
1789 	struct mempolicy *pol = __get_vma_policy(vma, addr);
1790 
1791 	if (!pol)
1792 		pol = get_task_policy(current);
1793 
1794 	return pol;
1795 }
1796 
1797 bool vma_policy_mof(struct vm_area_struct *vma)
1798 {
1799 	struct mempolicy *pol;
1800 
1801 	if (vma->vm_ops && vma->vm_ops->get_policy) {
1802 		bool ret = false;
1803 
1804 		pol = vma->vm_ops->get_policy(vma, vma->vm_start);
1805 		if (pol && (pol->flags & MPOL_F_MOF))
1806 			ret = true;
1807 		mpol_cond_put(pol);
1808 
1809 		return ret;
1810 	}
1811 
1812 	pol = vma->vm_policy;
1813 	if (!pol)
1814 		pol = get_task_policy(current);
1815 
1816 	return pol->flags & MPOL_F_MOF;
1817 }
1818 
1819 static int apply_policy_zone(struct mempolicy *policy, enum zone_type zone)
1820 {
1821 	enum zone_type dynamic_policy_zone = policy_zone;
1822 
1823 	BUG_ON(dynamic_policy_zone == ZONE_MOVABLE);
1824 
1825 	/*
1826 	 * if policy->nodes has movable memory only,
1827 	 * we apply policy when gfp_zone(gfp) = ZONE_MOVABLE only.
1828 	 *
1829 	 * policy->nodes is intersect with node_states[N_MEMORY].
1830 	 * so if the following test fails, it implies
1831 	 * policy->nodes has movable memory only.
1832 	 */
1833 	if (!nodes_intersects(policy->nodes, node_states[N_HIGH_MEMORY]))
1834 		dynamic_policy_zone = ZONE_MOVABLE;
1835 
1836 	return zone >= dynamic_policy_zone;
1837 }
1838 
1839 /*
1840  * Return a nodemask representing a mempolicy for filtering nodes for
1841  * page allocation
1842  */
1843 nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *policy)
1844 {
1845 	int mode = policy->mode;
1846 
1847 	/* Lower zones don't get a nodemask applied for MPOL_BIND */
1848 	if (unlikely(mode == MPOL_BIND) &&
1849 		apply_policy_zone(policy, gfp_zone(gfp)) &&
1850 		cpuset_nodemask_valid_mems_allowed(&policy->nodes))
1851 		return &policy->nodes;
1852 
1853 	if (mode == MPOL_PREFERRED_MANY)
1854 		return &policy->nodes;
1855 
1856 	return NULL;
1857 }
1858 
1859 /*
1860  * Return the  preferred node id for 'prefer' mempolicy, and return
1861  * the given id for all other policies.
1862  *
1863  * policy_node() is always coupled with policy_nodemask(), which
1864  * secures the nodemask limit for 'bind' and 'prefer-many' policy.
1865  */
1866 static int policy_node(gfp_t gfp, struct mempolicy *policy, int nd)
1867 {
1868 	if (policy->mode == MPOL_PREFERRED) {
1869 		nd = first_node(policy->nodes);
1870 	} else {
1871 		/*
1872 		 * __GFP_THISNODE shouldn't even be used with the bind policy
1873 		 * because we might easily break the expectation to stay on the
1874 		 * requested node and not break the policy.
1875 		 */
1876 		WARN_ON_ONCE(policy->mode == MPOL_BIND && (gfp & __GFP_THISNODE));
1877 	}
1878 
1879 	if ((policy->mode == MPOL_BIND ||
1880 	     policy->mode == MPOL_PREFERRED_MANY) &&
1881 	    policy->home_node != NUMA_NO_NODE)
1882 		return policy->home_node;
1883 
1884 	return nd;
1885 }
1886 
1887 /* Do dynamic interleaving for a process */
1888 static unsigned interleave_nodes(struct mempolicy *policy)
1889 {
1890 	unsigned next;
1891 	struct task_struct *me = current;
1892 
1893 	next = next_node_in(me->il_prev, policy->nodes);
1894 	if (next < MAX_NUMNODES)
1895 		me->il_prev = next;
1896 	return next;
1897 }
1898 
1899 /*
1900  * Depending on the memory policy provide a node from which to allocate the
1901  * next slab entry.
1902  */
1903 unsigned int mempolicy_slab_node(void)
1904 {
1905 	struct mempolicy *policy;
1906 	int node = numa_mem_id();
1907 
1908 	if (!in_task())
1909 		return node;
1910 
1911 	policy = current->mempolicy;
1912 	if (!policy)
1913 		return node;
1914 
1915 	switch (policy->mode) {
1916 	case MPOL_PREFERRED:
1917 		return first_node(policy->nodes);
1918 
1919 	case MPOL_INTERLEAVE:
1920 		return interleave_nodes(policy);
1921 
1922 	case MPOL_BIND:
1923 	case MPOL_PREFERRED_MANY:
1924 	{
1925 		struct zoneref *z;
1926 
1927 		/*
1928 		 * Follow bind policy behavior and start allocation at the
1929 		 * first node.
1930 		 */
1931 		struct zonelist *zonelist;
1932 		enum zone_type highest_zoneidx = gfp_zone(GFP_KERNEL);
1933 		zonelist = &NODE_DATA(node)->node_zonelists[ZONELIST_FALLBACK];
1934 		z = first_zones_zonelist(zonelist, highest_zoneidx,
1935 							&policy->nodes);
1936 		return z->zone ? zone_to_nid(z->zone) : node;
1937 	}
1938 	case MPOL_LOCAL:
1939 		return node;
1940 
1941 	default:
1942 		BUG();
1943 	}
1944 }
1945 
1946 /*
1947  * Do static interleaving for a VMA with known offset @n.  Returns the n'th
1948  * node in pol->nodes (starting from n=0), wrapping around if n exceeds the
1949  * number of present nodes.
1950  */
1951 static unsigned offset_il_node(struct mempolicy *pol, unsigned long n)
1952 {
1953 	nodemask_t nodemask = pol->nodes;
1954 	unsigned int target, nnodes;
1955 	int i;
1956 	int nid;
1957 	/*
1958 	 * The barrier will stabilize the nodemask in a register or on
1959 	 * the stack so that it will stop changing under the code.
1960 	 *
1961 	 * Between first_node() and next_node(), pol->nodes could be changed
1962 	 * by other threads. So we put pol->nodes in a local stack.
1963 	 */
1964 	barrier();
1965 
1966 	nnodes = nodes_weight(nodemask);
1967 	if (!nnodes)
1968 		return numa_node_id();
1969 	target = (unsigned int)n % nnodes;
1970 	nid = first_node(nodemask);
1971 	for (i = 0; i < target; i++)
1972 		nid = next_node(nid, nodemask);
1973 	return nid;
1974 }
1975 
1976 /* Determine a node number for interleave */
1977 static inline unsigned interleave_nid(struct mempolicy *pol,
1978 		 struct vm_area_struct *vma, unsigned long addr, int shift)
1979 {
1980 	if (vma) {
1981 		unsigned long off;
1982 
1983 		/*
1984 		 * for small pages, there is no difference between
1985 		 * shift and PAGE_SHIFT, so the bit-shift is safe.
1986 		 * for huge pages, since vm_pgoff is in units of small
1987 		 * pages, we need to shift off the always 0 bits to get
1988 		 * a useful offset.
1989 		 */
1990 		BUG_ON(shift < PAGE_SHIFT);
1991 		off = vma->vm_pgoff >> (shift - PAGE_SHIFT);
1992 		off += (addr - vma->vm_start) >> shift;
1993 		return offset_il_node(pol, off);
1994 	} else
1995 		return interleave_nodes(pol);
1996 }
1997 
1998 #ifdef CONFIG_HUGETLBFS
1999 /*
2000  * huge_node(@vma, @addr, @gfp_flags, @mpol)
2001  * @vma: virtual memory area whose policy is sought
2002  * @addr: address in @vma for shared policy lookup and interleave policy
2003  * @gfp_flags: for requested zone
2004  * @mpol: pointer to mempolicy pointer for reference counted mempolicy
2005  * @nodemask: pointer to nodemask pointer for 'bind' and 'prefer-many' policy
2006  *
2007  * Returns a nid suitable for a huge page allocation and a pointer
2008  * to the struct mempolicy for conditional unref after allocation.
2009  * If the effective policy is 'bind' or 'prefer-many', returns a pointer
2010  * to the mempolicy's @nodemask for filtering the zonelist.
2011  *
2012  * Must be protected by read_mems_allowed_begin()
2013  */
2014 int huge_node(struct vm_area_struct *vma, unsigned long addr, gfp_t gfp_flags,
2015 				struct mempolicy **mpol, nodemask_t **nodemask)
2016 {
2017 	int nid;
2018 	int mode;
2019 
2020 	*mpol = get_vma_policy(vma, addr);
2021 	*nodemask = NULL;
2022 	mode = (*mpol)->mode;
2023 
2024 	if (unlikely(mode == MPOL_INTERLEAVE)) {
2025 		nid = interleave_nid(*mpol, vma, addr,
2026 					huge_page_shift(hstate_vma(vma)));
2027 	} else {
2028 		nid = policy_node(gfp_flags, *mpol, numa_node_id());
2029 		if (mode == MPOL_BIND || mode == MPOL_PREFERRED_MANY)
2030 			*nodemask = &(*mpol)->nodes;
2031 	}
2032 	return nid;
2033 }
2034 
2035 /*
2036  * init_nodemask_of_mempolicy
2037  *
2038  * If the current task's mempolicy is "default" [NULL], return 'false'
2039  * to indicate default policy.  Otherwise, extract the policy nodemask
2040  * for 'bind' or 'interleave' policy into the argument nodemask, or
2041  * initialize the argument nodemask to contain the single node for
2042  * 'preferred' or 'local' policy and return 'true' to indicate presence
2043  * of non-default mempolicy.
2044  *
2045  * We don't bother with reference counting the mempolicy [mpol_get/put]
2046  * because the current task is examining it's own mempolicy and a task's
2047  * mempolicy is only ever changed by the task itself.
2048  *
2049  * N.B., it is the caller's responsibility to free a returned nodemask.
2050  */
2051 bool init_nodemask_of_mempolicy(nodemask_t *mask)
2052 {
2053 	struct mempolicy *mempolicy;
2054 
2055 	if (!(mask && current->mempolicy))
2056 		return false;
2057 
2058 	task_lock(current);
2059 	mempolicy = current->mempolicy;
2060 	switch (mempolicy->mode) {
2061 	case MPOL_PREFERRED:
2062 	case MPOL_PREFERRED_MANY:
2063 	case MPOL_BIND:
2064 	case MPOL_INTERLEAVE:
2065 		*mask = mempolicy->nodes;
2066 		break;
2067 
2068 	case MPOL_LOCAL:
2069 		init_nodemask_of_node(mask, numa_node_id());
2070 		break;
2071 
2072 	default:
2073 		BUG();
2074 	}
2075 	task_unlock(current);
2076 
2077 	return true;
2078 }
2079 #endif
2080 
2081 /*
2082  * mempolicy_in_oom_domain
2083  *
2084  * If tsk's mempolicy is "bind", check for intersection between mask and
2085  * the policy nodemask. Otherwise, return true for all other policies
2086  * including "interleave", as a tsk with "interleave" policy may have
2087  * memory allocated from all nodes in system.
2088  *
2089  * Takes task_lock(tsk) to prevent freeing of its mempolicy.
2090  */
2091 bool mempolicy_in_oom_domain(struct task_struct *tsk,
2092 					const nodemask_t *mask)
2093 {
2094 	struct mempolicy *mempolicy;
2095 	bool ret = true;
2096 
2097 	if (!mask)
2098 		return ret;
2099 
2100 	task_lock(tsk);
2101 	mempolicy = tsk->mempolicy;
2102 	if (mempolicy && mempolicy->mode == MPOL_BIND)
2103 		ret = nodes_intersects(mempolicy->nodes, *mask);
2104 	task_unlock(tsk);
2105 
2106 	return ret;
2107 }
2108 
2109 /* Allocate a page in interleaved policy.
2110    Own path because it needs to do special accounting. */
2111 static struct page *alloc_page_interleave(gfp_t gfp, unsigned order,
2112 					unsigned nid)
2113 {
2114 	struct page *page;
2115 
2116 	page = __alloc_pages(gfp, order, nid, NULL);
2117 	/* skip NUMA_INTERLEAVE_HIT counter update if numa stats is disabled */
2118 	if (!static_branch_likely(&vm_numa_stat_key))
2119 		return page;
2120 	if (page && page_to_nid(page) == nid) {
2121 		preempt_disable();
2122 		__count_numa_event(page_zone(page), NUMA_INTERLEAVE_HIT);
2123 		preempt_enable();
2124 	}
2125 	return page;
2126 }
2127 
2128 static struct page *alloc_pages_preferred_many(gfp_t gfp, unsigned int order,
2129 						int nid, struct mempolicy *pol)
2130 {
2131 	struct page *page;
2132 	gfp_t preferred_gfp;
2133 
2134 	/*
2135 	 * This is a two pass approach. The first pass will only try the
2136 	 * preferred nodes but skip the direct reclaim and allow the
2137 	 * allocation to fail, while the second pass will try all the
2138 	 * nodes in system.
2139 	 */
2140 	preferred_gfp = gfp | __GFP_NOWARN;
2141 	preferred_gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL);
2142 	page = __alloc_pages(preferred_gfp, order, nid, &pol->nodes);
2143 	if (!page)
2144 		page = __alloc_pages(gfp, order, nid, NULL);
2145 
2146 	return page;
2147 }
2148 
2149 /**
2150  * alloc_pages_vma - Allocate a page for a VMA.
2151  * @gfp: GFP flags.
2152  * @order: Order of the GFP allocation.
2153  * @vma: Pointer to VMA or NULL if not available.
2154  * @addr: Virtual address of the allocation.  Must be inside @vma.
2155  * @hugepage: For hugepages try only the preferred node if possible.
2156  *
2157  * Allocate a page for a specific address in @vma, using the appropriate
2158  * NUMA policy.  When @vma is not NULL the caller must hold the mmap_lock
2159  * of the mm_struct of the VMA to prevent it from going away.  Should be
2160  * used for all allocations for pages that will be mapped into user space.
2161  *
2162  * Return: The page on success or NULL if allocation fails.
2163  */
2164 struct page *alloc_pages_vma(gfp_t gfp, int order, struct vm_area_struct *vma,
2165 		unsigned long addr, bool hugepage)
2166 {
2167 	struct mempolicy *pol;
2168 	int node = numa_node_id();
2169 	struct page *page;
2170 	int preferred_nid;
2171 	nodemask_t *nmask;
2172 
2173 	pol = get_vma_policy(vma, addr);
2174 
2175 	if (pol->mode == MPOL_INTERLEAVE) {
2176 		unsigned nid;
2177 
2178 		nid = interleave_nid(pol, vma, addr, PAGE_SHIFT + order);
2179 		mpol_cond_put(pol);
2180 		page = alloc_page_interleave(gfp, order, nid);
2181 		goto out;
2182 	}
2183 
2184 	if (pol->mode == MPOL_PREFERRED_MANY) {
2185 		node = policy_node(gfp, pol, node);
2186 		page = alloc_pages_preferred_many(gfp, order, node, pol);
2187 		mpol_cond_put(pol);
2188 		goto out;
2189 	}
2190 
2191 	if (unlikely(IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && hugepage)) {
2192 		int hpage_node = node;
2193 
2194 		/*
2195 		 * For hugepage allocation and non-interleave policy which
2196 		 * allows the current node (or other explicitly preferred
2197 		 * node) we only try to allocate from the current/preferred
2198 		 * node and don't fall back to other nodes, as the cost of
2199 		 * remote accesses would likely offset THP benefits.
2200 		 *
2201 		 * If the policy is interleave or does not allow the current
2202 		 * node in its nodemask, we allocate the standard way.
2203 		 */
2204 		if (pol->mode == MPOL_PREFERRED)
2205 			hpage_node = first_node(pol->nodes);
2206 
2207 		nmask = policy_nodemask(gfp, pol);
2208 		if (!nmask || node_isset(hpage_node, *nmask)) {
2209 			mpol_cond_put(pol);
2210 			/*
2211 			 * First, try to allocate THP only on local node, but
2212 			 * don't reclaim unnecessarily, just compact.
2213 			 */
2214 			page = __alloc_pages_node(hpage_node,
2215 				gfp | __GFP_THISNODE | __GFP_NORETRY, order);
2216 
2217 			/*
2218 			 * If hugepage allocations are configured to always
2219 			 * synchronous compact or the vma has been madvised
2220 			 * to prefer hugepage backing, retry allowing remote
2221 			 * memory with both reclaim and compact as well.
2222 			 */
2223 			if (!page && (gfp & __GFP_DIRECT_RECLAIM))
2224 				page = __alloc_pages(gfp, order, hpage_node, nmask);
2225 
2226 			goto out;
2227 		}
2228 	}
2229 
2230 	nmask = policy_nodemask(gfp, pol);
2231 	preferred_nid = policy_node(gfp, pol, node);
2232 	page = __alloc_pages(gfp, order, preferred_nid, nmask);
2233 	mpol_cond_put(pol);
2234 out:
2235 	return page;
2236 }
2237 EXPORT_SYMBOL(alloc_pages_vma);
2238 
2239 /**
2240  * alloc_pages - Allocate pages.
2241  * @gfp: GFP flags.
2242  * @order: Power of two of number of pages to allocate.
2243  *
2244  * Allocate 1 << @order contiguous pages.  The physical address of the
2245  * first page is naturally aligned (eg an order-3 allocation will be aligned
2246  * to a multiple of 8 * PAGE_SIZE bytes).  The NUMA policy of the current
2247  * process is honoured when in process context.
2248  *
2249  * Context: Can be called from any context, providing the appropriate GFP
2250  * flags are used.
2251  * Return: The page on success or NULL if allocation fails.
2252  */
2253 struct page *alloc_pages(gfp_t gfp, unsigned order)
2254 {
2255 	struct mempolicy *pol = &default_policy;
2256 	struct page *page;
2257 
2258 	if (!in_interrupt() && !(gfp & __GFP_THISNODE))
2259 		pol = get_task_policy(current);
2260 
2261 	/*
2262 	 * No reference counting needed for current->mempolicy
2263 	 * nor system default_policy
2264 	 */
2265 	if (pol->mode == MPOL_INTERLEAVE)
2266 		page = alloc_page_interleave(gfp, order, interleave_nodes(pol));
2267 	else if (pol->mode == MPOL_PREFERRED_MANY)
2268 		page = alloc_pages_preferred_many(gfp, order,
2269 				  policy_node(gfp, pol, numa_node_id()), pol);
2270 	else
2271 		page = __alloc_pages(gfp, order,
2272 				policy_node(gfp, pol, numa_node_id()),
2273 				policy_nodemask(gfp, pol));
2274 
2275 	return page;
2276 }
2277 EXPORT_SYMBOL(alloc_pages);
2278 
2279 struct folio *folio_alloc(gfp_t gfp, unsigned order)
2280 {
2281 	struct page *page = alloc_pages(gfp | __GFP_COMP, order);
2282 
2283 	if (page && order > 1)
2284 		prep_transhuge_page(page);
2285 	return (struct folio *)page;
2286 }
2287 EXPORT_SYMBOL(folio_alloc);
2288 
2289 static unsigned long alloc_pages_bulk_array_interleave(gfp_t gfp,
2290 		struct mempolicy *pol, unsigned long nr_pages,
2291 		struct page **page_array)
2292 {
2293 	int nodes;
2294 	unsigned long nr_pages_per_node;
2295 	int delta;
2296 	int i;
2297 	unsigned long nr_allocated;
2298 	unsigned long total_allocated = 0;
2299 
2300 	nodes = nodes_weight(pol->nodes);
2301 	nr_pages_per_node = nr_pages / nodes;
2302 	delta = nr_pages - nodes * nr_pages_per_node;
2303 
2304 	for (i = 0; i < nodes; i++) {
2305 		if (delta) {
2306 			nr_allocated = __alloc_pages_bulk(gfp,
2307 					interleave_nodes(pol), NULL,
2308 					nr_pages_per_node + 1, NULL,
2309 					page_array);
2310 			delta--;
2311 		} else {
2312 			nr_allocated = __alloc_pages_bulk(gfp,
2313 					interleave_nodes(pol), NULL,
2314 					nr_pages_per_node, NULL, page_array);
2315 		}
2316 
2317 		page_array += nr_allocated;
2318 		total_allocated += nr_allocated;
2319 	}
2320 
2321 	return total_allocated;
2322 }
2323 
2324 static unsigned long alloc_pages_bulk_array_preferred_many(gfp_t gfp, int nid,
2325 		struct mempolicy *pol, unsigned long nr_pages,
2326 		struct page **page_array)
2327 {
2328 	gfp_t preferred_gfp;
2329 	unsigned long nr_allocated = 0;
2330 
2331 	preferred_gfp = gfp | __GFP_NOWARN;
2332 	preferred_gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL);
2333 
2334 	nr_allocated  = __alloc_pages_bulk(preferred_gfp, nid, &pol->nodes,
2335 					   nr_pages, NULL, page_array);
2336 
2337 	if (nr_allocated < nr_pages)
2338 		nr_allocated += __alloc_pages_bulk(gfp, numa_node_id(), NULL,
2339 				nr_pages - nr_allocated, NULL,
2340 				page_array + nr_allocated);
2341 	return nr_allocated;
2342 }
2343 
2344 /* alloc pages bulk and mempolicy should be considered at the
2345  * same time in some situation such as vmalloc.
2346  *
2347  * It can accelerate memory allocation especially interleaving
2348  * allocate memory.
2349  */
2350 unsigned long alloc_pages_bulk_array_mempolicy(gfp_t gfp,
2351 		unsigned long nr_pages, struct page **page_array)
2352 {
2353 	struct mempolicy *pol = &default_policy;
2354 
2355 	if (!in_interrupt() && !(gfp & __GFP_THISNODE))
2356 		pol = get_task_policy(current);
2357 
2358 	if (pol->mode == MPOL_INTERLEAVE)
2359 		return alloc_pages_bulk_array_interleave(gfp, pol,
2360 							 nr_pages, page_array);
2361 
2362 	if (pol->mode == MPOL_PREFERRED_MANY)
2363 		return alloc_pages_bulk_array_preferred_many(gfp,
2364 				numa_node_id(), pol, nr_pages, page_array);
2365 
2366 	return __alloc_pages_bulk(gfp, policy_node(gfp, pol, numa_node_id()),
2367 				  policy_nodemask(gfp, pol), nr_pages, NULL,
2368 				  page_array);
2369 }
2370 
2371 int vma_dup_policy(struct vm_area_struct *src, struct vm_area_struct *dst)
2372 {
2373 	struct mempolicy *pol = mpol_dup(vma_policy(src));
2374 
2375 	if (IS_ERR(pol))
2376 		return PTR_ERR(pol);
2377 	dst->vm_policy = pol;
2378 	return 0;
2379 }
2380 
2381 /*
2382  * If mpol_dup() sees current->cpuset == cpuset_being_rebound, then it
2383  * rebinds the mempolicy its copying by calling mpol_rebind_policy()
2384  * with the mems_allowed returned by cpuset_mems_allowed().  This
2385  * keeps mempolicies cpuset relative after its cpuset moves.  See
2386  * further kernel/cpuset.c update_nodemask().
2387  *
2388  * current's mempolicy may be rebinded by the other task(the task that changes
2389  * cpuset's mems), so we needn't do rebind work for current task.
2390  */
2391 
2392 /* Slow path of a mempolicy duplicate */
2393 struct mempolicy *__mpol_dup(struct mempolicy *old)
2394 {
2395 	struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
2396 
2397 	if (!new)
2398 		return ERR_PTR(-ENOMEM);
2399 
2400 	/* task's mempolicy is protected by alloc_lock */
2401 	if (old == current->mempolicy) {
2402 		task_lock(current);
2403 		*new = *old;
2404 		task_unlock(current);
2405 	} else
2406 		*new = *old;
2407 
2408 	if (current_cpuset_is_being_rebound()) {
2409 		nodemask_t mems = cpuset_mems_allowed(current);
2410 		mpol_rebind_policy(new, &mems);
2411 	}
2412 	atomic_set(&new->refcnt, 1);
2413 	return new;
2414 }
2415 
2416 /* Slow path of a mempolicy comparison */
2417 bool __mpol_equal(struct mempolicy *a, struct mempolicy *b)
2418 {
2419 	if (!a || !b)
2420 		return false;
2421 	if (a->mode != b->mode)
2422 		return false;
2423 	if (a->flags != b->flags)
2424 		return false;
2425 	if (a->home_node != b->home_node)
2426 		return false;
2427 	if (mpol_store_user_nodemask(a))
2428 		if (!nodes_equal(a->w.user_nodemask, b->w.user_nodemask))
2429 			return false;
2430 
2431 	switch (a->mode) {
2432 	case MPOL_BIND:
2433 	case MPOL_INTERLEAVE:
2434 	case MPOL_PREFERRED:
2435 	case MPOL_PREFERRED_MANY:
2436 		return !!nodes_equal(a->nodes, b->nodes);
2437 	case MPOL_LOCAL:
2438 		return true;
2439 	default:
2440 		BUG();
2441 		return false;
2442 	}
2443 }
2444 
2445 /*
2446  * Shared memory backing store policy support.
2447  *
2448  * Remember policies even when nobody has shared memory mapped.
2449  * The policies are kept in Red-Black tree linked from the inode.
2450  * They are protected by the sp->lock rwlock, which should be held
2451  * for any accesses to the tree.
2452  */
2453 
2454 /*
2455  * lookup first element intersecting start-end.  Caller holds sp->lock for
2456  * reading or for writing
2457  */
2458 static struct sp_node *
2459 sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end)
2460 {
2461 	struct rb_node *n = sp->root.rb_node;
2462 
2463 	while (n) {
2464 		struct sp_node *p = rb_entry(n, struct sp_node, nd);
2465 
2466 		if (start >= p->end)
2467 			n = n->rb_right;
2468 		else if (end <= p->start)
2469 			n = n->rb_left;
2470 		else
2471 			break;
2472 	}
2473 	if (!n)
2474 		return NULL;
2475 	for (;;) {
2476 		struct sp_node *w = NULL;
2477 		struct rb_node *prev = rb_prev(n);
2478 		if (!prev)
2479 			break;
2480 		w = rb_entry(prev, struct sp_node, nd);
2481 		if (w->end <= start)
2482 			break;
2483 		n = prev;
2484 	}
2485 	return rb_entry(n, struct sp_node, nd);
2486 }
2487 
2488 /*
2489  * Insert a new shared policy into the list.  Caller holds sp->lock for
2490  * writing.
2491  */
2492 static void sp_insert(struct shared_policy *sp, struct sp_node *new)
2493 {
2494 	struct rb_node **p = &sp->root.rb_node;
2495 	struct rb_node *parent = NULL;
2496 	struct sp_node *nd;
2497 
2498 	while (*p) {
2499 		parent = *p;
2500 		nd = rb_entry(parent, struct sp_node, nd);
2501 		if (new->start < nd->start)
2502 			p = &(*p)->rb_left;
2503 		else if (new->end > nd->end)
2504 			p = &(*p)->rb_right;
2505 		else
2506 			BUG();
2507 	}
2508 	rb_link_node(&new->nd, parent, p);
2509 	rb_insert_color(&new->nd, &sp->root);
2510 	pr_debug("inserting %lx-%lx: %d\n", new->start, new->end,
2511 		 new->policy ? new->policy->mode : 0);
2512 }
2513 
2514 /* Find shared policy intersecting idx */
2515 struct mempolicy *
2516 mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx)
2517 {
2518 	struct mempolicy *pol = NULL;
2519 	struct sp_node *sn;
2520 
2521 	if (!sp->root.rb_node)
2522 		return NULL;
2523 	read_lock(&sp->lock);
2524 	sn = sp_lookup(sp, idx, idx+1);
2525 	if (sn) {
2526 		mpol_get(sn->policy);
2527 		pol = sn->policy;
2528 	}
2529 	read_unlock(&sp->lock);
2530 	return pol;
2531 }
2532 
2533 static void sp_free(struct sp_node *n)
2534 {
2535 	mpol_put(n->policy);
2536 	kmem_cache_free(sn_cache, n);
2537 }
2538 
2539 /**
2540  * mpol_misplaced - check whether current page node is valid in policy
2541  *
2542  * @page: page to be checked
2543  * @vma: vm area where page mapped
2544  * @addr: virtual address where page mapped
2545  *
2546  * Lookup current policy node id for vma,addr and "compare to" page's
2547  * node id.  Policy determination "mimics" alloc_page_vma().
2548  * Called from fault path where we know the vma and faulting address.
2549  *
2550  * Return: NUMA_NO_NODE if the page is in a node that is valid for this
2551  * policy, or a suitable node ID to allocate a replacement page from.
2552  */
2553 int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long addr)
2554 {
2555 	struct mempolicy *pol;
2556 	struct zoneref *z;
2557 	int curnid = page_to_nid(page);
2558 	unsigned long pgoff;
2559 	int thiscpu = raw_smp_processor_id();
2560 	int thisnid = cpu_to_node(thiscpu);
2561 	int polnid = NUMA_NO_NODE;
2562 	int ret = NUMA_NO_NODE;
2563 
2564 	pol = get_vma_policy(vma, addr);
2565 	if (!(pol->flags & MPOL_F_MOF))
2566 		goto out;
2567 
2568 	switch (pol->mode) {
2569 	case MPOL_INTERLEAVE:
2570 		pgoff = vma->vm_pgoff;
2571 		pgoff += (addr - vma->vm_start) >> PAGE_SHIFT;
2572 		polnid = offset_il_node(pol, pgoff);
2573 		break;
2574 
2575 	case MPOL_PREFERRED:
2576 		if (node_isset(curnid, pol->nodes))
2577 			goto out;
2578 		polnid = first_node(pol->nodes);
2579 		break;
2580 
2581 	case MPOL_LOCAL:
2582 		polnid = numa_node_id();
2583 		break;
2584 
2585 	case MPOL_BIND:
2586 		/* Optimize placement among multiple nodes via NUMA balancing */
2587 		if (pol->flags & MPOL_F_MORON) {
2588 			if (node_isset(thisnid, pol->nodes))
2589 				break;
2590 			goto out;
2591 		}
2592 		fallthrough;
2593 
2594 	case MPOL_PREFERRED_MANY:
2595 		/*
2596 		 * use current page if in policy nodemask,
2597 		 * else select nearest allowed node, if any.
2598 		 * If no allowed nodes, use current [!misplaced].
2599 		 */
2600 		if (node_isset(curnid, pol->nodes))
2601 			goto out;
2602 		z = first_zones_zonelist(
2603 				node_zonelist(numa_node_id(), GFP_HIGHUSER),
2604 				gfp_zone(GFP_HIGHUSER),
2605 				&pol->nodes);
2606 		polnid = zone_to_nid(z->zone);
2607 		break;
2608 
2609 	default:
2610 		BUG();
2611 	}
2612 
2613 	/* Migrate the page towards the node whose CPU is referencing it */
2614 	if (pol->flags & MPOL_F_MORON) {
2615 		polnid = thisnid;
2616 
2617 		if (!should_numa_migrate_memory(current, page, curnid, thiscpu))
2618 			goto out;
2619 	}
2620 
2621 	if (curnid != polnid)
2622 		ret = polnid;
2623 out:
2624 	mpol_cond_put(pol);
2625 
2626 	return ret;
2627 }
2628 
2629 /*
2630  * Drop the (possibly final) reference to task->mempolicy.  It needs to be
2631  * dropped after task->mempolicy is set to NULL so that any allocation done as
2632  * part of its kmem_cache_free(), such as by KASAN, doesn't reference a freed
2633  * policy.
2634  */
2635 void mpol_put_task_policy(struct task_struct *task)
2636 {
2637 	struct mempolicy *pol;
2638 
2639 	task_lock(task);
2640 	pol = task->mempolicy;
2641 	task->mempolicy = NULL;
2642 	task_unlock(task);
2643 	mpol_put(pol);
2644 }
2645 
2646 static void sp_delete(struct shared_policy *sp, struct sp_node *n)
2647 {
2648 	pr_debug("deleting %lx-l%lx\n", n->start, n->end);
2649 	rb_erase(&n->nd, &sp->root);
2650 	sp_free(n);
2651 }
2652 
2653 static void sp_node_init(struct sp_node *node, unsigned long start,
2654 			unsigned long end, struct mempolicy *pol)
2655 {
2656 	node->start = start;
2657 	node->end = end;
2658 	node->policy = pol;
2659 }
2660 
2661 static struct sp_node *sp_alloc(unsigned long start, unsigned long end,
2662 				struct mempolicy *pol)
2663 {
2664 	struct sp_node *n;
2665 	struct mempolicy *newpol;
2666 
2667 	n = kmem_cache_alloc(sn_cache, GFP_KERNEL);
2668 	if (!n)
2669 		return NULL;
2670 
2671 	newpol = mpol_dup(pol);
2672 	if (IS_ERR(newpol)) {
2673 		kmem_cache_free(sn_cache, n);
2674 		return NULL;
2675 	}
2676 	newpol->flags |= MPOL_F_SHARED;
2677 	sp_node_init(n, start, end, newpol);
2678 
2679 	return n;
2680 }
2681 
2682 /* Replace a policy range. */
2683 static int shared_policy_replace(struct shared_policy *sp, unsigned long start,
2684 				 unsigned long end, struct sp_node *new)
2685 {
2686 	struct sp_node *n;
2687 	struct sp_node *n_new = NULL;
2688 	struct mempolicy *mpol_new = NULL;
2689 	int ret = 0;
2690 
2691 restart:
2692 	write_lock(&sp->lock);
2693 	n = sp_lookup(sp, start, end);
2694 	/* Take care of old policies in the same range. */
2695 	while (n && n->start < end) {
2696 		struct rb_node *next = rb_next(&n->nd);
2697 		if (n->start >= start) {
2698 			if (n->end <= end)
2699 				sp_delete(sp, n);
2700 			else
2701 				n->start = end;
2702 		} else {
2703 			/* Old policy spanning whole new range. */
2704 			if (n->end > end) {
2705 				if (!n_new)
2706 					goto alloc_new;
2707 
2708 				*mpol_new = *n->policy;
2709 				atomic_set(&mpol_new->refcnt, 1);
2710 				sp_node_init(n_new, end, n->end, mpol_new);
2711 				n->end = start;
2712 				sp_insert(sp, n_new);
2713 				n_new = NULL;
2714 				mpol_new = NULL;
2715 				break;
2716 			} else
2717 				n->end = start;
2718 		}
2719 		if (!next)
2720 			break;
2721 		n = rb_entry(next, struct sp_node, nd);
2722 	}
2723 	if (new)
2724 		sp_insert(sp, new);
2725 	write_unlock(&sp->lock);
2726 	ret = 0;
2727 
2728 err_out:
2729 	if (mpol_new)
2730 		mpol_put(mpol_new);
2731 	if (n_new)
2732 		kmem_cache_free(sn_cache, n_new);
2733 
2734 	return ret;
2735 
2736 alloc_new:
2737 	write_unlock(&sp->lock);
2738 	ret = -ENOMEM;
2739 	n_new = kmem_cache_alloc(sn_cache, GFP_KERNEL);
2740 	if (!n_new)
2741 		goto err_out;
2742 	mpol_new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
2743 	if (!mpol_new)
2744 		goto err_out;
2745 	goto restart;
2746 }
2747 
2748 /**
2749  * mpol_shared_policy_init - initialize shared policy for inode
2750  * @sp: pointer to inode shared policy
2751  * @mpol:  struct mempolicy to install
2752  *
2753  * Install non-NULL @mpol in inode's shared policy rb-tree.
2754  * On entry, the current task has a reference on a non-NULL @mpol.
2755  * This must be released on exit.
2756  * This is called at get_inode() calls and we can use GFP_KERNEL.
2757  */
2758 void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol)
2759 {
2760 	int ret;
2761 
2762 	sp->root = RB_ROOT;		/* empty tree == default mempolicy */
2763 	rwlock_init(&sp->lock);
2764 
2765 	if (mpol) {
2766 		struct vm_area_struct pvma;
2767 		struct mempolicy *new;
2768 		NODEMASK_SCRATCH(scratch);
2769 
2770 		if (!scratch)
2771 			goto put_mpol;
2772 		/* contextualize the tmpfs mount point mempolicy */
2773 		new = mpol_new(mpol->mode, mpol->flags, &mpol->w.user_nodemask);
2774 		if (IS_ERR(new))
2775 			goto free_scratch; /* no valid nodemask intersection */
2776 
2777 		task_lock(current);
2778 		ret = mpol_set_nodemask(new, &mpol->w.user_nodemask, scratch);
2779 		task_unlock(current);
2780 		if (ret)
2781 			goto put_new;
2782 
2783 		/* Create pseudo-vma that contains just the policy */
2784 		vma_init(&pvma, NULL);
2785 		pvma.vm_end = TASK_SIZE;	/* policy covers entire file */
2786 		mpol_set_shared_policy(sp, &pvma, new); /* adds ref */
2787 
2788 put_new:
2789 		mpol_put(new);			/* drop initial ref */
2790 free_scratch:
2791 		NODEMASK_SCRATCH_FREE(scratch);
2792 put_mpol:
2793 		mpol_put(mpol);	/* drop our incoming ref on sb mpol */
2794 	}
2795 }
2796 
2797 int mpol_set_shared_policy(struct shared_policy *info,
2798 			struct vm_area_struct *vma, struct mempolicy *npol)
2799 {
2800 	int err;
2801 	struct sp_node *new = NULL;
2802 	unsigned long sz = vma_pages(vma);
2803 
2804 	pr_debug("set_shared_policy %lx sz %lu %d %d %lx\n",
2805 		 vma->vm_pgoff,
2806 		 sz, npol ? npol->mode : -1,
2807 		 npol ? npol->flags : -1,
2808 		 npol ? nodes_addr(npol->nodes)[0] : NUMA_NO_NODE);
2809 
2810 	if (npol) {
2811 		new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, npol);
2812 		if (!new)
2813 			return -ENOMEM;
2814 	}
2815 	err = shared_policy_replace(info, vma->vm_pgoff, vma->vm_pgoff+sz, new);
2816 	if (err && new)
2817 		sp_free(new);
2818 	return err;
2819 }
2820 
2821 /* Free a backing policy store on inode delete. */
2822 void mpol_free_shared_policy(struct shared_policy *p)
2823 {
2824 	struct sp_node *n;
2825 	struct rb_node *next;
2826 
2827 	if (!p->root.rb_node)
2828 		return;
2829 	write_lock(&p->lock);
2830 	next = rb_first(&p->root);
2831 	while (next) {
2832 		n = rb_entry(next, struct sp_node, nd);
2833 		next = rb_next(&n->nd);
2834 		sp_delete(p, n);
2835 	}
2836 	write_unlock(&p->lock);
2837 }
2838 
2839 #ifdef CONFIG_NUMA_BALANCING
2840 static int __initdata numabalancing_override;
2841 
2842 static void __init check_numabalancing_enable(void)
2843 {
2844 	bool numabalancing_default = false;
2845 
2846 	if (IS_ENABLED(CONFIG_NUMA_BALANCING_DEFAULT_ENABLED))
2847 		numabalancing_default = true;
2848 
2849 	/* Parsed by setup_numabalancing. override == 1 enables, -1 disables */
2850 	if (numabalancing_override)
2851 		set_numabalancing_state(numabalancing_override == 1);
2852 
2853 	if (num_online_nodes() > 1 && !numabalancing_override) {
2854 		pr_info("%s automatic NUMA balancing. Configure with numa_balancing= or the kernel.numa_balancing sysctl\n",
2855 			numabalancing_default ? "Enabling" : "Disabling");
2856 		set_numabalancing_state(numabalancing_default);
2857 	}
2858 }
2859 
2860 static int __init setup_numabalancing(char *str)
2861 {
2862 	int ret = 0;
2863 	if (!str)
2864 		goto out;
2865 
2866 	if (!strcmp(str, "enable")) {
2867 		numabalancing_override = 1;
2868 		ret = 1;
2869 	} else if (!strcmp(str, "disable")) {
2870 		numabalancing_override = -1;
2871 		ret = 1;
2872 	}
2873 out:
2874 	if (!ret)
2875 		pr_warn("Unable to parse numa_balancing=\n");
2876 
2877 	return ret;
2878 }
2879 __setup("numa_balancing=", setup_numabalancing);
2880 #else
2881 static inline void __init check_numabalancing_enable(void)
2882 {
2883 }
2884 #endif /* CONFIG_NUMA_BALANCING */
2885 
2886 /* assumes fs == KERNEL_DS */
2887 void __init numa_policy_init(void)
2888 {
2889 	nodemask_t interleave_nodes;
2890 	unsigned long largest = 0;
2891 	int nid, prefer = 0;
2892 
2893 	policy_cache = kmem_cache_create("numa_policy",
2894 					 sizeof(struct mempolicy),
2895 					 0, SLAB_PANIC, NULL);
2896 
2897 	sn_cache = kmem_cache_create("shared_policy_node",
2898 				     sizeof(struct sp_node),
2899 				     0, SLAB_PANIC, NULL);
2900 
2901 	for_each_node(nid) {
2902 		preferred_node_policy[nid] = (struct mempolicy) {
2903 			.refcnt = ATOMIC_INIT(1),
2904 			.mode = MPOL_PREFERRED,
2905 			.flags = MPOL_F_MOF | MPOL_F_MORON,
2906 			.nodes = nodemask_of_node(nid),
2907 		};
2908 	}
2909 
2910 	/*
2911 	 * Set interleaving policy for system init. Interleaving is only
2912 	 * enabled across suitably sized nodes (default is >= 16MB), or
2913 	 * fall back to the largest node if they're all smaller.
2914 	 */
2915 	nodes_clear(interleave_nodes);
2916 	for_each_node_state(nid, N_MEMORY) {
2917 		unsigned long total_pages = node_present_pages(nid);
2918 
2919 		/* Preserve the largest node */
2920 		if (largest < total_pages) {
2921 			largest = total_pages;
2922 			prefer = nid;
2923 		}
2924 
2925 		/* Interleave this node? */
2926 		if ((total_pages << PAGE_SHIFT) >= (16 << 20))
2927 			node_set(nid, interleave_nodes);
2928 	}
2929 
2930 	/* All too small, use the largest */
2931 	if (unlikely(nodes_empty(interleave_nodes)))
2932 		node_set(prefer, interleave_nodes);
2933 
2934 	if (do_set_mempolicy(MPOL_INTERLEAVE, 0, &interleave_nodes))
2935 		pr_err("%s: interleaving failed\n", __func__);
2936 
2937 	check_numabalancing_enable();
2938 }
2939 
2940 /* Reset policy of current process to default */
2941 void numa_default_policy(void)
2942 {
2943 	do_set_mempolicy(MPOL_DEFAULT, 0, NULL);
2944 }
2945 
2946 /*
2947  * Parse and format mempolicy from/to strings
2948  */
2949 
2950 static const char * const policy_modes[] =
2951 {
2952 	[MPOL_DEFAULT]    = "default",
2953 	[MPOL_PREFERRED]  = "prefer",
2954 	[MPOL_BIND]       = "bind",
2955 	[MPOL_INTERLEAVE] = "interleave",
2956 	[MPOL_LOCAL]      = "local",
2957 	[MPOL_PREFERRED_MANY]  = "prefer (many)",
2958 };
2959 
2960 
2961 #ifdef CONFIG_TMPFS
2962 /**
2963  * mpol_parse_str - parse string to mempolicy, for tmpfs mpol mount option.
2964  * @str:  string containing mempolicy to parse
2965  * @mpol:  pointer to struct mempolicy pointer, returned on success.
2966  *
2967  * Format of input:
2968  *	<mode>[=<flags>][:<nodelist>]
2969  *
2970  * Return: %0 on success, else %1
2971  */
2972 int mpol_parse_str(char *str, struct mempolicy **mpol)
2973 {
2974 	struct mempolicy *new = NULL;
2975 	unsigned short mode_flags;
2976 	nodemask_t nodes;
2977 	char *nodelist = strchr(str, ':');
2978 	char *flags = strchr(str, '=');
2979 	int err = 1, mode;
2980 
2981 	if (flags)
2982 		*flags++ = '\0';	/* terminate mode string */
2983 
2984 	if (nodelist) {
2985 		/* NUL-terminate mode or flags string */
2986 		*nodelist++ = '\0';
2987 		if (nodelist_parse(nodelist, nodes))
2988 			goto out;
2989 		if (!nodes_subset(nodes, node_states[N_MEMORY]))
2990 			goto out;
2991 	} else
2992 		nodes_clear(nodes);
2993 
2994 	mode = match_string(policy_modes, MPOL_MAX, str);
2995 	if (mode < 0)
2996 		goto out;
2997 
2998 	switch (mode) {
2999 	case MPOL_PREFERRED:
3000 		/*
3001 		 * Insist on a nodelist of one node only, although later
3002 		 * we use first_node(nodes) to grab a single node, so here
3003 		 * nodelist (or nodes) cannot be empty.
3004 		 */
3005 		if (nodelist) {
3006 			char *rest = nodelist;
3007 			while (isdigit(*rest))
3008 				rest++;
3009 			if (*rest)
3010 				goto out;
3011 			if (nodes_empty(nodes))
3012 				goto out;
3013 		}
3014 		break;
3015 	case MPOL_INTERLEAVE:
3016 		/*
3017 		 * Default to online nodes with memory if no nodelist
3018 		 */
3019 		if (!nodelist)
3020 			nodes = node_states[N_MEMORY];
3021 		break;
3022 	case MPOL_LOCAL:
3023 		/*
3024 		 * Don't allow a nodelist;  mpol_new() checks flags
3025 		 */
3026 		if (nodelist)
3027 			goto out;
3028 		break;
3029 	case MPOL_DEFAULT:
3030 		/*
3031 		 * Insist on a empty nodelist
3032 		 */
3033 		if (!nodelist)
3034 			err = 0;
3035 		goto out;
3036 	case MPOL_PREFERRED_MANY:
3037 	case MPOL_BIND:
3038 		/*
3039 		 * Insist on a nodelist
3040 		 */
3041 		if (!nodelist)
3042 			goto out;
3043 	}
3044 
3045 	mode_flags = 0;
3046 	if (flags) {
3047 		/*
3048 		 * Currently, we only support two mutually exclusive
3049 		 * mode flags.
3050 		 */
3051 		if (!strcmp(flags, "static"))
3052 			mode_flags |= MPOL_F_STATIC_NODES;
3053 		else if (!strcmp(flags, "relative"))
3054 			mode_flags |= MPOL_F_RELATIVE_NODES;
3055 		else
3056 			goto out;
3057 	}
3058 
3059 	new = mpol_new(mode, mode_flags, &nodes);
3060 	if (IS_ERR(new))
3061 		goto out;
3062 
3063 	/*
3064 	 * Save nodes for mpol_to_str() to show the tmpfs mount options
3065 	 * for /proc/mounts, /proc/pid/mounts and /proc/pid/mountinfo.
3066 	 */
3067 	if (mode != MPOL_PREFERRED) {
3068 		new->nodes = nodes;
3069 	} else if (nodelist) {
3070 		nodes_clear(new->nodes);
3071 		node_set(first_node(nodes), new->nodes);
3072 	} else {
3073 		new->mode = MPOL_LOCAL;
3074 	}
3075 
3076 	/*
3077 	 * Save nodes for contextualization: this will be used to "clone"
3078 	 * the mempolicy in a specific context [cpuset] at a later time.
3079 	 */
3080 	new->w.user_nodemask = nodes;
3081 
3082 	err = 0;
3083 
3084 out:
3085 	/* Restore string for error message */
3086 	if (nodelist)
3087 		*--nodelist = ':';
3088 	if (flags)
3089 		*--flags = '=';
3090 	if (!err)
3091 		*mpol = new;
3092 	return err;
3093 }
3094 #endif /* CONFIG_TMPFS */
3095 
3096 /**
3097  * mpol_to_str - format a mempolicy structure for printing
3098  * @buffer:  to contain formatted mempolicy string
3099  * @maxlen:  length of @buffer
3100  * @pol:  pointer to mempolicy to be formatted
3101  *
3102  * Convert @pol into a string.  If @buffer is too short, truncate the string.
3103  * Recommend a @maxlen of at least 32 for the longest mode, "interleave", the
3104  * longest flag, "relative", and to display at least a few node ids.
3105  */
3106 void mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)
3107 {
3108 	char *p = buffer;
3109 	nodemask_t nodes = NODE_MASK_NONE;
3110 	unsigned short mode = MPOL_DEFAULT;
3111 	unsigned short flags = 0;
3112 
3113 	if (pol && pol != &default_policy && !(pol->flags & MPOL_F_MORON)) {
3114 		mode = pol->mode;
3115 		flags = pol->flags;
3116 	}
3117 
3118 	switch (mode) {
3119 	case MPOL_DEFAULT:
3120 	case MPOL_LOCAL:
3121 		break;
3122 	case MPOL_PREFERRED:
3123 	case MPOL_PREFERRED_MANY:
3124 	case MPOL_BIND:
3125 	case MPOL_INTERLEAVE:
3126 		nodes = pol->nodes;
3127 		break;
3128 	default:
3129 		WARN_ON_ONCE(1);
3130 		snprintf(p, maxlen, "unknown");
3131 		return;
3132 	}
3133 
3134 	p += snprintf(p, maxlen, "%s", policy_modes[mode]);
3135 
3136 	if (flags & MPOL_MODE_FLAGS) {
3137 		p += snprintf(p, buffer + maxlen - p, "=");
3138 
3139 		/*
3140 		 * Currently, the only defined flags are mutually exclusive
3141 		 */
3142 		if (flags & MPOL_F_STATIC_NODES)
3143 			p += snprintf(p, buffer + maxlen - p, "static");
3144 		else if (flags & MPOL_F_RELATIVE_NODES)
3145 			p += snprintf(p, buffer + maxlen - p, "relative");
3146 	}
3147 
3148 	if (!nodes_empty(nodes))
3149 		p += scnprintf(p, buffer + maxlen - p, ":%*pbl",
3150 			       nodemask_pr_args(&nodes));
3151 }
3152