xref: /openbmc/linux/mm/mempolicy.c (revision ab14e199)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Simple NUMA memory policy for the Linux kernel.
4  *
5  * Copyright 2003,2004 Andi Kleen, SuSE Labs.
6  * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
7  *
8  * NUMA policy allows the user to give hints in which node(s) memory should
9  * be allocated.
10  *
11  * Support four policies per VMA and per process:
12  *
13  * The VMA policy has priority over the process policy for a page fault.
14  *
15  * interleave     Allocate memory interleaved over a set of nodes,
16  *                with normal fallback if it fails.
17  *                For VMA based allocations this interleaves based on the
18  *                offset into the backing object or offset into the mapping
19  *                for anonymous memory. For process policy an process counter
20  *                is used.
21  *
22  * bind           Only allocate memory on a specific set of nodes,
23  *                no fallback.
24  *                FIXME: memory is allocated starting with the first node
25  *                to the last. It would be better if bind would truly restrict
26  *                the allocation to memory nodes instead
27  *
28  * preferred       Try a specific node first before normal fallback.
29  *                As a special case NUMA_NO_NODE here means do the allocation
30  *                on the local CPU. This is normally identical to default,
31  *                but useful to set in a VMA when you have a non default
32  *                process policy.
33  *
34  * preferred many Try a set of nodes first before normal fallback. This is
35  *                similar to preferred without the special case.
36  *
37  * default        Allocate on the local node first, or when on a VMA
38  *                use the process policy. This is what Linux always did
39  *		  in a NUMA aware kernel and still does by, ahem, default.
40  *
41  * The process policy is applied for most non interrupt memory allocations
42  * in that process' context. Interrupts ignore the policies and always
43  * try to allocate on the local CPU. The VMA policy is only applied for memory
44  * allocations for a VMA in the VM.
45  *
46  * Currently there are a few corner cases in swapping where the policy
47  * is not applied, but the majority should be handled. When process policy
48  * is used it is not remembered over swap outs/swap ins.
49  *
50  * Only the highest zone in the zone hierarchy gets policied. Allocations
51  * requesting a lower zone just use default policy. This implies that
52  * on systems with highmem kernel lowmem allocation don't get policied.
53  * Same with GFP_DMA allocations.
54  *
55  * For shmfs/tmpfs/hugetlbfs shared memory the policy is shared between
56  * all users and remembered even when nobody has memory mapped.
57  */
58 
59 /* Notebook:
60    fix mmap readahead to honour policy and enable policy for any page cache
61    object
62    statistics for bigpages
63    global policy for page cache? currently it uses process policy. Requires
64    first item above.
65    handle mremap for shared memory (currently ignored for the policy)
66    grows down?
67    make bind policy root only? It can trigger oom much faster and the
68    kernel is not always grateful with that.
69 */
70 
71 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
72 
73 #include <linux/mempolicy.h>
74 #include <linux/pagewalk.h>
75 #include <linux/highmem.h>
76 #include <linux/hugetlb.h>
77 #include <linux/kernel.h>
78 #include <linux/sched.h>
79 #include <linux/sched/mm.h>
80 #include <linux/sched/numa_balancing.h>
81 #include <linux/sched/task.h>
82 #include <linux/nodemask.h>
83 #include <linux/cpuset.h>
84 #include <linux/slab.h>
85 #include <linux/string.h>
86 #include <linux/export.h>
87 #include <linux/nsproxy.h>
88 #include <linux/interrupt.h>
89 #include <linux/init.h>
90 #include <linux/compat.h>
91 #include <linux/ptrace.h>
92 #include <linux/swap.h>
93 #include <linux/seq_file.h>
94 #include <linux/proc_fs.h>
95 #include <linux/migrate.h>
96 #include <linux/ksm.h>
97 #include <linux/rmap.h>
98 #include <linux/security.h>
99 #include <linux/syscalls.h>
100 #include <linux/ctype.h>
101 #include <linux/mm_inline.h>
102 #include <linux/mmu_notifier.h>
103 #include <linux/printk.h>
104 #include <linux/swapops.h>
105 
106 #include <asm/tlbflush.h>
107 #include <asm/tlb.h>
108 #include <linux/uaccess.h>
109 
110 #include "internal.h"
111 
112 /* Internal flags */
113 #define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0)	/* Skip checks for continuous vmas */
114 #define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1)		/* Invert check for nodemask */
115 
116 static struct kmem_cache *policy_cache;
117 static struct kmem_cache *sn_cache;
118 
119 /* Highest zone. An specific allocation for a zone below that is not
120    policied. */
121 enum zone_type policy_zone = 0;
122 
123 /*
124  * run-time system-wide default policy => local allocation
125  */
126 static struct mempolicy default_policy = {
127 	.refcnt = ATOMIC_INIT(1), /* never free it */
128 	.mode = MPOL_LOCAL,
129 };
130 
131 static struct mempolicy preferred_node_policy[MAX_NUMNODES];
132 
133 /**
134  * numa_nearest_node - Find nearest node by state
135  * @node: Node id to start the search
136  * @state: State to filter the search
137  *
138  * Lookup the closest node by distance if @nid is not in state.
139  *
140  * Return: this @node if it is in state, otherwise the closest node by distance
141  */
numa_nearest_node(int node,unsigned int state)142 int numa_nearest_node(int node, unsigned int state)
143 {
144 	int min_dist = INT_MAX, dist, n, min_node;
145 
146 	if (state >= NR_NODE_STATES)
147 		return -EINVAL;
148 
149 	if (node == NUMA_NO_NODE || node_state(node, state))
150 		return node;
151 
152 	min_node = node;
153 	for_each_node_state(n, state) {
154 		dist = node_distance(node, n);
155 		if (dist < min_dist) {
156 			min_dist = dist;
157 			min_node = n;
158 		}
159 	}
160 
161 	return min_node;
162 }
163 EXPORT_SYMBOL_GPL(numa_nearest_node);
164 
get_task_policy(struct task_struct * p)165 struct mempolicy *get_task_policy(struct task_struct *p)
166 {
167 	struct mempolicy *pol = p->mempolicy;
168 	int node;
169 
170 	if (pol)
171 		return pol;
172 
173 	node = numa_node_id();
174 	if (node != NUMA_NO_NODE) {
175 		pol = &preferred_node_policy[node];
176 		/* preferred_node_policy is not initialised early in boot */
177 		if (pol->mode)
178 			return pol;
179 	}
180 
181 	return &default_policy;
182 }
183 
184 static const struct mempolicy_operations {
185 	int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
186 	void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes);
187 } mpol_ops[MPOL_MAX];
188 
mpol_store_user_nodemask(const struct mempolicy * pol)189 static inline int mpol_store_user_nodemask(const struct mempolicy *pol)
190 {
191 	return pol->flags & MPOL_MODE_FLAGS;
192 }
193 
mpol_relative_nodemask(nodemask_t * ret,const nodemask_t * orig,const nodemask_t * rel)194 static void mpol_relative_nodemask(nodemask_t *ret, const nodemask_t *orig,
195 				   const nodemask_t *rel)
196 {
197 	nodemask_t tmp;
198 	nodes_fold(tmp, *orig, nodes_weight(*rel));
199 	nodes_onto(*ret, tmp, *rel);
200 }
201 
mpol_new_nodemask(struct mempolicy * pol,const nodemask_t * nodes)202 static int mpol_new_nodemask(struct mempolicy *pol, const nodemask_t *nodes)
203 {
204 	if (nodes_empty(*nodes))
205 		return -EINVAL;
206 	pol->nodes = *nodes;
207 	return 0;
208 }
209 
mpol_new_preferred(struct mempolicy * pol,const nodemask_t * nodes)210 static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes)
211 {
212 	if (nodes_empty(*nodes))
213 		return -EINVAL;
214 
215 	nodes_clear(pol->nodes);
216 	node_set(first_node(*nodes), pol->nodes);
217 	return 0;
218 }
219 
220 /*
221  * mpol_set_nodemask is called after mpol_new() to set up the nodemask, if
222  * any, for the new policy.  mpol_new() has already validated the nodes
223  * parameter with respect to the policy mode and flags.
224  *
225  * Must be called holding task's alloc_lock to protect task's mems_allowed
226  * and mempolicy.  May also be called holding the mmap_lock for write.
227  */
mpol_set_nodemask(struct mempolicy * pol,const nodemask_t * nodes,struct nodemask_scratch * nsc)228 static int mpol_set_nodemask(struct mempolicy *pol,
229 		     const nodemask_t *nodes, struct nodemask_scratch *nsc)
230 {
231 	int ret;
232 
233 	/*
234 	 * Default (pol==NULL) resp. local memory policies are not a
235 	 * subject of any remapping. They also do not need any special
236 	 * constructor.
237 	 */
238 	if (!pol || pol->mode == MPOL_LOCAL)
239 		return 0;
240 
241 	/* Check N_MEMORY */
242 	nodes_and(nsc->mask1,
243 		  cpuset_current_mems_allowed, node_states[N_MEMORY]);
244 
245 	VM_BUG_ON(!nodes);
246 
247 	if (pol->flags & MPOL_F_RELATIVE_NODES)
248 		mpol_relative_nodemask(&nsc->mask2, nodes, &nsc->mask1);
249 	else
250 		nodes_and(nsc->mask2, *nodes, nsc->mask1);
251 
252 	if (mpol_store_user_nodemask(pol))
253 		pol->w.user_nodemask = *nodes;
254 	else
255 		pol->w.cpuset_mems_allowed = cpuset_current_mems_allowed;
256 
257 	ret = mpol_ops[pol->mode].create(pol, &nsc->mask2);
258 	return ret;
259 }
260 
261 /*
262  * This function just creates a new policy, does some check and simple
263  * initialization. You must invoke mpol_set_nodemask() to set nodes.
264  */
mpol_new(unsigned short mode,unsigned short flags,nodemask_t * nodes)265 static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags,
266 				  nodemask_t *nodes)
267 {
268 	struct mempolicy *policy;
269 
270 	pr_debug("setting mode %d flags %d nodes[0] %lx\n",
271 		 mode, flags, nodes ? nodes_addr(*nodes)[0] : NUMA_NO_NODE);
272 
273 	if (mode == MPOL_DEFAULT) {
274 		if (nodes && !nodes_empty(*nodes))
275 			return ERR_PTR(-EINVAL);
276 		return NULL;
277 	}
278 	VM_BUG_ON(!nodes);
279 
280 	/*
281 	 * MPOL_PREFERRED cannot be used with MPOL_F_STATIC_NODES or
282 	 * MPOL_F_RELATIVE_NODES if the nodemask is empty (local allocation).
283 	 * All other modes require a valid pointer to a non-empty nodemask.
284 	 */
285 	if (mode == MPOL_PREFERRED) {
286 		if (nodes_empty(*nodes)) {
287 			if (((flags & MPOL_F_STATIC_NODES) ||
288 			     (flags & MPOL_F_RELATIVE_NODES)))
289 				return ERR_PTR(-EINVAL);
290 
291 			mode = MPOL_LOCAL;
292 		}
293 	} else if (mode == MPOL_LOCAL) {
294 		if (!nodes_empty(*nodes) ||
295 		    (flags & MPOL_F_STATIC_NODES) ||
296 		    (flags & MPOL_F_RELATIVE_NODES))
297 			return ERR_PTR(-EINVAL);
298 	} else if (nodes_empty(*nodes))
299 		return ERR_PTR(-EINVAL);
300 	policy = kmem_cache_alloc(policy_cache, GFP_KERNEL);
301 	if (!policy)
302 		return ERR_PTR(-ENOMEM);
303 	atomic_set(&policy->refcnt, 1);
304 	policy->mode = mode;
305 	policy->flags = flags;
306 	policy->home_node = NUMA_NO_NODE;
307 
308 	return policy;
309 }
310 
311 /* Slow path of a mpol destructor. */
__mpol_put(struct mempolicy * p)312 void __mpol_put(struct mempolicy *p)
313 {
314 	if (!atomic_dec_and_test(&p->refcnt))
315 		return;
316 	kmem_cache_free(policy_cache, p);
317 }
318 
mpol_rebind_default(struct mempolicy * pol,const nodemask_t * nodes)319 static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes)
320 {
321 }
322 
mpol_rebind_nodemask(struct mempolicy * pol,const nodemask_t * nodes)323 static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes)
324 {
325 	nodemask_t tmp;
326 
327 	if (pol->flags & MPOL_F_STATIC_NODES)
328 		nodes_and(tmp, pol->w.user_nodemask, *nodes);
329 	else if (pol->flags & MPOL_F_RELATIVE_NODES)
330 		mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
331 	else {
332 		nodes_remap(tmp, pol->nodes, pol->w.cpuset_mems_allowed,
333 								*nodes);
334 		pol->w.cpuset_mems_allowed = *nodes;
335 	}
336 
337 	if (nodes_empty(tmp))
338 		tmp = *nodes;
339 
340 	pol->nodes = tmp;
341 }
342 
mpol_rebind_preferred(struct mempolicy * pol,const nodemask_t * nodes)343 static void mpol_rebind_preferred(struct mempolicy *pol,
344 						const nodemask_t *nodes)
345 {
346 	pol->w.cpuset_mems_allowed = *nodes;
347 }
348 
349 /*
350  * mpol_rebind_policy - Migrate a policy to a different set of nodes
351  *
352  * Per-vma policies are protected by mmap_lock. Allocations using per-task
353  * policies are protected by task->mems_allowed_seq to prevent a premature
354  * OOM/allocation failure due to parallel nodemask modification.
355  */
mpol_rebind_policy(struct mempolicy * pol,const nodemask_t * newmask)356 static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask)
357 {
358 	if (!pol || pol->mode == MPOL_LOCAL)
359 		return;
360 	if (!mpol_store_user_nodemask(pol) &&
361 	    nodes_equal(pol->w.cpuset_mems_allowed, *newmask))
362 		return;
363 
364 	mpol_ops[pol->mode].rebind(pol, newmask);
365 }
366 
367 /*
368  * Wrapper for mpol_rebind_policy() that just requires task
369  * pointer, and updates task mempolicy.
370  *
371  * Called with task's alloc_lock held.
372  */
373 
mpol_rebind_task(struct task_struct * tsk,const nodemask_t * new)374 void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new)
375 {
376 	mpol_rebind_policy(tsk->mempolicy, new);
377 }
378 
379 /*
380  * Rebind each vma in mm to new nodemask.
381  *
382  * Call holding a reference to mm.  Takes mm->mmap_lock during call.
383  */
384 
mpol_rebind_mm(struct mm_struct * mm,nodemask_t * new)385 void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
386 {
387 	struct vm_area_struct *vma;
388 	VMA_ITERATOR(vmi, mm, 0);
389 
390 	mmap_write_lock(mm);
391 	for_each_vma(vmi, vma) {
392 		vma_start_write(vma);
393 		mpol_rebind_policy(vma->vm_policy, new);
394 	}
395 	mmap_write_unlock(mm);
396 }
397 
398 static const struct mempolicy_operations mpol_ops[MPOL_MAX] = {
399 	[MPOL_DEFAULT] = {
400 		.rebind = mpol_rebind_default,
401 	},
402 	[MPOL_INTERLEAVE] = {
403 		.create = mpol_new_nodemask,
404 		.rebind = mpol_rebind_nodemask,
405 	},
406 	[MPOL_PREFERRED] = {
407 		.create = mpol_new_preferred,
408 		.rebind = mpol_rebind_preferred,
409 	},
410 	[MPOL_BIND] = {
411 		.create = mpol_new_nodemask,
412 		.rebind = mpol_rebind_nodemask,
413 	},
414 	[MPOL_LOCAL] = {
415 		.rebind = mpol_rebind_default,
416 	},
417 	[MPOL_PREFERRED_MANY] = {
418 		.create = mpol_new_nodemask,
419 		.rebind = mpol_rebind_preferred,
420 	},
421 };
422 
423 static int migrate_folio_add(struct folio *folio, struct list_head *foliolist,
424 				unsigned long flags);
425 
426 struct queue_pages {
427 	struct list_head *pagelist;
428 	unsigned long flags;
429 	nodemask_t *nmask;
430 	unsigned long start;
431 	unsigned long end;
432 	struct vm_area_struct *first;
433 	bool has_unmovable;
434 };
435 
436 /*
437  * Check if the folio's nid is in qp->nmask.
438  *
439  * If MPOL_MF_INVERT is set in qp->flags, check if the nid is
440  * in the invert of qp->nmask.
441  */
queue_folio_required(struct folio * folio,struct queue_pages * qp)442 static inline bool queue_folio_required(struct folio *folio,
443 					struct queue_pages *qp)
444 {
445 	int nid = folio_nid(folio);
446 	unsigned long flags = qp->flags;
447 
448 	return node_isset(nid, *qp->nmask) == !(flags & MPOL_MF_INVERT);
449 }
450 
451 /*
452  * queue_folios_pmd() has three possible return values:
453  * 0 - folios are placed on the right node or queued successfully, or
454  *     special page is met, i.e. zero page, or unmovable page is found
455  *     but continue walking (indicated by queue_pages.has_unmovable).
456  * -EIO - is migration entry or only MPOL_MF_STRICT was specified and an
457  *        existing folio was already on a node that does not follow the
458  *        policy.
459  */
queue_folios_pmd(pmd_t * pmd,spinlock_t * ptl,unsigned long addr,unsigned long end,struct mm_walk * walk)460 static int queue_folios_pmd(pmd_t *pmd, spinlock_t *ptl, unsigned long addr,
461 				unsigned long end, struct mm_walk *walk)
462 	__releases(ptl)
463 {
464 	int ret = 0;
465 	struct folio *folio;
466 	struct queue_pages *qp = walk->private;
467 	unsigned long flags;
468 
469 	if (unlikely(is_pmd_migration_entry(*pmd))) {
470 		ret = -EIO;
471 		goto unlock;
472 	}
473 	folio = pfn_folio(pmd_pfn(*pmd));
474 	if (is_huge_zero_page(&folio->page)) {
475 		walk->action = ACTION_CONTINUE;
476 		goto unlock;
477 	}
478 	if (!queue_folio_required(folio, qp))
479 		goto unlock;
480 
481 	flags = qp->flags;
482 	/* go to folio migration */
483 	if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
484 		if (!vma_migratable(walk->vma) ||
485 		    migrate_folio_add(folio, qp->pagelist, flags)) {
486 			qp->has_unmovable = true;
487 			goto unlock;
488 		}
489 	} else
490 		ret = -EIO;
491 unlock:
492 	spin_unlock(ptl);
493 	return ret;
494 }
495 
496 /*
497  * Scan through pages checking if pages follow certain conditions,
498  * and move them to the pagelist if they do.
499  *
500  * queue_folios_pte_range() has three possible return values:
501  * 0 - folios are placed on the right node or queued successfully, or
502  *     special page is met, i.e. zero page, or unmovable page is found
503  *     but continue walking (indicated by queue_pages.has_unmovable).
504  * -EIO - only MPOL_MF_STRICT was specified and an existing folio was already
505  *        on a node that does not follow the policy.
506  */
queue_folios_pte_range(pmd_t * pmd,unsigned long addr,unsigned long end,struct mm_walk * walk)507 static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr,
508 			unsigned long end, struct mm_walk *walk)
509 {
510 	struct vm_area_struct *vma = walk->vma;
511 	struct folio *folio;
512 	struct queue_pages *qp = walk->private;
513 	unsigned long flags = qp->flags;
514 	pte_t *pte, *mapped_pte;
515 	pte_t ptent;
516 	spinlock_t *ptl;
517 
518 	ptl = pmd_trans_huge_lock(pmd, vma);
519 	if (ptl)
520 		return queue_folios_pmd(pmd, ptl, addr, end, walk);
521 
522 	mapped_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
523 	if (!pte) {
524 		walk->action = ACTION_AGAIN;
525 		return 0;
526 	}
527 	for (; addr != end; pte++, addr += PAGE_SIZE) {
528 		ptent = ptep_get(pte);
529 		if (!pte_present(ptent))
530 			continue;
531 		folio = vm_normal_folio(vma, addr, ptent);
532 		if (!folio || folio_is_zone_device(folio))
533 			continue;
534 		/*
535 		 * vm_normal_folio() filters out zero pages, but there might
536 		 * still be reserved folios to skip, perhaps in a VDSO.
537 		 */
538 		if (folio_test_reserved(folio))
539 			continue;
540 		if (!queue_folio_required(folio, qp))
541 			continue;
542 		if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
543 			/*
544 			 * MPOL_MF_STRICT must be specified if we get here.
545 			 * Continue walking vmas due to MPOL_MF_MOVE* flags.
546 			 */
547 			if (!vma_migratable(vma))
548 				qp->has_unmovable = true;
549 
550 			/*
551 			 * Do not abort immediately since there may be
552 			 * temporary off LRU pages in the range.  Still
553 			 * need migrate other LRU pages.
554 			 */
555 			if (migrate_folio_add(folio, qp->pagelist, flags))
556 				qp->has_unmovable = true;
557 		} else
558 			break;
559 	}
560 	pte_unmap_unlock(mapped_pte, ptl);
561 	cond_resched();
562 
563 	return addr != end ? -EIO : 0;
564 }
565 
queue_folios_hugetlb(pte_t * pte,unsigned long hmask,unsigned long addr,unsigned long end,struct mm_walk * walk)566 static int queue_folios_hugetlb(pte_t *pte, unsigned long hmask,
567 			       unsigned long addr, unsigned long end,
568 			       struct mm_walk *walk)
569 {
570 	int ret = 0;
571 #ifdef CONFIG_HUGETLB_PAGE
572 	struct queue_pages *qp = walk->private;
573 	unsigned long flags = (qp->flags & MPOL_MF_VALID);
574 	struct folio *folio;
575 	spinlock_t *ptl;
576 	pte_t entry;
577 
578 	ptl = huge_pte_lock(hstate_vma(walk->vma), walk->mm, pte);
579 	entry = huge_ptep_get(pte);
580 	if (!pte_present(entry))
581 		goto unlock;
582 	folio = pfn_folio(pte_pfn(entry));
583 	if (!queue_folio_required(folio, qp))
584 		goto unlock;
585 
586 	if (flags == MPOL_MF_STRICT) {
587 		/*
588 		 * STRICT alone means only detecting misplaced folio and no
589 		 * need to further check other vma.
590 		 */
591 		ret = -EIO;
592 		goto unlock;
593 	}
594 
595 	if (!vma_migratable(walk->vma)) {
596 		/*
597 		 * Must be STRICT with MOVE*, otherwise .test_walk() have
598 		 * stopped walking current vma.
599 		 * Detecting misplaced folio but allow migrating folios which
600 		 * have been queued.
601 		 */
602 		qp->has_unmovable = true;
603 		goto unlock;
604 	}
605 
606 	/*
607 	 * With MPOL_MF_MOVE, we try to migrate only unshared folios. If it
608 	 * is shared it is likely not worth migrating.
609 	 *
610 	 * To check if the folio is shared, ideally we want to make sure
611 	 * every page is mapped to the same process. Doing that is very
612 	 * expensive, so check the estimated mapcount of the folio instead.
613 	 */
614 	if (flags & (MPOL_MF_MOVE_ALL) ||
615 	    (flags & MPOL_MF_MOVE && folio_estimated_sharers(folio) == 1 &&
616 	     !hugetlb_pmd_shared(pte))) {
617 		if (!isolate_hugetlb(folio, qp->pagelist) &&
618 			(flags & MPOL_MF_STRICT))
619 			/*
620 			 * Failed to isolate folio but allow migrating pages
621 			 * which have been queued.
622 			 */
623 			qp->has_unmovable = true;
624 	}
625 unlock:
626 	spin_unlock(ptl);
627 #else
628 	BUG();
629 #endif
630 	return ret;
631 }
632 
633 #ifdef CONFIG_NUMA_BALANCING
634 /*
635  * This is used to mark a range of virtual addresses to be inaccessible.
636  * These are later cleared by a NUMA hinting fault. Depending on these
637  * faults, pages may be migrated for better NUMA placement.
638  *
639  * This is assuming that NUMA faults are handled using PROT_NONE. If
640  * an architecture makes a different choice, it will need further
641  * changes to the core.
642  */
change_prot_numa(struct vm_area_struct * vma,unsigned long addr,unsigned long end)643 unsigned long change_prot_numa(struct vm_area_struct *vma,
644 			unsigned long addr, unsigned long end)
645 {
646 	struct mmu_gather tlb;
647 	long nr_updated;
648 
649 	tlb_gather_mmu(&tlb, vma->vm_mm);
650 
651 	nr_updated = change_protection(&tlb, vma, addr, end, MM_CP_PROT_NUMA);
652 	if (nr_updated > 0)
653 		count_vm_numa_events(NUMA_PTE_UPDATES, nr_updated);
654 
655 	tlb_finish_mmu(&tlb);
656 
657 	return nr_updated;
658 }
659 #else
change_prot_numa(struct vm_area_struct * vma,unsigned long addr,unsigned long end)660 static unsigned long change_prot_numa(struct vm_area_struct *vma,
661 			unsigned long addr, unsigned long end)
662 {
663 	return 0;
664 }
665 #endif /* CONFIG_NUMA_BALANCING */
666 
queue_pages_test_walk(unsigned long start,unsigned long end,struct mm_walk * walk)667 static int queue_pages_test_walk(unsigned long start, unsigned long end,
668 				struct mm_walk *walk)
669 {
670 	struct vm_area_struct *next, *vma = walk->vma;
671 	struct queue_pages *qp = walk->private;
672 	unsigned long endvma = vma->vm_end;
673 	unsigned long flags = qp->flags;
674 
675 	/* range check first */
676 	VM_BUG_ON_VMA(!range_in_vma(vma, start, end), vma);
677 
678 	if (!qp->first) {
679 		qp->first = vma;
680 		if (!(flags & MPOL_MF_DISCONTIG_OK) &&
681 			(qp->start < vma->vm_start))
682 			/* hole at head side of range */
683 			return -EFAULT;
684 	}
685 	next = find_vma(vma->vm_mm, vma->vm_end);
686 	if (!(flags & MPOL_MF_DISCONTIG_OK) &&
687 		((vma->vm_end < qp->end) &&
688 		(!next || vma->vm_end < next->vm_start)))
689 		/* hole at middle or tail of range */
690 		return -EFAULT;
691 
692 	/*
693 	 * Need check MPOL_MF_STRICT to return -EIO if possible
694 	 * regardless of vma_migratable
695 	 */
696 	if (!vma_migratable(vma) &&
697 	    !(flags & MPOL_MF_STRICT))
698 		return 1;
699 
700 	if (endvma > end)
701 		endvma = end;
702 
703 	if (flags & MPOL_MF_LAZY) {
704 		/* Similar to task_numa_work, skip inaccessible VMAs */
705 		if (!is_vm_hugetlb_page(vma) && vma_is_accessible(vma) &&
706 			!(vma->vm_flags & VM_MIXEDMAP))
707 			change_prot_numa(vma, start, endvma);
708 		return 1;
709 	}
710 
711 	/* queue pages from current vma */
712 	if (flags & MPOL_MF_VALID)
713 		return 0;
714 	return 1;
715 }
716 
717 static const struct mm_walk_ops queue_pages_walk_ops = {
718 	.hugetlb_entry		= queue_folios_hugetlb,
719 	.pmd_entry		= queue_folios_pte_range,
720 	.test_walk		= queue_pages_test_walk,
721 	.walk_lock		= PGWALK_RDLOCK,
722 };
723 
724 static const struct mm_walk_ops queue_pages_lock_vma_walk_ops = {
725 	.hugetlb_entry		= queue_folios_hugetlb,
726 	.pmd_entry		= queue_folios_pte_range,
727 	.test_walk		= queue_pages_test_walk,
728 	.walk_lock		= PGWALK_WRLOCK,
729 };
730 
731 /*
732  * Walk through page tables and collect pages to be migrated.
733  *
734  * If pages found in a given range are on a set of nodes (determined by
735  * @nodes and @flags,) it's isolated and queued to the pagelist which is
736  * passed via @private.
737  *
738  * queue_pages_range() has three possible return values:
739  * 1 - there is unmovable page, but MPOL_MF_MOVE* & MPOL_MF_STRICT were
740  *     specified.
741  * 0 - queue pages successfully or no misplaced page.
742  * errno - i.e. misplaced pages with MPOL_MF_STRICT specified (-EIO) or
743  *         memory range specified by nodemask and maxnode points outside
744  *         your accessible address space (-EFAULT)
745  */
746 static int
queue_pages_range(struct mm_struct * mm,unsigned long start,unsigned long end,nodemask_t * nodes,unsigned long flags,struct list_head * pagelist,bool lock_vma)747 queue_pages_range(struct mm_struct *mm, unsigned long start, unsigned long end,
748 		nodemask_t *nodes, unsigned long flags,
749 		struct list_head *pagelist, bool lock_vma)
750 {
751 	int err;
752 	struct queue_pages qp = {
753 		.pagelist = pagelist,
754 		.flags = flags,
755 		.nmask = nodes,
756 		.start = start,
757 		.end = end,
758 		.first = NULL,
759 		.has_unmovable = false,
760 	};
761 	const struct mm_walk_ops *ops = lock_vma ?
762 			&queue_pages_lock_vma_walk_ops : &queue_pages_walk_ops;
763 
764 	err = walk_page_range(mm, start, end, ops, &qp);
765 
766 	if (qp.has_unmovable)
767 		err = 1;
768 	if (!qp.first)
769 		/* whole range in hole */
770 		err = -EFAULT;
771 
772 	return err;
773 }
774 
775 /*
776  * Apply policy to a single VMA
777  * This must be called with the mmap_lock held for writing.
778  */
vma_replace_policy(struct vm_area_struct * vma,struct mempolicy * pol)779 static int vma_replace_policy(struct vm_area_struct *vma,
780 						struct mempolicy *pol)
781 {
782 	int err;
783 	struct mempolicy *old;
784 	struct mempolicy *new;
785 
786 	vma_assert_write_locked(vma);
787 
788 	pr_debug("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n",
789 		 vma->vm_start, vma->vm_end, vma->vm_pgoff,
790 		 vma->vm_ops, vma->vm_file,
791 		 vma->vm_ops ? vma->vm_ops->set_policy : NULL);
792 
793 	new = mpol_dup(pol);
794 	if (IS_ERR(new))
795 		return PTR_ERR(new);
796 
797 	if (vma->vm_ops && vma->vm_ops->set_policy) {
798 		err = vma->vm_ops->set_policy(vma, new);
799 		if (err)
800 			goto err_out;
801 	}
802 
803 	old = vma->vm_policy;
804 	vma->vm_policy = new; /* protected by mmap_lock */
805 	mpol_put(old);
806 
807 	return 0;
808  err_out:
809 	mpol_put(new);
810 	return err;
811 }
812 
813 /* Split or merge the VMA (if required) and apply the new policy */
mbind_range(struct vma_iterator * vmi,struct vm_area_struct * vma,struct vm_area_struct ** prev,unsigned long start,unsigned long end,struct mempolicy * new_pol)814 static int mbind_range(struct vma_iterator *vmi, struct vm_area_struct *vma,
815 		struct vm_area_struct **prev, unsigned long start,
816 		unsigned long end, struct mempolicy *new_pol)
817 {
818 	struct vm_area_struct *merged;
819 	unsigned long vmstart, vmend;
820 	pgoff_t pgoff;
821 	int err;
822 
823 	vmend = min(end, vma->vm_end);
824 	if (start > vma->vm_start) {
825 		*prev = vma;
826 		vmstart = start;
827 	} else {
828 		vmstart = vma->vm_start;
829 	}
830 
831 	if (mpol_equal(vma_policy(vma), new_pol)) {
832 		*prev = vma;
833 		return 0;
834 	}
835 
836 	pgoff = vma->vm_pgoff + ((vmstart - vma->vm_start) >> PAGE_SHIFT);
837 	merged = vma_merge(vmi, vma->vm_mm, *prev, vmstart, vmend, vma->vm_flags,
838 			 vma->anon_vma, vma->vm_file, pgoff, new_pol,
839 			 vma->vm_userfaultfd_ctx, anon_vma_name(vma));
840 	if (merged) {
841 		*prev = merged;
842 		return vma_replace_policy(merged, new_pol);
843 	}
844 
845 	if (vma->vm_start != vmstart) {
846 		err = split_vma(vmi, vma, vmstart, 1);
847 		if (err)
848 			return err;
849 	}
850 
851 	if (vma->vm_end != vmend) {
852 		err = split_vma(vmi, vma, vmend, 0);
853 		if (err)
854 			return err;
855 	}
856 
857 	*prev = vma;
858 	return vma_replace_policy(vma, new_pol);
859 }
860 
861 /* Set the process memory policy */
do_set_mempolicy(unsigned short mode,unsigned short flags,nodemask_t * nodes)862 static long do_set_mempolicy(unsigned short mode, unsigned short flags,
863 			     nodemask_t *nodes)
864 {
865 	struct mempolicy *new, *old;
866 	NODEMASK_SCRATCH(scratch);
867 	int ret;
868 
869 	if (!scratch)
870 		return -ENOMEM;
871 
872 	new = mpol_new(mode, flags, nodes);
873 	if (IS_ERR(new)) {
874 		ret = PTR_ERR(new);
875 		goto out;
876 	}
877 
878 	task_lock(current);
879 	ret = mpol_set_nodemask(new, nodes, scratch);
880 	if (ret) {
881 		task_unlock(current);
882 		mpol_put(new);
883 		goto out;
884 	}
885 
886 	old = current->mempolicy;
887 	current->mempolicy = new;
888 	if (new && new->mode == MPOL_INTERLEAVE)
889 		current->il_prev = MAX_NUMNODES-1;
890 	task_unlock(current);
891 	mpol_put(old);
892 	ret = 0;
893 out:
894 	NODEMASK_SCRATCH_FREE(scratch);
895 	return ret;
896 }
897 
898 /*
899  * Return nodemask for policy for get_mempolicy() query
900  *
901  * Called with task's alloc_lock held
902  */
get_policy_nodemask(struct mempolicy * p,nodemask_t * nodes)903 static void get_policy_nodemask(struct mempolicy *p, nodemask_t *nodes)
904 {
905 	nodes_clear(*nodes);
906 	if (p == &default_policy)
907 		return;
908 
909 	switch (p->mode) {
910 	case MPOL_BIND:
911 	case MPOL_INTERLEAVE:
912 	case MPOL_PREFERRED:
913 	case MPOL_PREFERRED_MANY:
914 		*nodes = p->nodes;
915 		break;
916 	case MPOL_LOCAL:
917 		/* return empty node mask for local allocation */
918 		break;
919 	default:
920 		BUG();
921 	}
922 }
923 
lookup_node(struct mm_struct * mm,unsigned long addr)924 static int lookup_node(struct mm_struct *mm, unsigned long addr)
925 {
926 	struct page *p = NULL;
927 	int ret;
928 
929 	ret = get_user_pages_fast(addr & PAGE_MASK, 1, 0, &p);
930 	if (ret > 0) {
931 		ret = page_to_nid(p);
932 		put_page(p);
933 	}
934 	return ret;
935 }
936 
937 /* Retrieve NUMA policy */
do_get_mempolicy(int * policy,nodemask_t * nmask,unsigned long addr,unsigned long flags)938 static long do_get_mempolicy(int *policy, nodemask_t *nmask,
939 			     unsigned long addr, unsigned long flags)
940 {
941 	int err;
942 	struct mm_struct *mm = current->mm;
943 	struct vm_area_struct *vma = NULL;
944 	struct mempolicy *pol = current->mempolicy, *pol_refcount = NULL;
945 
946 	if (flags &
947 		~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR|MPOL_F_MEMS_ALLOWED))
948 		return -EINVAL;
949 
950 	if (flags & MPOL_F_MEMS_ALLOWED) {
951 		if (flags & (MPOL_F_NODE|MPOL_F_ADDR))
952 			return -EINVAL;
953 		*policy = 0;	/* just so it's initialized */
954 		task_lock(current);
955 		*nmask  = cpuset_current_mems_allowed;
956 		task_unlock(current);
957 		return 0;
958 	}
959 
960 	if (flags & MPOL_F_ADDR) {
961 		/*
962 		 * Do NOT fall back to task policy if the
963 		 * vma/shared policy at addr is NULL.  We
964 		 * want to return MPOL_DEFAULT in this case.
965 		 */
966 		mmap_read_lock(mm);
967 		vma = vma_lookup(mm, addr);
968 		if (!vma) {
969 			mmap_read_unlock(mm);
970 			return -EFAULT;
971 		}
972 		if (vma->vm_ops && vma->vm_ops->get_policy)
973 			pol = vma->vm_ops->get_policy(vma, addr);
974 		else
975 			pol = vma->vm_policy;
976 	} else if (addr)
977 		return -EINVAL;
978 
979 	if (!pol)
980 		pol = &default_policy;	/* indicates default behavior */
981 
982 	if (flags & MPOL_F_NODE) {
983 		if (flags & MPOL_F_ADDR) {
984 			/*
985 			 * Take a refcount on the mpol, because we are about to
986 			 * drop the mmap_lock, after which only "pol" remains
987 			 * valid, "vma" is stale.
988 			 */
989 			pol_refcount = pol;
990 			vma = NULL;
991 			mpol_get(pol);
992 			mmap_read_unlock(mm);
993 			err = lookup_node(mm, addr);
994 			if (err < 0)
995 				goto out;
996 			*policy = err;
997 		} else if (pol == current->mempolicy &&
998 				pol->mode == MPOL_INTERLEAVE) {
999 			*policy = next_node_in(current->il_prev, pol->nodes);
1000 		} else {
1001 			err = -EINVAL;
1002 			goto out;
1003 		}
1004 	} else {
1005 		*policy = pol == &default_policy ? MPOL_DEFAULT :
1006 						pol->mode;
1007 		/*
1008 		 * Internal mempolicy flags must be masked off before exposing
1009 		 * the policy to userspace.
1010 		 */
1011 		*policy |= (pol->flags & MPOL_MODE_FLAGS);
1012 	}
1013 
1014 	err = 0;
1015 	if (nmask) {
1016 		if (mpol_store_user_nodemask(pol)) {
1017 			*nmask = pol->w.user_nodemask;
1018 		} else {
1019 			task_lock(current);
1020 			get_policy_nodemask(pol, nmask);
1021 			task_unlock(current);
1022 		}
1023 	}
1024 
1025  out:
1026 	mpol_cond_put(pol);
1027 	if (vma)
1028 		mmap_read_unlock(mm);
1029 	if (pol_refcount)
1030 		mpol_put(pol_refcount);
1031 	return err;
1032 }
1033 
1034 #ifdef CONFIG_MIGRATION
migrate_folio_add(struct folio * folio,struct list_head * foliolist,unsigned long flags)1035 static int migrate_folio_add(struct folio *folio, struct list_head *foliolist,
1036 				unsigned long flags)
1037 {
1038 	/*
1039 	 * We try to migrate only unshared folios. If it is shared it
1040 	 * is likely not worth migrating.
1041 	 *
1042 	 * To check if the folio is shared, ideally we want to make sure
1043 	 * every page is mapped to the same process. Doing that is very
1044 	 * expensive, so check the estimated mapcount of the folio instead.
1045 	 */
1046 	if ((flags & MPOL_MF_MOVE_ALL) || folio_estimated_sharers(folio) == 1) {
1047 		if (folio_isolate_lru(folio)) {
1048 			list_add_tail(&folio->lru, foliolist);
1049 			node_stat_mod_folio(folio,
1050 				NR_ISOLATED_ANON + folio_is_file_lru(folio),
1051 				folio_nr_pages(folio));
1052 		} else if (flags & MPOL_MF_STRICT) {
1053 			/*
1054 			 * Non-movable folio may reach here.  And, there may be
1055 			 * temporary off LRU folios or non-LRU movable folios.
1056 			 * Treat them as unmovable folios since they can't be
1057 			 * isolated, so they can't be moved at the moment.  It
1058 			 * should return -EIO for this case too.
1059 			 */
1060 			return -EIO;
1061 		}
1062 	}
1063 
1064 	return 0;
1065 }
1066 
1067 /*
1068  * Migrate pages from one node to a target node.
1069  * Returns error or the number of pages not migrated.
1070  */
migrate_to_node(struct mm_struct * mm,int source,int dest,int flags)1071 static int migrate_to_node(struct mm_struct *mm, int source, int dest,
1072 			   int flags)
1073 {
1074 	nodemask_t nmask;
1075 	struct vm_area_struct *vma;
1076 	LIST_HEAD(pagelist);
1077 	int err = 0;
1078 	struct migration_target_control mtc = {
1079 		.nid = dest,
1080 		.gfp_mask = GFP_HIGHUSER_MOVABLE | __GFP_THISNODE,
1081 	};
1082 
1083 	nodes_clear(nmask);
1084 	node_set(source, nmask);
1085 
1086 	/*
1087 	 * This does not "check" the range but isolates all pages that
1088 	 * need migration.  Between passing in the full user address
1089 	 * space range and MPOL_MF_DISCONTIG_OK, this call can not fail.
1090 	 */
1091 	vma = find_vma(mm, 0);
1092 	VM_BUG_ON(!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)));
1093 	queue_pages_range(mm, vma->vm_start, mm->task_size, &nmask,
1094 			flags | MPOL_MF_DISCONTIG_OK, &pagelist, false);
1095 
1096 	if (!list_empty(&pagelist)) {
1097 		err = migrate_pages(&pagelist, alloc_migration_target, NULL,
1098 				(unsigned long)&mtc, MIGRATE_SYNC, MR_SYSCALL, NULL);
1099 		if (err)
1100 			putback_movable_pages(&pagelist);
1101 	}
1102 
1103 	return err;
1104 }
1105 
1106 /*
1107  * Move pages between the two nodesets so as to preserve the physical
1108  * layout as much as possible.
1109  *
1110  * Returns the number of page that could not be moved.
1111  */
do_migrate_pages(struct mm_struct * mm,const nodemask_t * from,const nodemask_t * to,int flags)1112 int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
1113 		     const nodemask_t *to, int flags)
1114 {
1115 	int busy = 0;
1116 	int err = 0;
1117 	nodemask_t tmp;
1118 
1119 	lru_cache_disable();
1120 
1121 	mmap_read_lock(mm);
1122 
1123 	/*
1124 	 * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
1125 	 * bit in 'to' is not also set in 'tmp'.  Clear the found 'source'
1126 	 * bit in 'tmp', and return that <source, dest> pair for migration.
1127 	 * The pair of nodemasks 'to' and 'from' define the map.
1128 	 *
1129 	 * If no pair of bits is found that way, fallback to picking some
1130 	 * pair of 'source' and 'dest' bits that are not the same.  If the
1131 	 * 'source' and 'dest' bits are the same, this represents a node
1132 	 * that will be migrating to itself, so no pages need move.
1133 	 *
1134 	 * If no bits are left in 'tmp', or if all remaining bits left
1135 	 * in 'tmp' correspond to the same bit in 'to', return false
1136 	 * (nothing left to migrate).
1137 	 *
1138 	 * This lets us pick a pair of nodes to migrate between, such that
1139 	 * if possible the dest node is not already occupied by some other
1140 	 * source node, minimizing the risk of overloading the memory on a
1141 	 * node that would happen if we migrated incoming memory to a node
1142 	 * before migrating outgoing memory source that same node.
1143 	 *
1144 	 * A single scan of tmp is sufficient.  As we go, we remember the
1145 	 * most recent <s, d> pair that moved (s != d).  If we find a pair
1146 	 * that not only moved, but what's better, moved to an empty slot
1147 	 * (d is not set in tmp), then we break out then, with that pair.
1148 	 * Otherwise when we finish scanning from_tmp, we at least have the
1149 	 * most recent <s, d> pair that moved.  If we get all the way through
1150 	 * the scan of tmp without finding any node that moved, much less
1151 	 * moved to an empty node, then there is nothing left worth migrating.
1152 	 */
1153 
1154 	tmp = *from;
1155 	while (!nodes_empty(tmp)) {
1156 		int s, d;
1157 		int source = NUMA_NO_NODE;
1158 		int dest = 0;
1159 
1160 		for_each_node_mask(s, tmp) {
1161 
1162 			/*
1163 			 * do_migrate_pages() tries to maintain the relative
1164 			 * node relationship of the pages established between
1165 			 * threads and memory areas.
1166                          *
1167 			 * However if the number of source nodes is not equal to
1168 			 * the number of destination nodes we can not preserve
1169 			 * this node relative relationship.  In that case, skip
1170 			 * copying memory from a node that is in the destination
1171 			 * mask.
1172 			 *
1173 			 * Example: [2,3,4] -> [3,4,5] moves everything.
1174 			 *          [0-7] - > [3,4,5] moves only 0,1,2,6,7.
1175 			 */
1176 
1177 			if ((nodes_weight(*from) != nodes_weight(*to)) &&
1178 						(node_isset(s, *to)))
1179 				continue;
1180 
1181 			d = node_remap(s, *from, *to);
1182 			if (s == d)
1183 				continue;
1184 
1185 			source = s;	/* Node moved. Memorize */
1186 			dest = d;
1187 
1188 			/* dest not in remaining from nodes? */
1189 			if (!node_isset(dest, tmp))
1190 				break;
1191 		}
1192 		if (source == NUMA_NO_NODE)
1193 			break;
1194 
1195 		node_clear(source, tmp);
1196 		err = migrate_to_node(mm, source, dest, flags);
1197 		if (err > 0)
1198 			busy += err;
1199 		if (err < 0)
1200 			break;
1201 	}
1202 	mmap_read_unlock(mm);
1203 
1204 	lru_cache_enable();
1205 	if (err < 0)
1206 		return err;
1207 	return busy;
1208 
1209 }
1210 
1211 /*
1212  * Allocate a new page for page migration based on vma policy.
1213  * Start by assuming the page is mapped by the same vma as contains @start.
1214  * Search forward from there, if not.  N.B., this assumes that the
1215  * list of pages handed to migrate_pages()--which is how we get here--
1216  * is in virtual address order.
1217  */
new_folio(struct folio * src,unsigned long start)1218 static struct folio *new_folio(struct folio *src, unsigned long start)
1219 {
1220 	struct vm_area_struct *vma;
1221 	unsigned long address;
1222 	VMA_ITERATOR(vmi, current->mm, start);
1223 	gfp_t gfp = GFP_HIGHUSER_MOVABLE | __GFP_RETRY_MAYFAIL;
1224 
1225 	for_each_vma(vmi, vma) {
1226 		address = page_address_in_vma(&src->page, vma);
1227 		if (address != -EFAULT)
1228 			break;
1229 	}
1230 
1231 	if (folio_test_hugetlb(src)) {
1232 		return alloc_hugetlb_folio_vma(folio_hstate(src),
1233 				vma, address);
1234 	}
1235 
1236 	if (folio_test_large(src))
1237 		gfp = GFP_TRANSHUGE;
1238 
1239 	/*
1240 	 * if !vma, vma_alloc_folio() will use task or system default policy
1241 	 */
1242 	return vma_alloc_folio(gfp, folio_order(src), vma, address,
1243 			folio_test_large(src));
1244 }
1245 #else
1246 
migrate_folio_add(struct folio * folio,struct list_head * foliolist,unsigned long flags)1247 static int migrate_folio_add(struct folio *folio, struct list_head *foliolist,
1248 				unsigned long flags)
1249 {
1250 	return -EIO;
1251 }
1252 
do_migrate_pages(struct mm_struct * mm,const nodemask_t * from,const nodemask_t * to,int flags)1253 int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
1254 		     const nodemask_t *to, int flags)
1255 {
1256 	return -ENOSYS;
1257 }
1258 
new_folio(struct folio * src,unsigned long start)1259 static struct folio *new_folio(struct folio *src, unsigned long start)
1260 {
1261 	return NULL;
1262 }
1263 #endif
1264 
do_mbind(unsigned long start,unsigned long len,unsigned short mode,unsigned short mode_flags,nodemask_t * nmask,unsigned long flags)1265 static long do_mbind(unsigned long start, unsigned long len,
1266 		     unsigned short mode, unsigned short mode_flags,
1267 		     nodemask_t *nmask, unsigned long flags)
1268 {
1269 	struct mm_struct *mm = current->mm;
1270 	struct vm_area_struct *vma, *prev;
1271 	struct vma_iterator vmi;
1272 	struct mempolicy *new;
1273 	unsigned long end;
1274 	int err;
1275 	int ret;
1276 	LIST_HEAD(pagelist);
1277 
1278 	if (flags & ~(unsigned long)MPOL_MF_VALID)
1279 		return -EINVAL;
1280 	if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
1281 		return -EPERM;
1282 
1283 	if (start & ~PAGE_MASK)
1284 		return -EINVAL;
1285 
1286 	if (mode == MPOL_DEFAULT)
1287 		flags &= ~MPOL_MF_STRICT;
1288 
1289 	len = PAGE_ALIGN(len);
1290 	end = start + len;
1291 
1292 	if (end < start)
1293 		return -EINVAL;
1294 	if (end == start)
1295 		return 0;
1296 
1297 	new = mpol_new(mode, mode_flags, nmask);
1298 	if (IS_ERR(new))
1299 		return PTR_ERR(new);
1300 
1301 	if (flags & MPOL_MF_LAZY)
1302 		new->flags |= MPOL_F_MOF;
1303 
1304 	/*
1305 	 * If we are using the default policy then operation
1306 	 * on discontinuous address spaces is okay after all
1307 	 */
1308 	if (!new)
1309 		flags |= MPOL_MF_DISCONTIG_OK;
1310 
1311 	pr_debug("mbind %lx-%lx mode:%d flags:%d nodes:%lx\n",
1312 		 start, start + len, mode, mode_flags,
1313 		 nmask ? nodes_addr(*nmask)[0] : NUMA_NO_NODE);
1314 
1315 	if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
1316 
1317 		lru_cache_disable();
1318 	}
1319 	{
1320 		NODEMASK_SCRATCH(scratch);
1321 		if (scratch) {
1322 			mmap_write_lock(mm);
1323 			err = mpol_set_nodemask(new, nmask, scratch);
1324 			if (err)
1325 				mmap_write_unlock(mm);
1326 		} else
1327 			err = -ENOMEM;
1328 		NODEMASK_SCRATCH_FREE(scratch);
1329 	}
1330 	if (err)
1331 		goto mpol_out;
1332 
1333 	/*
1334 	 * Lock the VMAs before scanning for pages to migrate, to ensure we don't
1335 	 * miss a concurrently inserted page.
1336 	 */
1337 	ret = queue_pages_range(mm, start, end, nmask,
1338 			  flags | MPOL_MF_INVERT, &pagelist, true);
1339 
1340 	if (ret < 0) {
1341 		err = ret;
1342 		goto up_out;
1343 	}
1344 
1345 	vma_iter_init(&vmi, mm, start);
1346 	prev = vma_prev(&vmi);
1347 	for_each_vma_range(vmi, vma, end) {
1348 		err = mbind_range(&vmi, vma, &prev, start, end, new);
1349 		if (err)
1350 			break;
1351 	}
1352 
1353 	if (!err) {
1354 		int nr_failed = 0;
1355 
1356 		if (!list_empty(&pagelist)) {
1357 			WARN_ON_ONCE(flags & MPOL_MF_LAZY);
1358 			nr_failed = migrate_pages(&pagelist, new_folio, NULL,
1359 				start, MIGRATE_SYNC, MR_MEMPOLICY_MBIND, NULL);
1360 			if (nr_failed)
1361 				putback_movable_pages(&pagelist);
1362 		}
1363 
1364 		if (((ret > 0) || nr_failed) && (flags & MPOL_MF_STRICT))
1365 			err = -EIO;
1366 	} else {
1367 up_out:
1368 		if (!list_empty(&pagelist))
1369 			putback_movable_pages(&pagelist);
1370 	}
1371 
1372 	mmap_write_unlock(mm);
1373 mpol_out:
1374 	mpol_put(new);
1375 	if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
1376 		lru_cache_enable();
1377 	return err;
1378 }
1379 
1380 /*
1381  * User space interface with variable sized bitmaps for nodelists.
1382  */
get_bitmap(unsigned long * mask,const unsigned long __user * nmask,unsigned long maxnode)1383 static int get_bitmap(unsigned long *mask, const unsigned long __user *nmask,
1384 		      unsigned long maxnode)
1385 {
1386 	unsigned long nlongs = BITS_TO_LONGS(maxnode);
1387 	int ret;
1388 
1389 	if (in_compat_syscall())
1390 		ret = compat_get_bitmap(mask,
1391 					(const compat_ulong_t __user *)nmask,
1392 					maxnode);
1393 	else
1394 		ret = copy_from_user(mask, nmask,
1395 				     nlongs * sizeof(unsigned long));
1396 
1397 	if (ret)
1398 		return -EFAULT;
1399 
1400 	if (maxnode % BITS_PER_LONG)
1401 		mask[nlongs - 1] &= (1UL << (maxnode % BITS_PER_LONG)) - 1;
1402 
1403 	return 0;
1404 }
1405 
1406 /* Copy a node mask from user space. */
get_nodes(nodemask_t * nodes,const unsigned long __user * nmask,unsigned long maxnode)1407 static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask,
1408 		     unsigned long maxnode)
1409 {
1410 	--maxnode;
1411 	nodes_clear(*nodes);
1412 	if (maxnode == 0 || !nmask)
1413 		return 0;
1414 	if (maxnode > PAGE_SIZE*BITS_PER_BYTE)
1415 		return -EINVAL;
1416 
1417 	/*
1418 	 * When the user specified more nodes than supported just check
1419 	 * if the non supported part is all zero, one word at a time,
1420 	 * starting at the end.
1421 	 */
1422 	while (maxnode > MAX_NUMNODES) {
1423 		unsigned long bits = min_t(unsigned long, maxnode, BITS_PER_LONG);
1424 		unsigned long t;
1425 
1426 		if (get_bitmap(&t, &nmask[(maxnode - 1) / BITS_PER_LONG], bits))
1427 			return -EFAULT;
1428 
1429 		if (maxnode - bits >= MAX_NUMNODES) {
1430 			maxnode -= bits;
1431 		} else {
1432 			maxnode = MAX_NUMNODES;
1433 			t &= ~((1UL << (MAX_NUMNODES % BITS_PER_LONG)) - 1);
1434 		}
1435 		if (t)
1436 			return -EINVAL;
1437 	}
1438 
1439 	return get_bitmap(nodes_addr(*nodes), nmask, maxnode);
1440 }
1441 
1442 /* Copy a kernel node mask to user space */
copy_nodes_to_user(unsigned long __user * mask,unsigned long maxnode,nodemask_t * nodes)1443 static int copy_nodes_to_user(unsigned long __user *mask, unsigned long maxnode,
1444 			      nodemask_t *nodes)
1445 {
1446 	unsigned long copy = ALIGN(maxnode-1, 64) / 8;
1447 	unsigned int nbytes = BITS_TO_LONGS(nr_node_ids) * sizeof(long);
1448 	bool compat = in_compat_syscall();
1449 
1450 	if (compat)
1451 		nbytes = BITS_TO_COMPAT_LONGS(nr_node_ids) * sizeof(compat_long_t);
1452 
1453 	if (copy > nbytes) {
1454 		if (copy > PAGE_SIZE)
1455 			return -EINVAL;
1456 		if (clear_user((char __user *)mask + nbytes, copy - nbytes))
1457 			return -EFAULT;
1458 		copy = nbytes;
1459 		maxnode = nr_node_ids;
1460 	}
1461 
1462 	if (compat)
1463 		return compat_put_bitmap((compat_ulong_t __user *)mask,
1464 					 nodes_addr(*nodes), maxnode);
1465 
1466 	return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0;
1467 }
1468 
1469 /* Basic parameter sanity check used by both mbind() and set_mempolicy() */
sanitize_mpol_flags(int * mode,unsigned short * flags)1470 static inline int sanitize_mpol_flags(int *mode, unsigned short *flags)
1471 {
1472 	*flags = *mode & MPOL_MODE_FLAGS;
1473 	*mode &= ~MPOL_MODE_FLAGS;
1474 
1475 	if ((unsigned int)(*mode) >=  MPOL_MAX)
1476 		return -EINVAL;
1477 	if ((*flags & MPOL_F_STATIC_NODES) && (*flags & MPOL_F_RELATIVE_NODES))
1478 		return -EINVAL;
1479 	if (*flags & MPOL_F_NUMA_BALANCING) {
1480 		if (*mode != MPOL_BIND)
1481 			return -EINVAL;
1482 		*flags |= (MPOL_F_MOF | MPOL_F_MORON);
1483 	}
1484 	return 0;
1485 }
1486 
kernel_mbind(unsigned long start,unsigned long len,unsigned long mode,const unsigned long __user * nmask,unsigned long maxnode,unsigned int flags)1487 static long kernel_mbind(unsigned long start, unsigned long len,
1488 			 unsigned long mode, const unsigned long __user *nmask,
1489 			 unsigned long maxnode, unsigned int flags)
1490 {
1491 	unsigned short mode_flags;
1492 	nodemask_t nodes;
1493 	int lmode = mode;
1494 	int err;
1495 
1496 	start = untagged_addr(start);
1497 	err = sanitize_mpol_flags(&lmode, &mode_flags);
1498 	if (err)
1499 		return err;
1500 
1501 	err = get_nodes(&nodes, nmask, maxnode);
1502 	if (err)
1503 		return err;
1504 
1505 	return do_mbind(start, len, lmode, mode_flags, &nodes, flags);
1506 }
1507 
SYSCALL_DEFINE4(set_mempolicy_home_node,unsigned long,start,unsigned long,len,unsigned long,home_node,unsigned long,flags)1508 SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, len,
1509 		unsigned long, home_node, unsigned long, flags)
1510 {
1511 	struct mm_struct *mm = current->mm;
1512 	struct vm_area_struct *vma, *prev;
1513 	struct mempolicy *new, *old;
1514 	unsigned long end;
1515 	int err = -ENOENT;
1516 	VMA_ITERATOR(vmi, mm, start);
1517 
1518 	start = untagged_addr(start);
1519 	if (start & ~PAGE_MASK)
1520 		return -EINVAL;
1521 	/*
1522 	 * flags is used for future extension if any.
1523 	 */
1524 	if (flags != 0)
1525 		return -EINVAL;
1526 
1527 	/*
1528 	 * Check home_node is online to avoid accessing uninitialized
1529 	 * NODE_DATA.
1530 	 */
1531 	if (home_node >= MAX_NUMNODES || !node_online(home_node))
1532 		return -EINVAL;
1533 
1534 	len = PAGE_ALIGN(len);
1535 	end = start + len;
1536 
1537 	if (end < start)
1538 		return -EINVAL;
1539 	if (end == start)
1540 		return 0;
1541 	mmap_write_lock(mm);
1542 	prev = vma_prev(&vmi);
1543 	for_each_vma_range(vmi, vma, end) {
1544 		/*
1545 		 * If any vma in the range got policy other than MPOL_BIND
1546 		 * or MPOL_PREFERRED_MANY we return error. We don't reset
1547 		 * the home node for vmas we already updated before.
1548 		 */
1549 		old = vma_policy(vma);
1550 		if (!old) {
1551 			prev = vma;
1552 			continue;
1553 		}
1554 		if (old->mode != MPOL_BIND && old->mode != MPOL_PREFERRED_MANY) {
1555 			err = -EOPNOTSUPP;
1556 			break;
1557 		}
1558 		new = mpol_dup(old);
1559 		if (IS_ERR(new)) {
1560 			err = PTR_ERR(new);
1561 			break;
1562 		}
1563 
1564 		vma_start_write(vma);
1565 		new->home_node = home_node;
1566 		err = mbind_range(&vmi, vma, &prev, start, end, new);
1567 		mpol_put(new);
1568 		if (err)
1569 			break;
1570 	}
1571 	mmap_write_unlock(mm);
1572 	return err;
1573 }
1574 
SYSCALL_DEFINE6(mbind,unsigned long,start,unsigned long,len,unsigned long,mode,const unsigned long __user *,nmask,unsigned long,maxnode,unsigned int,flags)1575 SYSCALL_DEFINE6(mbind, unsigned long, start, unsigned long, len,
1576 		unsigned long, mode, const unsigned long __user *, nmask,
1577 		unsigned long, maxnode, unsigned int, flags)
1578 {
1579 	return kernel_mbind(start, len, mode, nmask, maxnode, flags);
1580 }
1581 
1582 /* Set the process memory policy */
kernel_set_mempolicy(int mode,const unsigned long __user * nmask,unsigned long maxnode)1583 static long kernel_set_mempolicy(int mode, const unsigned long __user *nmask,
1584 				 unsigned long maxnode)
1585 {
1586 	unsigned short mode_flags;
1587 	nodemask_t nodes;
1588 	int lmode = mode;
1589 	int err;
1590 
1591 	err = sanitize_mpol_flags(&lmode, &mode_flags);
1592 	if (err)
1593 		return err;
1594 
1595 	err = get_nodes(&nodes, nmask, maxnode);
1596 	if (err)
1597 		return err;
1598 
1599 	return do_set_mempolicy(lmode, mode_flags, &nodes);
1600 }
1601 
SYSCALL_DEFINE3(set_mempolicy,int,mode,const unsigned long __user *,nmask,unsigned long,maxnode)1602 SYSCALL_DEFINE3(set_mempolicy, int, mode, const unsigned long __user *, nmask,
1603 		unsigned long, maxnode)
1604 {
1605 	return kernel_set_mempolicy(mode, nmask, maxnode);
1606 }
1607 
kernel_migrate_pages(pid_t pid,unsigned long maxnode,const unsigned long __user * old_nodes,const unsigned long __user * new_nodes)1608 static int kernel_migrate_pages(pid_t pid, unsigned long maxnode,
1609 				const unsigned long __user *old_nodes,
1610 				const unsigned long __user *new_nodes)
1611 {
1612 	struct mm_struct *mm = NULL;
1613 	struct task_struct *task;
1614 	nodemask_t task_nodes;
1615 	int err;
1616 	nodemask_t *old;
1617 	nodemask_t *new;
1618 	NODEMASK_SCRATCH(scratch);
1619 
1620 	if (!scratch)
1621 		return -ENOMEM;
1622 
1623 	old = &scratch->mask1;
1624 	new = &scratch->mask2;
1625 
1626 	err = get_nodes(old, old_nodes, maxnode);
1627 	if (err)
1628 		goto out;
1629 
1630 	err = get_nodes(new, new_nodes, maxnode);
1631 	if (err)
1632 		goto out;
1633 
1634 	/* Find the mm_struct */
1635 	rcu_read_lock();
1636 	task = pid ? find_task_by_vpid(pid) : current;
1637 	if (!task) {
1638 		rcu_read_unlock();
1639 		err = -ESRCH;
1640 		goto out;
1641 	}
1642 	get_task_struct(task);
1643 
1644 	err = -EINVAL;
1645 
1646 	/*
1647 	 * Check if this process has the right to modify the specified process.
1648 	 * Use the regular "ptrace_may_access()" checks.
1649 	 */
1650 	if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) {
1651 		rcu_read_unlock();
1652 		err = -EPERM;
1653 		goto out_put;
1654 	}
1655 	rcu_read_unlock();
1656 
1657 	task_nodes = cpuset_mems_allowed(task);
1658 	/* Is the user allowed to access the target nodes? */
1659 	if (!nodes_subset(*new, task_nodes) && !capable(CAP_SYS_NICE)) {
1660 		err = -EPERM;
1661 		goto out_put;
1662 	}
1663 
1664 	task_nodes = cpuset_mems_allowed(current);
1665 	nodes_and(*new, *new, task_nodes);
1666 	if (nodes_empty(*new))
1667 		goto out_put;
1668 
1669 	err = security_task_movememory(task);
1670 	if (err)
1671 		goto out_put;
1672 
1673 	mm = get_task_mm(task);
1674 	put_task_struct(task);
1675 
1676 	if (!mm) {
1677 		err = -EINVAL;
1678 		goto out;
1679 	}
1680 
1681 	err = do_migrate_pages(mm, old, new,
1682 		capable(CAP_SYS_NICE) ? MPOL_MF_MOVE_ALL : MPOL_MF_MOVE);
1683 
1684 	mmput(mm);
1685 out:
1686 	NODEMASK_SCRATCH_FREE(scratch);
1687 
1688 	return err;
1689 
1690 out_put:
1691 	put_task_struct(task);
1692 	goto out;
1693 
1694 }
1695 
SYSCALL_DEFINE4(migrate_pages,pid_t,pid,unsigned long,maxnode,const unsigned long __user *,old_nodes,const unsigned long __user *,new_nodes)1696 SYSCALL_DEFINE4(migrate_pages, pid_t, pid, unsigned long, maxnode,
1697 		const unsigned long __user *, old_nodes,
1698 		const unsigned long __user *, new_nodes)
1699 {
1700 	return kernel_migrate_pages(pid, maxnode, old_nodes, new_nodes);
1701 }
1702 
1703 
1704 /* Retrieve NUMA policy */
kernel_get_mempolicy(int __user * policy,unsigned long __user * nmask,unsigned long maxnode,unsigned long addr,unsigned long flags)1705 static int kernel_get_mempolicy(int __user *policy,
1706 				unsigned long __user *nmask,
1707 				unsigned long maxnode,
1708 				unsigned long addr,
1709 				unsigned long flags)
1710 {
1711 	int err;
1712 	int pval;
1713 	nodemask_t nodes;
1714 
1715 	if (nmask != NULL && maxnode < nr_node_ids)
1716 		return -EINVAL;
1717 
1718 	addr = untagged_addr(addr);
1719 
1720 	err = do_get_mempolicy(&pval, &nodes, addr, flags);
1721 
1722 	if (err)
1723 		return err;
1724 
1725 	if (policy && put_user(pval, policy))
1726 		return -EFAULT;
1727 
1728 	if (nmask)
1729 		err = copy_nodes_to_user(nmask, maxnode, &nodes);
1730 
1731 	return err;
1732 }
1733 
SYSCALL_DEFINE5(get_mempolicy,int __user *,policy,unsigned long __user *,nmask,unsigned long,maxnode,unsigned long,addr,unsigned long,flags)1734 SYSCALL_DEFINE5(get_mempolicy, int __user *, policy,
1735 		unsigned long __user *, nmask, unsigned long, maxnode,
1736 		unsigned long, addr, unsigned long, flags)
1737 {
1738 	return kernel_get_mempolicy(policy, nmask, maxnode, addr, flags);
1739 }
1740 
vma_migratable(struct vm_area_struct * vma)1741 bool vma_migratable(struct vm_area_struct *vma)
1742 {
1743 	if (vma->vm_flags & (VM_IO | VM_PFNMAP))
1744 		return false;
1745 
1746 	/*
1747 	 * DAX device mappings require predictable access latency, so avoid
1748 	 * incurring periodic faults.
1749 	 */
1750 	if (vma_is_dax(vma))
1751 		return false;
1752 
1753 	if (is_vm_hugetlb_page(vma) &&
1754 		!hugepage_migration_supported(hstate_vma(vma)))
1755 		return false;
1756 
1757 	/*
1758 	 * Migration allocates pages in the highest zone. If we cannot
1759 	 * do so then migration (at least from node to node) is not
1760 	 * possible.
1761 	 */
1762 	if (vma->vm_file &&
1763 		gfp_zone(mapping_gfp_mask(vma->vm_file->f_mapping))
1764 			< policy_zone)
1765 		return false;
1766 	return true;
1767 }
1768 
__get_vma_policy(struct vm_area_struct * vma,unsigned long addr)1769 struct mempolicy *__get_vma_policy(struct vm_area_struct *vma,
1770 						unsigned long addr)
1771 {
1772 	struct mempolicy *pol = NULL;
1773 
1774 	if (vma) {
1775 		if (vma->vm_ops && vma->vm_ops->get_policy) {
1776 			pol = vma->vm_ops->get_policy(vma, addr);
1777 		} else if (vma->vm_policy) {
1778 			pol = vma->vm_policy;
1779 
1780 			/*
1781 			 * shmem_alloc_page() passes MPOL_F_SHARED policy with
1782 			 * a pseudo vma whose vma->vm_ops=NULL. Take a reference
1783 			 * count on these policies which will be dropped by
1784 			 * mpol_cond_put() later
1785 			 */
1786 			if (mpol_needs_cond_ref(pol))
1787 				mpol_get(pol);
1788 		}
1789 	}
1790 
1791 	return pol;
1792 }
1793 
1794 /*
1795  * get_vma_policy(@vma, @addr)
1796  * @vma: virtual memory area whose policy is sought
1797  * @addr: address in @vma for shared policy lookup
1798  *
1799  * Returns effective policy for a VMA at specified address.
1800  * Falls back to current->mempolicy or system default policy, as necessary.
1801  * Shared policies [those marked as MPOL_F_SHARED] require an extra reference
1802  * count--added by the get_policy() vm_op, as appropriate--to protect against
1803  * freeing by another task.  It is the caller's responsibility to free the
1804  * extra reference for shared policies.
1805  */
get_vma_policy(struct vm_area_struct * vma,unsigned long addr)1806 static struct mempolicy *get_vma_policy(struct vm_area_struct *vma,
1807 						unsigned long addr)
1808 {
1809 	struct mempolicy *pol = __get_vma_policy(vma, addr);
1810 
1811 	if (!pol)
1812 		pol = get_task_policy(current);
1813 
1814 	return pol;
1815 }
1816 
vma_policy_mof(struct vm_area_struct * vma)1817 bool vma_policy_mof(struct vm_area_struct *vma)
1818 {
1819 	struct mempolicy *pol;
1820 
1821 	if (vma->vm_ops && vma->vm_ops->get_policy) {
1822 		bool ret = false;
1823 
1824 		pol = vma->vm_ops->get_policy(vma, vma->vm_start);
1825 		if (pol && (pol->flags & MPOL_F_MOF))
1826 			ret = true;
1827 		mpol_cond_put(pol);
1828 
1829 		return ret;
1830 	}
1831 
1832 	pol = vma->vm_policy;
1833 	if (!pol)
1834 		pol = get_task_policy(current);
1835 
1836 	return pol->flags & MPOL_F_MOF;
1837 }
1838 
apply_policy_zone(struct mempolicy * policy,enum zone_type zone)1839 bool apply_policy_zone(struct mempolicy *policy, enum zone_type zone)
1840 {
1841 	enum zone_type dynamic_policy_zone = policy_zone;
1842 
1843 	BUG_ON(dynamic_policy_zone == ZONE_MOVABLE);
1844 
1845 	/*
1846 	 * if policy->nodes has movable memory only,
1847 	 * we apply policy when gfp_zone(gfp) = ZONE_MOVABLE only.
1848 	 *
1849 	 * policy->nodes is intersect with node_states[N_MEMORY].
1850 	 * so if the following test fails, it implies
1851 	 * policy->nodes has movable memory only.
1852 	 */
1853 	if (!nodes_intersects(policy->nodes, node_states[N_HIGH_MEMORY]))
1854 		dynamic_policy_zone = ZONE_MOVABLE;
1855 
1856 	return zone >= dynamic_policy_zone;
1857 }
1858 
1859 /*
1860  * Return a nodemask representing a mempolicy for filtering nodes for
1861  * page allocation
1862  */
policy_nodemask(gfp_t gfp,struct mempolicy * policy)1863 nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *policy)
1864 {
1865 	int mode = policy->mode;
1866 
1867 	/* Lower zones don't get a nodemask applied for MPOL_BIND */
1868 	if (unlikely(mode == MPOL_BIND) &&
1869 		apply_policy_zone(policy, gfp_zone(gfp)) &&
1870 		cpuset_nodemask_valid_mems_allowed(&policy->nodes))
1871 		return &policy->nodes;
1872 
1873 	if (mode == MPOL_PREFERRED_MANY)
1874 		return &policy->nodes;
1875 
1876 	return NULL;
1877 }
1878 
1879 /*
1880  * Return the  preferred node id for 'prefer' mempolicy, and return
1881  * the given id for all other policies.
1882  *
1883  * policy_node() is always coupled with policy_nodemask(), which
1884  * secures the nodemask limit for 'bind' and 'prefer-many' policy.
1885  */
policy_node(gfp_t gfp,struct mempolicy * policy,int nd)1886 static int policy_node(gfp_t gfp, struct mempolicy *policy, int nd)
1887 {
1888 	if (policy->mode == MPOL_PREFERRED) {
1889 		nd = first_node(policy->nodes);
1890 	} else {
1891 		/*
1892 		 * __GFP_THISNODE shouldn't even be used with the bind policy
1893 		 * because we might easily break the expectation to stay on the
1894 		 * requested node and not break the policy.
1895 		 */
1896 		WARN_ON_ONCE(policy->mode == MPOL_BIND && (gfp & __GFP_THISNODE));
1897 	}
1898 
1899 	if ((policy->mode == MPOL_BIND ||
1900 	     policy->mode == MPOL_PREFERRED_MANY) &&
1901 	    policy->home_node != NUMA_NO_NODE)
1902 		return policy->home_node;
1903 
1904 	return nd;
1905 }
1906 
1907 /* Do dynamic interleaving for a process */
interleave_nodes(struct mempolicy * policy)1908 static unsigned interleave_nodes(struct mempolicy *policy)
1909 {
1910 	unsigned next;
1911 	struct task_struct *me = current;
1912 
1913 	next = next_node_in(me->il_prev, policy->nodes);
1914 	if (next < MAX_NUMNODES)
1915 		me->il_prev = next;
1916 	return next;
1917 }
1918 
1919 /*
1920  * Depending on the memory policy provide a node from which to allocate the
1921  * next slab entry.
1922  */
mempolicy_slab_node(void)1923 unsigned int mempolicy_slab_node(void)
1924 {
1925 	struct mempolicy *policy;
1926 	int node = numa_mem_id();
1927 
1928 	if (!in_task())
1929 		return node;
1930 
1931 	policy = current->mempolicy;
1932 	if (!policy)
1933 		return node;
1934 
1935 	switch (policy->mode) {
1936 	case MPOL_PREFERRED:
1937 		return first_node(policy->nodes);
1938 
1939 	case MPOL_INTERLEAVE:
1940 		return interleave_nodes(policy);
1941 
1942 	case MPOL_BIND:
1943 	case MPOL_PREFERRED_MANY:
1944 	{
1945 		struct zoneref *z;
1946 
1947 		/*
1948 		 * Follow bind policy behavior and start allocation at the
1949 		 * first node.
1950 		 */
1951 		struct zonelist *zonelist;
1952 		enum zone_type highest_zoneidx = gfp_zone(GFP_KERNEL);
1953 		zonelist = &NODE_DATA(node)->node_zonelists[ZONELIST_FALLBACK];
1954 		z = first_zones_zonelist(zonelist, highest_zoneidx,
1955 							&policy->nodes);
1956 		return z->zone ? zone_to_nid(z->zone) : node;
1957 	}
1958 	case MPOL_LOCAL:
1959 		return node;
1960 
1961 	default:
1962 		BUG();
1963 	}
1964 }
1965 
1966 /*
1967  * Do static interleaving for a VMA with known offset @n.  Returns the n'th
1968  * node in pol->nodes (starting from n=0), wrapping around if n exceeds the
1969  * number of present nodes.
1970  */
offset_il_node(struct mempolicy * pol,unsigned long n)1971 static unsigned offset_il_node(struct mempolicy *pol, unsigned long n)
1972 {
1973 	nodemask_t nodemask = pol->nodes;
1974 	unsigned int target, nnodes;
1975 	int i;
1976 	int nid;
1977 	/*
1978 	 * The barrier will stabilize the nodemask in a register or on
1979 	 * the stack so that it will stop changing under the code.
1980 	 *
1981 	 * Between first_node() and next_node(), pol->nodes could be changed
1982 	 * by other threads. So we put pol->nodes in a local stack.
1983 	 */
1984 	barrier();
1985 
1986 	nnodes = nodes_weight(nodemask);
1987 	if (!nnodes)
1988 		return numa_node_id();
1989 	target = (unsigned int)n % nnodes;
1990 	nid = first_node(nodemask);
1991 	for (i = 0; i < target; i++)
1992 		nid = next_node(nid, nodemask);
1993 	return nid;
1994 }
1995 
1996 /* Determine a node number for interleave */
interleave_nid(struct mempolicy * pol,struct vm_area_struct * vma,unsigned long addr,int shift)1997 static inline unsigned interleave_nid(struct mempolicy *pol,
1998 		 struct vm_area_struct *vma, unsigned long addr, int shift)
1999 {
2000 	if (vma) {
2001 		unsigned long off;
2002 
2003 		/*
2004 		 * for small pages, there is no difference between
2005 		 * shift and PAGE_SHIFT, so the bit-shift is safe.
2006 		 * for huge pages, since vm_pgoff is in units of small
2007 		 * pages, we need to shift off the always 0 bits to get
2008 		 * a useful offset.
2009 		 */
2010 		BUG_ON(shift < PAGE_SHIFT);
2011 		off = vma->vm_pgoff >> (shift - PAGE_SHIFT);
2012 		off += (addr - vma->vm_start) >> shift;
2013 		return offset_il_node(pol, off);
2014 	} else
2015 		return interleave_nodes(pol);
2016 }
2017 
2018 #ifdef CONFIG_HUGETLBFS
2019 /*
2020  * huge_node(@vma, @addr, @gfp_flags, @mpol)
2021  * @vma: virtual memory area whose policy is sought
2022  * @addr: address in @vma for shared policy lookup and interleave policy
2023  * @gfp_flags: for requested zone
2024  * @mpol: pointer to mempolicy pointer for reference counted mempolicy
2025  * @nodemask: pointer to nodemask pointer for 'bind' and 'prefer-many' policy
2026  *
2027  * Returns a nid suitable for a huge page allocation and a pointer
2028  * to the struct mempolicy for conditional unref after allocation.
2029  * If the effective policy is 'bind' or 'prefer-many', returns a pointer
2030  * to the mempolicy's @nodemask for filtering the zonelist.
2031  *
2032  * Must be protected by read_mems_allowed_begin()
2033  */
huge_node(struct vm_area_struct * vma,unsigned long addr,gfp_t gfp_flags,struct mempolicy ** mpol,nodemask_t ** nodemask)2034 int huge_node(struct vm_area_struct *vma, unsigned long addr, gfp_t gfp_flags,
2035 				struct mempolicy **mpol, nodemask_t **nodemask)
2036 {
2037 	int nid;
2038 	int mode;
2039 
2040 	*mpol = get_vma_policy(vma, addr);
2041 	*nodemask = NULL;
2042 	mode = (*mpol)->mode;
2043 
2044 	if (unlikely(mode == MPOL_INTERLEAVE)) {
2045 		nid = interleave_nid(*mpol, vma, addr,
2046 					huge_page_shift(hstate_vma(vma)));
2047 	} else {
2048 		nid = policy_node(gfp_flags, *mpol, numa_node_id());
2049 		if (mode == MPOL_BIND || mode == MPOL_PREFERRED_MANY)
2050 			*nodemask = &(*mpol)->nodes;
2051 	}
2052 	return nid;
2053 }
2054 
2055 /*
2056  * init_nodemask_of_mempolicy
2057  *
2058  * If the current task's mempolicy is "default" [NULL], return 'false'
2059  * to indicate default policy.  Otherwise, extract the policy nodemask
2060  * for 'bind' or 'interleave' policy into the argument nodemask, or
2061  * initialize the argument nodemask to contain the single node for
2062  * 'preferred' or 'local' policy and return 'true' to indicate presence
2063  * of non-default mempolicy.
2064  *
2065  * We don't bother with reference counting the mempolicy [mpol_get/put]
2066  * because the current task is examining it's own mempolicy and a task's
2067  * mempolicy is only ever changed by the task itself.
2068  *
2069  * N.B., it is the caller's responsibility to free a returned nodemask.
2070  */
init_nodemask_of_mempolicy(nodemask_t * mask)2071 bool init_nodemask_of_mempolicy(nodemask_t *mask)
2072 {
2073 	struct mempolicy *mempolicy;
2074 
2075 	if (!(mask && current->mempolicy))
2076 		return false;
2077 
2078 	task_lock(current);
2079 	mempolicy = current->mempolicy;
2080 	switch (mempolicy->mode) {
2081 	case MPOL_PREFERRED:
2082 	case MPOL_PREFERRED_MANY:
2083 	case MPOL_BIND:
2084 	case MPOL_INTERLEAVE:
2085 		*mask = mempolicy->nodes;
2086 		break;
2087 
2088 	case MPOL_LOCAL:
2089 		init_nodemask_of_node(mask, numa_node_id());
2090 		break;
2091 
2092 	default:
2093 		BUG();
2094 	}
2095 	task_unlock(current);
2096 
2097 	return true;
2098 }
2099 #endif
2100 
2101 /*
2102  * mempolicy_in_oom_domain
2103  *
2104  * If tsk's mempolicy is "bind", check for intersection between mask and
2105  * the policy nodemask. Otherwise, return true for all other policies
2106  * including "interleave", as a tsk with "interleave" policy may have
2107  * memory allocated from all nodes in system.
2108  *
2109  * Takes task_lock(tsk) to prevent freeing of its mempolicy.
2110  */
mempolicy_in_oom_domain(struct task_struct * tsk,const nodemask_t * mask)2111 bool mempolicy_in_oom_domain(struct task_struct *tsk,
2112 					const nodemask_t *mask)
2113 {
2114 	struct mempolicy *mempolicy;
2115 	bool ret = true;
2116 
2117 	if (!mask)
2118 		return ret;
2119 
2120 	task_lock(tsk);
2121 	mempolicy = tsk->mempolicy;
2122 	if (mempolicy && mempolicy->mode == MPOL_BIND)
2123 		ret = nodes_intersects(mempolicy->nodes, *mask);
2124 	task_unlock(tsk);
2125 
2126 	return ret;
2127 }
2128 
2129 /* Allocate a page in interleaved policy.
2130    Own path because it needs to do special accounting. */
alloc_page_interleave(gfp_t gfp,unsigned order,unsigned nid)2131 static struct page *alloc_page_interleave(gfp_t gfp, unsigned order,
2132 					unsigned nid)
2133 {
2134 	struct page *page;
2135 
2136 	page = __alloc_pages(gfp, order, nid, NULL);
2137 	/* skip NUMA_INTERLEAVE_HIT counter update if numa stats is disabled */
2138 	if (!static_branch_likely(&vm_numa_stat_key))
2139 		return page;
2140 	if (page && page_to_nid(page) == nid) {
2141 		preempt_disable();
2142 		__count_numa_event(page_zone(page), NUMA_INTERLEAVE_HIT);
2143 		preempt_enable();
2144 	}
2145 	return page;
2146 }
2147 
alloc_pages_preferred_many(gfp_t gfp,unsigned int order,int nid,struct mempolicy * pol)2148 static struct page *alloc_pages_preferred_many(gfp_t gfp, unsigned int order,
2149 						int nid, struct mempolicy *pol)
2150 {
2151 	struct page *page;
2152 	gfp_t preferred_gfp;
2153 
2154 	/*
2155 	 * This is a two pass approach. The first pass will only try the
2156 	 * preferred nodes but skip the direct reclaim and allow the
2157 	 * allocation to fail, while the second pass will try all the
2158 	 * nodes in system.
2159 	 */
2160 	preferred_gfp = gfp | __GFP_NOWARN;
2161 	preferred_gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL);
2162 	page = __alloc_pages(preferred_gfp, order, nid, &pol->nodes);
2163 	if (!page)
2164 		page = __alloc_pages(gfp, order, nid, NULL);
2165 
2166 	return page;
2167 }
2168 
2169 /**
2170  * vma_alloc_folio - Allocate a folio for a VMA.
2171  * @gfp: GFP flags.
2172  * @order: Order of the folio.
2173  * @vma: Pointer to VMA or NULL if not available.
2174  * @addr: Virtual address of the allocation.  Must be inside @vma.
2175  * @hugepage: For hugepages try only the preferred node if possible.
2176  *
2177  * Allocate a folio for a specific address in @vma, using the appropriate
2178  * NUMA policy.  When @vma is not NULL the caller must hold the mmap_lock
2179  * of the mm_struct of the VMA to prevent it from going away.  Should be
2180  * used for all allocations for folios that will be mapped into user space.
2181  *
2182  * Return: The folio on success or NULL if allocation fails.
2183  */
vma_alloc_folio(gfp_t gfp,int order,struct vm_area_struct * vma,unsigned long addr,bool hugepage)2184 struct folio *vma_alloc_folio(gfp_t gfp, int order, struct vm_area_struct *vma,
2185 		unsigned long addr, bool hugepage)
2186 {
2187 	struct mempolicy *pol;
2188 	int node = numa_node_id();
2189 	struct folio *folio;
2190 	int preferred_nid;
2191 	nodemask_t *nmask;
2192 
2193 	pol = get_vma_policy(vma, addr);
2194 
2195 	if (pol->mode == MPOL_INTERLEAVE) {
2196 		struct page *page;
2197 		unsigned nid;
2198 
2199 		nid = interleave_nid(pol, vma, addr, PAGE_SHIFT + order);
2200 		mpol_cond_put(pol);
2201 		gfp |= __GFP_COMP;
2202 		page = alloc_page_interleave(gfp, order, nid);
2203 		folio = (struct folio *)page;
2204 		if (folio && order > 1)
2205 			folio_prep_large_rmappable(folio);
2206 		goto out;
2207 	}
2208 
2209 	if (pol->mode == MPOL_PREFERRED_MANY) {
2210 		struct page *page;
2211 
2212 		node = policy_node(gfp, pol, node);
2213 		gfp |= __GFP_COMP;
2214 		page = alloc_pages_preferred_many(gfp, order, node, pol);
2215 		mpol_cond_put(pol);
2216 		folio = (struct folio *)page;
2217 		if (folio && order > 1)
2218 			folio_prep_large_rmappable(folio);
2219 		goto out;
2220 	}
2221 
2222 	if (unlikely(IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && hugepage)) {
2223 		int hpage_node = node;
2224 
2225 		/*
2226 		 * For hugepage allocation and non-interleave policy which
2227 		 * allows the current node (or other explicitly preferred
2228 		 * node) we only try to allocate from the current/preferred
2229 		 * node and don't fall back to other nodes, as the cost of
2230 		 * remote accesses would likely offset THP benefits.
2231 		 *
2232 		 * If the policy is interleave or does not allow the current
2233 		 * node in its nodemask, we allocate the standard way.
2234 		 */
2235 		if (pol->mode == MPOL_PREFERRED)
2236 			hpage_node = first_node(pol->nodes);
2237 
2238 		nmask = policy_nodemask(gfp, pol);
2239 		if (!nmask || node_isset(hpage_node, *nmask)) {
2240 			mpol_cond_put(pol);
2241 			/*
2242 			 * First, try to allocate THP only on local node, but
2243 			 * don't reclaim unnecessarily, just compact.
2244 			 */
2245 			folio = __folio_alloc_node(gfp | __GFP_THISNODE |
2246 					__GFP_NORETRY, order, hpage_node);
2247 
2248 			/*
2249 			 * If hugepage allocations are configured to always
2250 			 * synchronous compact or the vma has been madvised
2251 			 * to prefer hugepage backing, retry allowing remote
2252 			 * memory with both reclaim and compact as well.
2253 			 */
2254 			if (!folio && (gfp & __GFP_DIRECT_RECLAIM))
2255 				folio = __folio_alloc(gfp, order, hpage_node,
2256 						      nmask);
2257 
2258 			goto out;
2259 		}
2260 	}
2261 
2262 	nmask = policy_nodemask(gfp, pol);
2263 	preferred_nid = policy_node(gfp, pol, node);
2264 	folio = __folio_alloc(gfp, order, preferred_nid, nmask);
2265 	mpol_cond_put(pol);
2266 out:
2267 	return folio;
2268 }
2269 EXPORT_SYMBOL(vma_alloc_folio);
2270 
2271 /**
2272  * alloc_pages - Allocate pages.
2273  * @gfp: GFP flags.
2274  * @order: Power of two of number of pages to allocate.
2275  *
2276  * Allocate 1 << @order contiguous pages.  The physical address of the
2277  * first page is naturally aligned (eg an order-3 allocation will be aligned
2278  * to a multiple of 8 * PAGE_SIZE bytes).  The NUMA policy of the current
2279  * process is honoured when in process context.
2280  *
2281  * Context: Can be called from any context, providing the appropriate GFP
2282  * flags are used.
2283  * Return: The page on success or NULL if allocation fails.
2284  */
alloc_pages(gfp_t gfp,unsigned order)2285 struct page *alloc_pages(gfp_t gfp, unsigned order)
2286 {
2287 	struct mempolicy *pol = &default_policy;
2288 	struct page *page;
2289 
2290 	if (!in_interrupt() && !(gfp & __GFP_THISNODE))
2291 		pol = get_task_policy(current);
2292 
2293 	/*
2294 	 * No reference counting needed for current->mempolicy
2295 	 * nor system default_policy
2296 	 */
2297 	if (pol->mode == MPOL_INTERLEAVE)
2298 		page = alloc_page_interleave(gfp, order, interleave_nodes(pol));
2299 	else if (pol->mode == MPOL_PREFERRED_MANY)
2300 		page = alloc_pages_preferred_many(gfp, order,
2301 				  policy_node(gfp, pol, numa_node_id()), pol);
2302 	else
2303 		page = __alloc_pages(gfp, order,
2304 				policy_node(gfp, pol, numa_node_id()),
2305 				policy_nodemask(gfp, pol));
2306 
2307 	return page;
2308 }
2309 EXPORT_SYMBOL(alloc_pages);
2310 
folio_alloc(gfp_t gfp,unsigned order)2311 struct folio *folio_alloc(gfp_t gfp, unsigned order)
2312 {
2313 	struct page *page = alloc_pages(gfp | __GFP_COMP, order);
2314 	struct folio *folio = (struct folio *)page;
2315 
2316 	if (folio && order > 1)
2317 		folio_prep_large_rmappable(folio);
2318 	return folio;
2319 }
2320 EXPORT_SYMBOL(folio_alloc);
2321 
alloc_pages_bulk_array_interleave(gfp_t gfp,struct mempolicy * pol,unsigned long nr_pages,struct page ** page_array)2322 static unsigned long alloc_pages_bulk_array_interleave(gfp_t gfp,
2323 		struct mempolicy *pol, unsigned long nr_pages,
2324 		struct page **page_array)
2325 {
2326 	int nodes;
2327 	unsigned long nr_pages_per_node;
2328 	int delta;
2329 	int i;
2330 	unsigned long nr_allocated;
2331 	unsigned long total_allocated = 0;
2332 
2333 	nodes = nodes_weight(pol->nodes);
2334 	nr_pages_per_node = nr_pages / nodes;
2335 	delta = nr_pages - nodes * nr_pages_per_node;
2336 
2337 	for (i = 0; i < nodes; i++) {
2338 		if (delta) {
2339 			nr_allocated = __alloc_pages_bulk(gfp,
2340 					interleave_nodes(pol), NULL,
2341 					nr_pages_per_node + 1, NULL,
2342 					page_array);
2343 			delta--;
2344 		} else {
2345 			nr_allocated = __alloc_pages_bulk(gfp,
2346 					interleave_nodes(pol), NULL,
2347 					nr_pages_per_node, NULL, page_array);
2348 		}
2349 
2350 		page_array += nr_allocated;
2351 		total_allocated += nr_allocated;
2352 	}
2353 
2354 	return total_allocated;
2355 }
2356 
alloc_pages_bulk_array_preferred_many(gfp_t gfp,int nid,struct mempolicy * pol,unsigned long nr_pages,struct page ** page_array)2357 static unsigned long alloc_pages_bulk_array_preferred_many(gfp_t gfp, int nid,
2358 		struct mempolicy *pol, unsigned long nr_pages,
2359 		struct page **page_array)
2360 {
2361 	gfp_t preferred_gfp;
2362 	unsigned long nr_allocated = 0;
2363 
2364 	preferred_gfp = gfp | __GFP_NOWARN;
2365 	preferred_gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL);
2366 
2367 	nr_allocated  = __alloc_pages_bulk(preferred_gfp, nid, &pol->nodes,
2368 					   nr_pages, NULL, page_array);
2369 
2370 	if (nr_allocated < nr_pages)
2371 		nr_allocated += __alloc_pages_bulk(gfp, numa_node_id(), NULL,
2372 				nr_pages - nr_allocated, NULL,
2373 				page_array + nr_allocated);
2374 	return nr_allocated;
2375 }
2376 
2377 /* alloc pages bulk and mempolicy should be considered at the
2378  * same time in some situation such as vmalloc.
2379  *
2380  * It can accelerate memory allocation especially interleaving
2381  * allocate memory.
2382  */
alloc_pages_bulk_array_mempolicy(gfp_t gfp,unsigned long nr_pages,struct page ** page_array)2383 unsigned long alloc_pages_bulk_array_mempolicy(gfp_t gfp,
2384 		unsigned long nr_pages, struct page **page_array)
2385 {
2386 	struct mempolicy *pol = &default_policy;
2387 
2388 	if (!in_interrupt() && !(gfp & __GFP_THISNODE))
2389 		pol = get_task_policy(current);
2390 
2391 	if (pol->mode == MPOL_INTERLEAVE)
2392 		return alloc_pages_bulk_array_interleave(gfp, pol,
2393 							 nr_pages, page_array);
2394 
2395 	if (pol->mode == MPOL_PREFERRED_MANY)
2396 		return alloc_pages_bulk_array_preferred_many(gfp,
2397 				numa_node_id(), pol, nr_pages, page_array);
2398 
2399 	return __alloc_pages_bulk(gfp, policy_node(gfp, pol, numa_node_id()),
2400 				  policy_nodemask(gfp, pol), nr_pages, NULL,
2401 				  page_array);
2402 }
2403 
vma_dup_policy(struct vm_area_struct * src,struct vm_area_struct * dst)2404 int vma_dup_policy(struct vm_area_struct *src, struct vm_area_struct *dst)
2405 {
2406 	struct mempolicy *pol = mpol_dup(vma_policy(src));
2407 
2408 	if (IS_ERR(pol))
2409 		return PTR_ERR(pol);
2410 	dst->vm_policy = pol;
2411 	return 0;
2412 }
2413 
2414 /*
2415  * If mpol_dup() sees current->cpuset == cpuset_being_rebound, then it
2416  * rebinds the mempolicy its copying by calling mpol_rebind_policy()
2417  * with the mems_allowed returned by cpuset_mems_allowed().  This
2418  * keeps mempolicies cpuset relative after its cpuset moves.  See
2419  * further kernel/cpuset.c update_nodemask().
2420  *
2421  * current's mempolicy may be rebinded by the other task(the task that changes
2422  * cpuset's mems), so we needn't do rebind work for current task.
2423  */
2424 
2425 /* Slow path of a mempolicy duplicate */
__mpol_dup(struct mempolicy * old)2426 struct mempolicy *__mpol_dup(struct mempolicy *old)
2427 {
2428 	struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
2429 
2430 	if (!new)
2431 		return ERR_PTR(-ENOMEM);
2432 
2433 	/* task's mempolicy is protected by alloc_lock */
2434 	if (old == current->mempolicy) {
2435 		task_lock(current);
2436 		*new = *old;
2437 		task_unlock(current);
2438 	} else
2439 		*new = *old;
2440 
2441 	if (current_cpuset_is_being_rebound()) {
2442 		nodemask_t mems = cpuset_mems_allowed(current);
2443 		mpol_rebind_policy(new, &mems);
2444 	}
2445 	atomic_set(&new->refcnt, 1);
2446 	return new;
2447 }
2448 
2449 /* Slow path of a mempolicy comparison */
__mpol_equal(struct mempolicy * a,struct mempolicy * b)2450 bool __mpol_equal(struct mempolicy *a, struct mempolicy *b)
2451 {
2452 	if (!a || !b)
2453 		return false;
2454 	if (a->mode != b->mode)
2455 		return false;
2456 	if (a->flags != b->flags)
2457 		return false;
2458 	if (a->home_node != b->home_node)
2459 		return false;
2460 	if (mpol_store_user_nodemask(a))
2461 		if (!nodes_equal(a->w.user_nodemask, b->w.user_nodemask))
2462 			return false;
2463 
2464 	switch (a->mode) {
2465 	case MPOL_BIND:
2466 	case MPOL_INTERLEAVE:
2467 	case MPOL_PREFERRED:
2468 	case MPOL_PREFERRED_MANY:
2469 		return !!nodes_equal(a->nodes, b->nodes);
2470 	case MPOL_LOCAL:
2471 		return true;
2472 	default:
2473 		BUG();
2474 		return false;
2475 	}
2476 }
2477 
2478 /*
2479  * Shared memory backing store policy support.
2480  *
2481  * Remember policies even when nobody has shared memory mapped.
2482  * The policies are kept in Red-Black tree linked from the inode.
2483  * They are protected by the sp->lock rwlock, which should be held
2484  * for any accesses to the tree.
2485  */
2486 
2487 /*
2488  * lookup first element intersecting start-end.  Caller holds sp->lock for
2489  * reading or for writing
2490  */
2491 static struct sp_node *
sp_lookup(struct shared_policy * sp,unsigned long start,unsigned long end)2492 sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end)
2493 {
2494 	struct rb_node *n = sp->root.rb_node;
2495 
2496 	while (n) {
2497 		struct sp_node *p = rb_entry(n, struct sp_node, nd);
2498 
2499 		if (start >= p->end)
2500 			n = n->rb_right;
2501 		else if (end <= p->start)
2502 			n = n->rb_left;
2503 		else
2504 			break;
2505 	}
2506 	if (!n)
2507 		return NULL;
2508 	for (;;) {
2509 		struct sp_node *w = NULL;
2510 		struct rb_node *prev = rb_prev(n);
2511 		if (!prev)
2512 			break;
2513 		w = rb_entry(prev, struct sp_node, nd);
2514 		if (w->end <= start)
2515 			break;
2516 		n = prev;
2517 	}
2518 	return rb_entry(n, struct sp_node, nd);
2519 }
2520 
2521 /*
2522  * Insert a new shared policy into the list.  Caller holds sp->lock for
2523  * writing.
2524  */
sp_insert(struct shared_policy * sp,struct sp_node * new)2525 static void sp_insert(struct shared_policy *sp, struct sp_node *new)
2526 {
2527 	struct rb_node **p = &sp->root.rb_node;
2528 	struct rb_node *parent = NULL;
2529 	struct sp_node *nd;
2530 
2531 	while (*p) {
2532 		parent = *p;
2533 		nd = rb_entry(parent, struct sp_node, nd);
2534 		if (new->start < nd->start)
2535 			p = &(*p)->rb_left;
2536 		else if (new->end > nd->end)
2537 			p = &(*p)->rb_right;
2538 		else
2539 			BUG();
2540 	}
2541 	rb_link_node(&new->nd, parent, p);
2542 	rb_insert_color(&new->nd, &sp->root);
2543 	pr_debug("inserting %lx-%lx: %d\n", new->start, new->end,
2544 		 new->policy ? new->policy->mode : 0);
2545 }
2546 
2547 /* Find shared policy intersecting idx */
2548 struct mempolicy *
mpol_shared_policy_lookup(struct shared_policy * sp,unsigned long idx)2549 mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx)
2550 {
2551 	struct mempolicy *pol = NULL;
2552 	struct sp_node *sn;
2553 
2554 	if (!sp->root.rb_node)
2555 		return NULL;
2556 	read_lock(&sp->lock);
2557 	sn = sp_lookup(sp, idx, idx+1);
2558 	if (sn) {
2559 		mpol_get(sn->policy);
2560 		pol = sn->policy;
2561 	}
2562 	read_unlock(&sp->lock);
2563 	return pol;
2564 }
2565 
sp_free(struct sp_node * n)2566 static void sp_free(struct sp_node *n)
2567 {
2568 	mpol_put(n->policy);
2569 	kmem_cache_free(sn_cache, n);
2570 }
2571 
2572 /**
2573  * mpol_misplaced - check whether current page node is valid in policy
2574  *
2575  * @page: page to be checked
2576  * @vma: vm area where page mapped
2577  * @addr: virtual address where page mapped
2578  *
2579  * Lookup current policy node id for vma,addr and "compare to" page's
2580  * node id.  Policy determination "mimics" alloc_page_vma().
2581  * Called from fault path where we know the vma and faulting address.
2582  *
2583  * Return: NUMA_NO_NODE if the page is in a node that is valid for this
2584  * policy, or a suitable node ID to allocate a replacement page from.
2585  */
mpol_misplaced(struct page * page,struct vm_area_struct * vma,unsigned long addr)2586 int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long addr)
2587 {
2588 	struct mempolicy *pol;
2589 	struct zoneref *z;
2590 	int curnid = page_to_nid(page);
2591 	unsigned long pgoff;
2592 	int thiscpu = raw_smp_processor_id();
2593 	int thisnid = cpu_to_node(thiscpu);
2594 	int polnid = NUMA_NO_NODE;
2595 	int ret = NUMA_NO_NODE;
2596 
2597 	pol = get_vma_policy(vma, addr);
2598 	if (!(pol->flags & MPOL_F_MOF))
2599 		goto out;
2600 
2601 	switch (pol->mode) {
2602 	case MPOL_INTERLEAVE:
2603 		pgoff = vma->vm_pgoff;
2604 		pgoff += (addr - vma->vm_start) >> PAGE_SHIFT;
2605 		polnid = offset_il_node(pol, pgoff);
2606 		break;
2607 
2608 	case MPOL_PREFERRED:
2609 		if (node_isset(curnid, pol->nodes))
2610 			goto out;
2611 		polnid = first_node(pol->nodes);
2612 		break;
2613 
2614 	case MPOL_LOCAL:
2615 		polnid = numa_node_id();
2616 		break;
2617 
2618 	case MPOL_BIND:
2619 		/* Optimize placement among multiple nodes via NUMA balancing */
2620 		if (pol->flags & MPOL_F_MORON) {
2621 			if (node_isset(thisnid, pol->nodes))
2622 				break;
2623 			goto out;
2624 		}
2625 		fallthrough;
2626 
2627 	case MPOL_PREFERRED_MANY:
2628 		/*
2629 		 * use current page if in policy nodemask,
2630 		 * else select nearest allowed node, if any.
2631 		 * If no allowed nodes, use current [!misplaced].
2632 		 */
2633 		if (node_isset(curnid, pol->nodes))
2634 			goto out;
2635 		z = first_zones_zonelist(
2636 				node_zonelist(numa_node_id(), GFP_HIGHUSER),
2637 				gfp_zone(GFP_HIGHUSER),
2638 				&pol->nodes);
2639 		polnid = zone_to_nid(z->zone);
2640 		break;
2641 
2642 	default:
2643 		BUG();
2644 	}
2645 
2646 	/* Migrate the page towards the node whose CPU is referencing it */
2647 	if (pol->flags & MPOL_F_MORON) {
2648 		polnid = thisnid;
2649 
2650 		if (!should_numa_migrate_memory(current, page, curnid, thiscpu))
2651 			goto out;
2652 	}
2653 
2654 	if (curnid != polnid)
2655 		ret = polnid;
2656 out:
2657 	mpol_cond_put(pol);
2658 
2659 	return ret;
2660 }
2661 
2662 /*
2663  * Drop the (possibly final) reference to task->mempolicy.  It needs to be
2664  * dropped after task->mempolicy is set to NULL so that any allocation done as
2665  * part of its kmem_cache_free(), such as by KASAN, doesn't reference a freed
2666  * policy.
2667  */
mpol_put_task_policy(struct task_struct * task)2668 void mpol_put_task_policy(struct task_struct *task)
2669 {
2670 	struct mempolicy *pol;
2671 
2672 	task_lock(task);
2673 	pol = task->mempolicy;
2674 	task->mempolicy = NULL;
2675 	task_unlock(task);
2676 	mpol_put(pol);
2677 }
2678 
sp_delete(struct shared_policy * sp,struct sp_node * n)2679 static void sp_delete(struct shared_policy *sp, struct sp_node *n)
2680 {
2681 	pr_debug("deleting %lx-l%lx\n", n->start, n->end);
2682 	rb_erase(&n->nd, &sp->root);
2683 	sp_free(n);
2684 }
2685 
sp_node_init(struct sp_node * node,unsigned long start,unsigned long end,struct mempolicy * pol)2686 static void sp_node_init(struct sp_node *node, unsigned long start,
2687 			unsigned long end, struct mempolicy *pol)
2688 {
2689 	node->start = start;
2690 	node->end = end;
2691 	node->policy = pol;
2692 }
2693 
sp_alloc(unsigned long start,unsigned long end,struct mempolicy * pol)2694 static struct sp_node *sp_alloc(unsigned long start, unsigned long end,
2695 				struct mempolicy *pol)
2696 {
2697 	struct sp_node *n;
2698 	struct mempolicy *newpol;
2699 
2700 	n = kmem_cache_alloc(sn_cache, GFP_KERNEL);
2701 	if (!n)
2702 		return NULL;
2703 
2704 	newpol = mpol_dup(pol);
2705 	if (IS_ERR(newpol)) {
2706 		kmem_cache_free(sn_cache, n);
2707 		return NULL;
2708 	}
2709 	newpol->flags |= MPOL_F_SHARED;
2710 	sp_node_init(n, start, end, newpol);
2711 
2712 	return n;
2713 }
2714 
2715 /* Replace a policy range. */
shared_policy_replace(struct shared_policy * sp,unsigned long start,unsigned long end,struct sp_node * new)2716 static int shared_policy_replace(struct shared_policy *sp, unsigned long start,
2717 				 unsigned long end, struct sp_node *new)
2718 {
2719 	struct sp_node *n;
2720 	struct sp_node *n_new = NULL;
2721 	struct mempolicy *mpol_new = NULL;
2722 	int ret = 0;
2723 
2724 restart:
2725 	write_lock(&sp->lock);
2726 	n = sp_lookup(sp, start, end);
2727 	/* Take care of old policies in the same range. */
2728 	while (n && n->start < end) {
2729 		struct rb_node *next = rb_next(&n->nd);
2730 		if (n->start >= start) {
2731 			if (n->end <= end)
2732 				sp_delete(sp, n);
2733 			else
2734 				n->start = end;
2735 		} else {
2736 			/* Old policy spanning whole new range. */
2737 			if (n->end > end) {
2738 				if (!n_new)
2739 					goto alloc_new;
2740 
2741 				*mpol_new = *n->policy;
2742 				atomic_set(&mpol_new->refcnt, 1);
2743 				sp_node_init(n_new, end, n->end, mpol_new);
2744 				n->end = start;
2745 				sp_insert(sp, n_new);
2746 				n_new = NULL;
2747 				mpol_new = NULL;
2748 				break;
2749 			} else
2750 				n->end = start;
2751 		}
2752 		if (!next)
2753 			break;
2754 		n = rb_entry(next, struct sp_node, nd);
2755 	}
2756 	if (new)
2757 		sp_insert(sp, new);
2758 	write_unlock(&sp->lock);
2759 	ret = 0;
2760 
2761 err_out:
2762 	if (mpol_new)
2763 		mpol_put(mpol_new);
2764 	if (n_new)
2765 		kmem_cache_free(sn_cache, n_new);
2766 
2767 	return ret;
2768 
2769 alloc_new:
2770 	write_unlock(&sp->lock);
2771 	ret = -ENOMEM;
2772 	n_new = kmem_cache_alloc(sn_cache, GFP_KERNEL);
2773 	if (!n_new)
2774 		goto err_out;
2775 	mpol_new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
2776 	if (!mpol_new)
2777 		goto err_out;
2778 	atomic_set(&mpol_new->refcnt, 1);
2779 	goto restart;
2780 }
2781 
2782 /**
2783  * mpol_shared_policy_init - initialize shared policy for inode
2784  * @sp: pointer to inode shared policy
2785  * @mpol:  struct mempolicy to install
2786  *
2787  * Install non-NULL @mpol in inode's shared policy rb-tree.
2788  * On entry, the current task has a reference on a non-NULL @mpol.
2789  * This must be released on exit.
2790  * This is called at get_inode() calls and we can use GFP_KERNEL.
2791  */
mpol_shared_policy_init(struct shared_policy * sp,struct mempolicy * mpol)2792 void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol)
2793 {
2794 	int ret;
2795 
2796 	sp->root = RB_ROOT;		/* empty tree == default mempolicy */
2797 	rwlock_init(&sp->lock);
2798 
2799 	if (mpol) {
2800 		struct vm_area_struct pvma;
2801 		struct mempolicy *new;
2802 		NODEMASK_SCRATCH(scratch);
2803 
2804 		if (!scratch)
2805 			goto put_mpol;
2806 		/* contextualize the tmpfs mount point mempolicy */
2807 		new = mpol_new(mpol->mode, mpol->flags, &mpol->w.user_nodemask);
2808 		if (IS_ERR(new))
2809 			goto free_scratch; /* no valid nodemask intersection */
2810 
2811 		task_lock(current);
2812 		ret = mpol_set_nodemask(new, &mpol->w.user_nodemask, scratch);
2813 		task_unlock(current);
2814 		if (ret)
2815 			goto put_new;
2816 
2817 		/* Create pseudo-vma that contains just the policy */
2818 		vma_init(&pvma, NULL);
2819 		pvma.vm_end = TASK_SIZE;	/* policy covers entire file */
2820 		mpol_set_shared_policy(sp, &pvma, new); /* adds ref */
2821 
2822 put_new:
2823 		mpol_put(new);			/* drop initial ref */
2824 free_scratch:
2825 		NODEMASK_SCRATCH_FREE(scratch);
2826 put_mpol:
2827 		mpol_put(mpol);	/* drop our incoming ref on sb mpol */
2828 	}
2829 }
2830 
mpol_set_shared_policy(struct shared_policy * info,struct vm_area_struct * vma,struct mempolicy * npol)2831 int mpol_set_shared_policy(struct shared_policy *info,
2832 			struct vm_area_struct *vma, struct mempolicy *npol)
2833 {
2834 	int err;
2835 	struct sp_node *new = NULL;
2836 	unsigned long sz = vma_pages(vma);
2837 
2838 	pr_debug("set_shared_policy %lx sz %lu %d %d %lx\n",
2839 		 vma->vm_pgoff,
2840 		 sz, npol ? npol->mode : -1,
2841 		 npol ? npol->flags : -1,
2842 		 npol ? nodes_addr(npol->nodes)[0] : NUMA_NO_NODE);
2843 
2844 	if (npol) {
2845 		new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, npol);
2846 		if (!new)
2847 			return -ENOMEM;
2848 	}
2849 	err = shared_policy_replace(info, vma->vm_pgoff, vma->vm_pgoff+sz, new);
2850 	if (err && new)
2851 		sp_free(new);
2852 	return err;
2853 }
2854 
2855 /* Free a backing policy store on inode delete. */
mpol_free_shared_policy(struct shared_policy * p)2856 void mpol_free_shared_policy(struct shared_policy *p)
2857 {
2858 	struct sp_node *n;
2859 	struct rb_node *next;
2860 
2861 	if (!p->root.rb_node)
2862 		return;
2863 	write_lock(&p->lock);
2864 	next = rb_first(&p->root);
2865 	while (next) {
2866 		n = rb_entry(next, struct sp_node, nd);
2867 		next = rb_next(&n->nd);
2868 		sp_delete(p, n);
2869 	}
2870 	write_unlock(&p->lock);
2871 }
2872 
2873 #ifdef CONFIG_NUMA_BALANCING
2874 static int __initdata numabalancing_override;
2875 
check_numabalancing_enable(void)2876 static void __init check_numabalancing_enable(void)
2877 {
2878 	bool numabalancing_default = false;
2879 
2880 	if (IS_ENABLED(CONFIG_NUMA_BALANCING_DEFAULT_ENABLED))
2881 		numabalancing_default = true;
2882 
2883 	/* Parsed by setup_numabalancing. override == 1 enables, -1 disables */
2884 	if (numabalancing_override)
2885 		set_numabalancing_state(numabalancing_override == 1);
2886 
2887 	if (num_online_nodes() > 1 && !numabalancing_override) {
2888 		pr_info("%s automatic NUMA balancing. Configure with numa_balancing= or the kernel.numa_balancing sysctl\n",
2889 			numabalancing_default ? "Enabling" : "Disabling");
2890 		set_numabalancing_state(numabalancing_default);
2891 	}
2892 }
2893 
setup_numabalancing(char * str)2894 static int __init setup_numabalancing(char *str)
2895 {
2896 	int ret = 0;
2897 	if (!str)
2898 		goto out;
2899 
2900 	if (!strcmp(str, "enable")) {
2901 		numabalancing_override = 1;
2902 		ret = 1;
2903 	} else if (!strcmp(str, "disable")) {
2904 		numabalancing_override = -1;
2905 		ret = 1;
2906 	}
2907 out:
2908 	if (!ret)
2909 		pr_warn("Unable to parse numa_balancing=\n");
2910 
2911 	return ret;
2912 }
2913 __setup("numa_balancing=", setup_numabalancing);
2914 #else
check_numabalancing_enable(void)2915 static inline void __init check_numabalancing_enable(void)
2916 {
2917 }
2918 #endif /* CONFIG_NUMA_BALANCING */
2919 
2920 /* assumes fs == KERNEL_DS */
numa_policy_init(void)2921 void __init numa_policy_init(void)
2922 {
2923 	nodemask_t interleave_nodes;
2924 	unsigned long largest = 0;
2925 	int nid, prefer = 0;
2926 
2927 	policy_cache = kmem_cache_create("numa_policy",
2928 					 sizeof(struct mempolicy),
2929 					 0, SLAB_PANIC, NULL);
2930 
2931 	sn_cache = kmem_cache_create("shared_policy_node",
2932 				     sizeof(struct sp_node),
2933 				     0, SLAB_PANIC, NULL);
2934 
2935 	for_each_node(nid) {
2936 		preferred_node_policy[nid] = (struct mempolicy) {
2937 			.refcnt = ATOMIC_INIT(1),
2938 			.mode = MPOL_PREFERRED,
2939 			.flags = MPOL_F_MOF | MPOL_F_MORON,
2940 			.nodes = nodemask_of_node(nid),
2941 		};
2942 	}
2943 
2944 	/*
2945 	 * Set interleaving policy for system init. Interleaving is only
2946 	 * enabled across suitably sized nodes (default is >= 16MB), or
2947 	 * fall back to the largest node if they're all smaller.
2948 	 */
2949 	nodes_clear(interleave_nodes);
2950 	for_each_node_state(nid, N_MEMORY) {
2951 		unsigned long total_pages = node_present_pages(nid);
2952 
2953 		/* Preserve the largest node */
2954 		if (largest < total_pages) {
2955 			largest = total_pages;
2956 			prefer = nid;
2957 		}
2958 
2959 		/* Interleave this node? */
2960 		if ((total_pages << PAGE_SHIFT) >= (16 << 20))
2961 			node_set(nid, interleave_nodes);
2962 	}
2963 
2964 	/* All too small, use the largest */
2965 	if (unlikely(nodes_empty(interleave_nodes)))
2966 		node_set(prefer, interleave_nodes);
2967 
2968 	if (do_set_mempolicy(MPOL_INTERLEAVE, 0, &interleave_nodes))
2969 		pr_err("%s: interleaving failed\n", __func__);
2970 
2971 	check_numabalancing_enable();
2972 }
2973 
2974 /* Reset policy of current process to default */
numa_default_policy(void)2975 void numa_default_policy(void)
2976 {
2977 	do_set_mempolicy(MPOL_DEFAULT, 0, NULL);
2978 }
2979 
2980 /*
2981  * Parse and format mempolicy from/to strings
2982  */
2983 
2984 static const char * const policy_modes[] =
2985 {
2986 	[MPOL_DEFAULT]    = "default",
2987 	[MPOL_PREFERRED]  = "prefer",
2988 	[MPOL_BIND]       = "bind",
2989 	[MPOL_INTERLEAVE] = "interleave",
2990 	[MPOL_LOCAL]      = "local",
2991 	[MPOL_PREFERRED_MANY]  = "prefer (many)",
2992 };
2993 
2994 
2995 #ifdef CONFIG_TMPFS
2996 /**
2997  * mpol_parse_str - parse string to mempolicy, for tmpfs mpol mount option.
2998  * @str:  string containing mempolicy to parse
2999  * @mpol:  pointer to struct mempolicy pointer, returned on success.
3000  *
3001  * Format of input:
3002  *	<mode>[=<flags>][:<nodelist>]
3003  *
3004  * Return: %0 on success, else %1
3005  */
mpol_parse_str(char * str,struct mempolicy ** mpol)3006 int mpol_parse_str(char *str, struct mempolicy **mpol)
3007 {
3008 	struct mempolicy *new = NULL;
3009 	unsigned short mode_flags;
3010 	nodemask_t nodes;
3011 	char *nodelist = strchr(str, ':');
3012 	char *flags = strchr(str, '=');
3013 	int err = 1, mode;
3014 
3015 	if (flags)
3016 		*flags++ = '\0';	/* terminate mode string */
3017 
3018 	if (nodelist) {
3019 		/* NUL-terminate mode or flags string */
3020 		*nodelist++ = '\0';
3021 		if (nodelist_parse(nodelist, nodes))
3022 			goto out;
3023 		if (!nodes_subset(nodes, node_states[N_MEMORY]))
3024 			goto out;
3025 	} else
3026 		nodes_clear(nodes);
3027 
3028 	mode = match_string(policy_modes, MPOL_MAX, str);
3029 	if (mode < 0)
3030 		goto out;
3031 
3032 	switch (mode) {
3033 	case MPOL_PREFERRED:
3034 		/*
3035 		 * Insist on a nodelist of one node only, although later
3036 		 * we use first_node(nodes) to grab a single node, so here
3037 		 * nodelist (or nodes) cannot be empty.
3038 		 */
3039 		if (nodelist) {
3040 			char *rest = nodelist;
3041 			while (isdigit(*rest))
3042 				rest++;
3043 			if (*rest)
3044 				goto out;
3045 			if (nodes_empty(nodes))
3046 				goto out;
3047 		}
3048 		break;
3049 	case MPOL_INTERLEAVE:
3050 		/*
3051 		 * Default to online nodes with memory if no nodelist
3052 		 */
3053 		if (!nodelist)
3054 			nodes = node_states[N_MEMORY];
3055 		break;
3056 	case MPOL_LOCAL:
3057 		/*
3058 		 * Don't allow a nodelist;  mpol_new() checks flags
3059 		 */
3060 		if (nodelist)
3061 			goto out;
3062 		break;
3063 	case MPOL_DEFAULT:
3064 		/*
3065 		 * Insist on a empty nodelist
3066 		 */
3067 		if (!nodelist)
3068 			err = 0;
3069 		goto out;
3070 	case MPOL_PREFERRED_MANY:
3071 	case MPOL_BIND:
3072 		/*
3073 		 * Insist on a nodelist
3074 		 */
3075 		if (!nodelist)
3076 			goto out;
3077 	}
3078 
3079 	mode_flags = 0;
3080 	if (flags) {
3081 		/*
3082 		 * Currently, we only support two mutually exclusive
3083 		 * mode flags.
3084 		 */
3085 		if (!strcmp(flags, "static"))
3086 			mode_flags |= MPOL_F_STATIC_NODES;
3087 		else if (!strcmp(flags, "relative"))
3088 			mode_flags |= MPOL_F_RELATIVE_NODES;
3089 		else
3090 			goto out;
3091 	}
3092 
3093 	new = mpol_new(mode, mode_flags, &nodes);
3094 	if (IS_ERR(new))
3095 		goto out;
3096 
3097 	/*
3098 	 * Save nodes for mpol_to_str() to show the tmpfs mount options
3099 	 * for /proc/mounts, /proc/pid/mounts and /proc/pid/mountinfo.
3100 	 */
3101 	if (mode != MPOL_PREFERRED) {
3102 		new->nodes = nodes;
3103 	} else if (nodelist) {
3104 		nodes_clear(new->nodes);
3105 		node_set(first_node(nodes), new->nodes);
3106 	} else {
3107 		new->mode = MPOL_LOCAL;
3108 	}
3109 
3110 	/*
3111 	 * Save nodes for contextualization: this will be used to "clone"
3112 	 * the mempolicy in a specific context [cpuset] at a later time.
3113 	 */
3114 	new->w.user_nodemask = nodes;
3115 
3116 	err = 0;
3117 
3118 out:
3119 	/* Restore string for error message */
3120 	if (nodelist)
3121 		*--nodelist = ':';
3122 	if (flags)
3123 		*--flags = '=';
3124 	if (!err)
3125 		*mpol = new;
3126 	return err;
3127 }
3128 #endif /* CONFIG_TMPFS */
3129 
3130 /**
3131  * mpol_to_str - format a mempolicy structure for printing
3132  * @buffer:  to contain formatted mempolicy string
3133  * @maxlen:  length of @buffer
3134  * @pol:  pointer to mempolicy to be formatted
3135  *
3136  * Convert @pol into a string.  If @buffer is too short, truncate the string.
3137  * Recommend a @maxlen of at least 51 for the longest mode, "weighted
3138  * interleave", plus the longest flag flags, "relative|balancing", and to
3139  * display at least a few node ids.
3140  */
mpol_to_str(char * buffer,int maxlen,struct mempolicy * pol)3141 void mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)
3142 {
3143 	char *p = buffer;
3144 	nodemask_t nodes = NODE_MASK_NONE;
3145 	unsigned short mode = MPOL_DEFAULT;
3146 	unsigned short flags = 0;
3147 
3148 	if (pol &&
3149 	    pol != &default_policy &&
3150 	    !(pol >= &preferred_node_policy[0] &&
3151 	      pol <= &preferred_node_policy[ARRAY_SIZE(preferred_node_policy) - 1])) {
3152 		mode = pol->mode;
3153 		flags = pol->flags;
3154 	}
3155 
3156 	switch (mode) {
3157 	case MPOL_DEFAULT:
3158 	case MPOL_LOCAL:
3159 		break;
3160 	case MPOL_PREFERRED:
3161 	case MPOL_PREFERRED_MANY:
3162 	case MPOL_BIND:
3163 	case MPOL_INTERLEAVE:
3164 		nodes = pol->nodes;
3165 		break;
3166 	default:
3167 		WARN_ON_ONCE(1);
3168 		snprintf(p, maxlen, "unknown");
3169 		return;
3170 	}
3171 
3172 	p += snprintf(p, maxlen, "%s", policy_modes[mode]);
3173 
3174 	if (flags & MPOL_MODE_FLAGS) {
3175 		p += snprintf(p, buffer + maxlen - p, "=");
3176 
3177 		/*
3178 		 * Static and relative are mutually exclusive.
3179 		 */
3180 		if (flags & MPOL_F_STATIC_NODES)
3181 			p += snprintf(p, buffer + maxlen - p, "static");
3182 		else if (flags & MPOL_F_RELATIVE_NODES)
3183 			p += snprintf(p, buffer + maxlen - p, "relative");
3184 
3185 		if (flags & MPOL_F_NUMA_BALANCING) {
3186 			if (!is_power_of_2(flags & MPOL_MODE_FLAGS))
3187 				p += snprintf(p, buffer + maxlen - p, "|");
3188 			p += snprintf(p, buffer + maxlen - p, "balancing");
3189 		}
3190 	}
3191 
3192 	if (!nodes_empty(nodes))
3193 		p += scnprintf(p, buffer + maxlen - p, ":%*pbl",
3194 			       nodemask_pr_args(&nodes));
3195 }
3196