1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Simple NUMA memory policy for the Linux kernel. 4 * 5 * Copyright 2003,2004 Andi Kleen, SuSE Labs. 6 * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc. 7 * 8 * NUMA policy allows the user to give hints in which node(s) memory should 9 * be allocated. 10 * 11 * Support four policies per VMA and per process: 12 * 13 * The VMA policy has priority over the process policy for a page fault. 14 * 15 * interleave Allocate memory interleaved over a set of nodes, 16 * with normal fallback if it fails. 17 * For VMA based allocations this interleaves based on the 18 * offset into the backing object or offset into the mapping 19 * for anonymous memory. For process policy an process counter 20 * is used. 21 * 22 * bind Only allocate memory on a specific set of nodes, 23 * no fallback. 24 * FIXME: memory is allocated starting with the first node 25 * to the last. It would be better if bind would truly restrict 26 * the allocation to memory nodes instead 27 * 28 * preferred Try a specific node first before normal fallback. 29 * As a special case NUMA_NO_NODE here means do the allocation 30 * on the local CPU. This is normally identical to default, 31 * but useful to set in a VMA when you have a non default 32 * process policy. 33 * 34 * preferred many Try a set of nodes first before normal fallback. This is 35 * similar to preferred without the special case. 36 * 37 * default Allocate on the local node first, or when on a VMA 38 * use the process policy. This is what Linux always did 39 * in a NUMA aware kernel and still does by, ahem, default. 40 * 41 * The process policy is applied for most non interrupt memory allocations 42 * in that process' context. Interrupts ignore the policies and always 43 * try to allocate on the local CPU. The VMA policy is only applied for memory 44 * allocations for a VMA in the VM. 45 * 46 * Currently there are a few corner cases in swapping where the policy 47 * is not applied, but the majority should be handled. When process policy 48 * is used it is not remembered over swap outs/swap ins. 49 * 50 * Only the highest zone in the zone hierarchy gets policied. Allocations 51 * requesting a lower zone just use default policy. This implies that 52 * on systems with highmem kernel lowmem allocation don't get policied. 53 * Same with GFP_DMA allocations. 54 * 55 * For shmfs/tmpfs/hugetlbfs shared memory the policy is shared between 56 * all users and remembered even when nobody has memory mapped. 57 */ 58 59 /* Notebook: 60 fix mmap readahead to honour policy and enable policy for any page cache 61 object 62 statistics for bigpages 63 global policy for page cache? currently it uses process policy. Requires 64 first item above. 65 handle mremap for shared memory (currently ignored for the policy) 66 grows down? 67 make bind policy root only? It can trigger oom much faster and the 68 kernel is not always grateful with that. 69 */ 70 71 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 72 73 #include <linux/mempolicy.h> 74 #include <linux/pagewalk.h> 75 #include <linux/highmem.h> 76 #include <linux/hugetlb.h> 77 #include <linux/kernel.h> 78 #include <linux/sched.h> 79 #include <linux/sched/mm.h> 80 #include <linux/sched/numa_balancing.h> 81 #include <linux/sched/task.h> 82 #include <linux/nodemask.h> 83 #include <linux/cpuset.h> 84 #include <linux/slab.h> 85 #include <linux/string.h> 86 #include <linux/export.h> 87 #include <linux/nsproxy.h> 88 #include <linux/interrupt.h> 89 #include <linux/init.h> 90 #include <linux/compat.h> 91 #include <linux/ptrace.h> 92 #include <linux/swap.h> 93 #include <linux/seq_file.h> 94 #include <linux/proc_fs.h> 95 #include <linux/migrate.h> 96 #include <linux/ksm.h> 97 #include <linux/rmap.h> 98 #include <linux/security.h> 99 #include <linux/syscalls.h> 100 #include <linux/ctype.h> 101 #include <linux/mm_inline.h> 102 #include <linux/mmu_notifier.h> 103 #include <linux/printk.h> 104 #include <linux/swapops.h> 105 106 #include <asm/tlbflush.h> 107 #include <linux/uaccess.h> 108 109 #include "internal.h" 110 111 /* Internal flags */ 112 #define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0) /* Skip checks for continuous vmas */ 113 #define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1) /* Invert check for nodemask */ 114 115 static struct kmem_cache *policy_cache; 116 static struct kmem_cache *sn_cache; 117 118 /* Highest zone. An specific allocation for a zone below that is not 119 policied. */ 120 enum zone_type policy_zone = 0; 121 122 /* 123 * run-time system-wide default policy => local allocation 124 */ 125 static struct mempolicy default_policy = { 126 .refcnt = ATOMIC_INIT(1), /* never free it */ 127 .mode = MPOL_LOCAL, 128 }; 129 130 static struct mempolicy preferred_node_policy[MAX_NUMNODES]; 131 132 /** 133 * numa_map_to_online_node - Find closest online node 134 * @node: Node id to start the search 135 * 136 * Lookup the next closest node by distance if @nid is not online. 137 * 138 * Return: this @node if it is online, otherwise the closest node by distance 139 */ 140 int numa_map_to_online_node(int node) 141 { 142 int min_dist = INT_MAX, dist, n, min_node; 143 144 if (node == NUMA_NO_NODE || node_online(node)) 145 return node; 146 147 min_node = node; 148 for_each_online_node(n) { 149 dist = node_distance(node, n); 150 if (dist < min_dist) { 151 min_dist = dist; 152 min_node = n; 153 } 154 } 155 156 return min_node; 157 } 158 EXPORT_SYMBOL_GPL(numa_map_to_online_node); 159 160 struct mempolicy *get_task_policy(struct task_struct *p) 161 { 162 struct mempolicy *pol = p->mempolicy; 163 int node; 164 165 if (pol) 166 return pol; 167 168 node = numa_node_id(); 169 if (node != NUMA_NO_NODE) { 170 pol = &preferred_node_policy[node]; 171 /* preferred_node_policy is not initialised early in boot */ 172 if (pol->mode) 173 return pol; 174 } 175 176 return &default_policy; 177 } 178 179 static const struct mempolicy_operations { 180 int (*create)(struct mempolicy *pol, const nodemask_t *nodes); 181 void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes); 182 } mpol_ops[MPOL_MAX]; 183 184 static inline int mpol_store_user_nodemask(const struct mempolicy *pol) 185 { 186 return pol->flags & MPOL_MODE_FLAGS; 187 } 188 189 static void mpol_relative_nodemask(nodemask_t *ret, const nodemask_t *orig, 190 const nodemask_t *rel) 191 { 192 nodemask_t tmp; 193 nodes_fold(tmp, *orig, nodes_weight(*rel)); 194 nodes_onto(*ret, tmp, *rel); 195 } 196 197 static int mpol_new_nodemask(struct mempolicy *pol, const nodemask_t *nodes) 198 { 199 if (nodes_empty(*nodes)) 200 return -EINVAL; 201 pol->nodes = *nodes; 202 return 0; 203 } 204 205 static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes) 206 { 207 if (nodes_empty(*nodes)) 208 return -EINVAL; 209 210 nodes_clear(pol->nodes); 211 node_set(first_node(*nodes), pol->nodes); 212 return 0; 213 } 214 215 /* 216 * mpol_set_nodemask is called after mpol_new() to set up the nodemask, if 217 * any, for the new policy. mpol_new() has already validated the nodes 218 * parameter with respect to the policy mode and flags. 219 * 220 * Must be called holding task's alloc_lock to protect task's mems_allowed 221 * and mempolicy. May also be called holding the mmap_lock for write. 222 */ 223 static int mpol_set_nodemask(struct mempolicy *pol, 224 const nodemask_t *nodes, struct nodemask_scratch *nsc) 225 { 226 int ret; 227 228 /* 229 * Default (pol==NULL) resp. local memory policies are not a 230 * subject of any remapping. They also do not need any special 231 * constructor. 232 */ 233 if (!pol || pol->mode == MPOL_LOCAL) 234 return 0; 235 236 /* Check N_MEMORY */ 237 nodes_and(nsc->mask1, 238 cpuset_current_mems_allowed, node_states[N_MEMORY]); 239 240 VM_BUG_ON(!nodes); 241 242 if (pol->flags & MPOL_F_RELATIVE_NODES) 243 mpol_relative_nodemask(&nsc->mask2, nodes, &nsc->mask1); 244 else 245 nodes_and(nsc->mask2, *nodes, nsc->mask1); 246 247 if (mpol_store_user_nodemask(pol)) 248 pol->w.user_nodemask = *nodes; 249 else 250 pol->w.cpuset_mems_allowed = cpuset_current_mems_allowed; 251 252 ret = mpol_ops[pol->mode].create(pol, &nsc->mask2); 253 return ret; 254 } 255 256 /* 257 * This function just creates a new policy, does some check and simple 258 * initialization. You must invoke mpol_set_nodemask() to set nodes. 259 */ 260 static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags, 261 nodemask_t *nodes) 262 { 263 struct mempolicy *policy; 264 265 pr_debug("setting mode %d flags %d nodes[0] %lx\n", 266 mode, flags, nodes ? nodes_addr(*nodes)[0] : NUMA_NO_NODE); 267 268 if (mode == MPOL_DEFAULT) { 269 if (nodes && !nodes_empty(*nodes)) 270 return ERR_PTR(-EINVAL); 271 return NULL; 272 } 273 VM_BUG_ON(!nodes); 274 275 /* 276 * MPOL_PREFERRED cannot be used with MPOL_F_STATIC_NODES or 277 * MPOL_F_RELATIVE_NODES if the nodemask is empty (local allocation). 278 * All other modes require a valid pointer to a non-empty nodemask. 279 */ 280 if (mode == MPOL_PREFERRED) { 281 if (nodes_empty(*nodes)) { 282 if (((flags & MPOL_F_STATIC_NODES) || 283 (flags & MPOL_F_RELATIVE_NODES))) 284 return ERR_PTR(-EINVAL); 285 286 mode = MPOL_LOCAL; 287 } 288 } else if (mode == MPOL_LOCAL) { 289 if (!nodes_empty(*nodes) || 290 (flags & MPOL_F_STATIC_NODES) || 291 (flags & MPOL_F_RELATIVE_NODES)) 292 return ERR_PTR(-EINVAL); 293 } else if (nodes_empty(*nodes)) 294 return ERR_PTR(-EINVAL); 295 policy = kmem_cache_alloc(policy_cache, GFP_KERNEL); 296 if (!policy) 297 return ERR_PTR(-ENOMEM); 298 atomic_set(&policy->refcnt, 1); 299 policy->mode = mode; 300 policy->flags = flags; 301 policy->home_node = NUMA_NO_NODE; 302 303 return policy; 304 } 305 306 /* Slow path of a mpol destructor. */ 307 void __mpol_put(struct mempolicy *p) 308 { 309 if (!atomic_dec_and_test(&p->refcnt)) 310 return; 311 kmem_cache_free(policy_cache, p); 312 } 313 314 static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes) 315 { 316 } 317 318 static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes) 319 { 320 nodemask_t tmp; 321 322 if (pol->flags & MPOL_F_STATIC_NODES) 323 nodes_and(tmp, pol->w.user_nodemask, *nodes); 324 else if (pol->flags & MPOL_F_RELATIVE_NODES) 325 mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes); 326 else { 327 nodes_remap(tmp, pol->nodes, pol->w.cpuset_mems_allowed, 328 *nodes); 329 pol->w.cpuset_mems_allowed = *nodes; 330 } 331 332 if (nodes_empty(tmp)) 333 tmp = *nodes; 334 335 pol->nodes = tmp; 336 } 337 338 static void mpol_rebind_preferred(struct mempolicy *pol, 339 const nodemask_t *nodes) 340 { 341 pol->w.cpuset_mems_allowed = *nodes; 342 } 343 344 /* 345 * mpol_rebind_policy - Migrate a policy to a different set of nodes 346 * 347 * Per-vma policies are protected by mmap_lock. Allocations using per-task 348 * policies are protected by task->mems_allowed_seq to prevent a premature 349 * OOM/allocation failure due to parallel nodemask modification. 350 */ 351 static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask) 352 { 353 if (!pol) 354 return; 355 if (!mpol_store_user_nodemask(pol) && 356 nodes_equal(pol->w.cpuset_mems_allowed, *newmask)) 357 return; 358 359 mpol_ops[pol->mode].rebind(pol, newmask); 360 } 361 362 /* 363 * Wrapper for mpol_rebind_policy() that just requires task 364 * pointer, and updates task mempolicy. 365 * 366 * Called with task's alloc_lock held. 367 */ 368 369 void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new) 370 { 371 mpol_rebind_policy(tsk->mempolicy, new); 372 } 373 374 /* 375 * Rebind each vma in mm to new nodemask. 376 * 377 * Call holding a reference to mm. Takes mm->mmap_lock during call. 378 */ 379 380 void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new) 381 { 382 struct vm_area_struct *vma; 383 384 mmap_write_lock(mm); 385 for (vma = mm->mmap; vma; vma = vma->vm_next) 386 mpol_rebind_policy(vma->vm_policy, new); 387 mmap_write_unlock(mm); 388 } 389 390 static const struct mempolicy_operations mpol_ops[MPOL_MAX] = { 391 [MPOL_DEFAULT] = { 392 .rebind = mpol_rebind_default, 393 }, 394 [MPOL_INTERLEAVE] = { 395 .create = mpol_new_nodemask, 396 .rebind = mpol_rebind_nodemask, 397 }, 398 [MPOL_PREFERRED] = { 399 .create = mpol_new_preferred, 400 .rebind = mpol_rebind_preferred, 401 }, 402 [MPOL_BIND] = { 403 .create = mpol_new_nodemask, 404 .rebind = mpol_rebind_nodemask, 405 }, 406 [MPOL_LOCAL] = { 407 .rebind = mpol_rebind_default, 408 }, 409 [MPOL_PREFERRED_MANY] = { 410 .create = mpol_new_nodemask, 411 .rebind = mpol_rebind_preferred, 412 }, 413 }; 414 415 static int migrate_page_add(struct page *page, struct list_head *pagelist, 416 unsigned long flags); 417 418 struct queue_pages { 419 struct list_head *pagelist; 420 unsigned long flags; 421 nodemask_t *nmask; 422 unsigned long start; 423 unsigned long end; 424 struct vm_area_struct *first; 425 }; 426 427 /* 428 * Check if the page's nid is in qp->nmask. 429 * 430 * If MPOL_MF_INVERT is set in qp->flags, check if the nid is 431 * in the invert of qp->nmask. 432 */ 433 static inline bool queue_pages_required(struct page *page, 434 struct queue_pages *qp) 435 { 436 int nid = page_to_nid(page); 437 unsigned long flags = qp->flags; 438 439 return node_isset(nid, *qp->nmask) == !(flags & MPOL_MF_INVERT); 440 } 441 442 /* 443 * queue_pages_pmd() has four possible return values: 444 * 0 - pages are placed on the right node or queued successfully, or 445 * special page is met, i.e. huge zero page. 446 * 1 - there is unmovable page, and MPOL_MF_MOVE* & MPOL_MF_STRICT were 447 * specified. 448 * 2 - THP was split. 449 * -EIO - is migration entry or only MPOL_MF_STRICT was specified and an 450 * existing page was already on a node that does not follow the 451 * policy. 452 */ 453 static int queue_pages_pmd(pmd_t *pmd, spinlock_t *ptl, unsigned long addr, 454 unsigned long end, struct mm_walk *walk) 455 __releases(ptl) 456 { 457 int ret = 0; 458 struct page *page; 459 struct queue_pages *qp = walk->private; 460 unsigned long flags; 461 462 if (unlikely(is_pmd_migration_entry(*pmd))) { 463 ret = -EIO; 464 goto unlock; 465 } 466 page = pmd_page(*pmd); 467 if (is_huge_zero_page(page)) { 468 spin_unlock(ptl); 469 walk->action = ACTION_CONTINUE; 470 goto out; 471 } 472 if (!queue_pages_required(page, qp)) 473 goto unlock; 474 475 flags = qp->flags; 476 /* go to thp migration */ 477 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) { 478 if (!vma_migratable(walk->vma) || 479 migrate_page_add(page, qp->pagelist, flags)) { 480 ret = 1; 481 goto unlock; 482 } 483 } else 484 ret = -EIO; 485 unlock: 486 spin_unlock(ptl); 487 out: 488 return ret; 489 } 490 491 /* 492 * Scan through pages checking if pages follow certain conditions, 493 * and move them to the pagelist if they do. 494 * 495 * queue_pages_pte_range() has three possible return values: 496 * 0 - pages are placed on the right node or queued successfully, or 497 * special page is met, i.e. zero page. 498 * 1 - there is unmovable page, and MPOL_MF_MOVE* & MPOL_MF_STRICT were 499 * specified. 500 * -EIO - only MPOL_MF_STRICT was specified and an existing page was already 501 * on a node that does not follow the policy. 502 */ 503 static int queue_pages_pte_range(pmd_t *pmd, unsigned long addr, 504 unsigned long end, struct mm_walk *walk) 505 { 506 struct vm_area_struct *vma = walk->vma; 507 struct page *page; 508 struct queue_pages *qp = walk->private; 509 unsigned long flags = qp->flags; 510 int ret; 511 bool has_unmovable = false; 512 pte_t *pte, *mapped_pte; 513 spinlock_t *ptl; 514 515 ptl = pmd_trans_huge_lock(pmd, vma); 516 if (ptl) { 517 ret = queue_pages_pmd(pmd, ptl, addr, end, walk); 518 if (ret != 2) 519 return ret; 520 } 521 /* THP was split, fall through to pte walk */ 522 523 if (pmd_trans_unstable(pmd)) 524 return 0; 525 526 mapped_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl); 527 for (; addr != end; pte++, addr += PAGE_SIZE) { 528 if (!pte_present(*pte)) 529 continue; 530 page = vm_normal_page(vma, addr, *pte); 531 if (!page) 532 continue; 533 /* 534 * vm_normal_page() filters out zero pages, but there might 535 * still be PageReserved pages to skip, perhaps in a VDSO. 536 */ 537 if (PageReserved(page)) 538 continue; 539 if (!queue_pages_required(page, qp)) 540 continue; 541 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) { 542 /* MPOL_MF_STRICT must be specified if we get here */ 543 if (!vma_migratable(vma)) { 544 has_unmovable = true; 545 break; 546 } 547 548 /* 549 * Do not abort immediately since there may be 550 * temporary off LRU pages in the range. Still 551 * need migrate other LRU pages. 552 */ 553 if (migrate_page_add(page, qp->pagelist, flags)) 554 has_unmovable = true; 555 } else 556 break; 557 } 558 pte_unmap_unlock(mapped_pte, ptl); 559 cond_resched(); 560 561 if (has_unmovable) 562 return 1; 563 564 return addr != end ? -EIO : 0; 565 } 566 567 static int queue_pages_hugetlb(pte_t *pte, unsigned long hmask, 568 unsigned long addr, unsigned long end, 569 struct mm_walk *walk) 570 { 571 int ret = 0; 572 #ifdef CONFIG_HUGETLB_PAGE 573 struct queue_pages *qp = walk->private; 574 unsigned long flags = (qp->flags & MPOL_MF_VALID); 575 struct page *page; 576 spinlock_t *ptl; 577 pte_t entry; 578 579 ptl = huge_pte_lock(hstate_vma(walk->vma), walk->mm, pte); 580 entry = huge_ptep_get(pte); 581 if (!pte_present(entry)) 582 goto unlock; 583 page = pte_page(entry); 584 if (!queue_pages_required(page, qp)) 585 goto unlock; 586 587 if (flags == MPOL_MF_STRICT) { 588 /* 589 * STRICT alone means only detecting misplaced page and no 590 * need to further check other vma. 591 */ 592 ret = -EIO; 593 goto unlock; 594 } 595 596 if (!vma_migratable(walk->vma)) { 597 /* 598 * Must be STRICT with MOVE*, otherwise .test_walk() have 599 * stopped walking current vma. 600 * Detecting misplaced page but allow migrating pages which 601 * have been queued. 602 */ 603 ret = 1; 604 goto unlock; 605 } 606 607 /* With MPOL_MF_MOVE, we migrate only unshared hugepage. */ 608 if (flags & (MPOL_MF_MOVE_ALL) || 609 (flags & MPOL_MF_MOVE && page_mapcount(page) == 1)) { 610 if (!isolate_huge_page(page, qp->pagelist) && 611 (flags & MPOL_MF_STRICT)) 612 /* 613 * Failed to isolate page but allow migrating pages 614 * which have been queued. 615 */ 616 ret = 1; 617 } 618 unlock: 619 spin_unlock(ptl); 620 #else 621 BUG(); 622 #endif 623 return ret; 624 } 625 626 #ifdef CONFIG_NUMA_BALANCING 627 /* 628 * This is used to mark a range of virtual addresses to be inaccessible. 629 * These are later cleared by a NUMA hinting fault. Depending on these 630 * faults, pages may be migrated for better NUMA placement. 631 * 632 * This is assuming that NUMA faults are handled using PROT_NONE. If 633 * an architecture makes a different choice, it will need further 634 * changes to the core. 635 */ 636 unsigned long change_prot_numa(struct vm_area_struct *vma, 637 unsigned long addr, unsigned long end) 638 { 639 int nr_updated; 640 641 nr_updated = change_protection(vma, addr, end, PAGE_NONE, MM_CP_PROT_NUMA); 642 if (nr_updated) 643 count_vm_numa_events(NUMA_PTE_UPDATES, nr_updated); 644 645 return nr_updated; 646 } 647 #else 648 static unsigned long change_prot_numa(struct vm_area_struct *vma, 649 unsigned long addr, unsigned long end) 650 { 651 return 0; 652 } 653 #endif /* CONFIG_NUMA_BALANCING */ 654 655 static int queue_pages_test_walk(unsigned long start, unsigned long end, 656 struct mm_walk *walk) 657 { 658 struct vm_area_struct *vma = walk->vma; 659 struct queue_pages *qp = walk->private; 660 unsigned long endvma = vma->vm_end; 661 unsigned long flags = qp->flags; 662 663 /* range check first */ 664 VM_BUG_ON_VMA(!range_in_vma(vma, start, end), vma); 665 666 if (!qp->first) { 667 qp->first = vma; 668 if (!(flags & MPOL_MF_DISCONTIG_OK) && 669 (qp->start < vma->vm_start)) 670 /* hole at head side of range */ 671 return -EFAULT; 672 } 673 if (!(flags & MPOL_MF_DISCONTIG_OK) && 674 ((vma->vm_end < qp->end) && 675 (!vma->vm_next || vma->vm_end < vma->vm_next->vm_start))) 676 /* hole at middle or tail of range */ 677 return -EFAULT; 678 679 /* 680 * Need check MPOL_MF_STRICT to return -EIO if possible 681 * regardless of vma_migratable 682 */ 683 if (!vma_migratable(vma) && 684 !(flags & MPOL_MF_STRICT)) 685 return 1; 686 687 if (endvma > end) 688 endvma = end; 689 690 if (flags & MPOL_MF_LAZY) { 691 /* Similar to task_numa_work, skip inaccessible VMAs */ 692 if (!is_vm_hugetlb_page(vma) && vma_is_accessible(vma) && 693 !(vma->vm_flags & VM_MIXEDMAP)) 694 change_prot_numa(vma, start, endvma); 695 return 1; 696 } 697 698 /* queue pages from current vma */ 699 if (flags & MPOL_MF_VALID) 700 return 0; 701 return 1; 702 } 703 704 static const struct mm_walk_ops queue_pages_walk_ops = { 705 .hugetlb_entry = queue_pages_hugetlb, 706 .pmd_entry = queue_pages_pte_range, 707 .test_walk = queue_pages_test_walk, 708 }; 709 710 /* 711 * Walk through page tables and collect pages to be migrated. 712 * 713 * If pages found in a given range are on a set of nodes (determined by 714 * @nodes and @flags,) it's isolated and queued to the pagelist which is 715 * passed via @private. 716 * 717 * queue_pages_range() has three possible return values: 718 * 1 - there is unmovable page, but MPOL_MF_MOVE* & MPOL_MF_STRICT were 719 * specified. 720 * 0 - queue pages successfully or no misplaced page. 721 * errno - i.e. misplaced pages with MPOL_MF_STRICT specified (-EIO) or 722 * memory range specified by nodemask and maxnode points outside 723 * your accessible address space (-EFAULT) 724 */ 725 static int 726 queue_pages_range(struct mm_struct *mm, unsigned long start, unsigned long end, 727 nodemask_t *nodes, unsigned long flags, 728 struct list_head *pagelist) 729 { 730 int err; 731 struct queue_pages qp = { 732 .pagelist = pagelist, 733 .flags = flags, 734 .nmask = nodes, 735 .start = start, 736 .end = end, 737 .first = NULL, 738 }; 739 740 err = walk_page_range(mm, start, end, &queue_pages_walk_ops, &qp); 741 742 if (!qp.first) 743 /* whole range in hole */ 744 err = -EFAULT; 745 746 return err; 747 } 748 749 /* 750 * Apply policy to a single VMA 751 * This must be called with the mmap_lock held for writing. 752 */ 753 static int vma_replace_policy(struct vm_area_struct *vma, 754 struct mempolicy *pol) 755 { 756 int err; 757 struct mempolicy *old; 758 struct mempolicy *new; 759 760 pr_debug("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n", 761 vma->vm_start, vma->vm_end, vma->vm_pgoff, 762 vma->vm_ops, vma->vm_file, 763 vma->vm_ops ? vma->vm_ops->set_policy : NULL); 764 765 new = mpol_dup(pol); 766 if (IS_ERR(new)) 767 return PTR_ERR(new); 768 769 if (vma->vm_ops && vma->vm_ops->set_policy) { 770 err = vma->vm_ops->set_policy(vma, new); 771 if (err) 772 goto err_out; 773 } 774 775 old = vma->vm_policy; 776 vma->vm_policy = new; /* protected by mmap_lock */ 777 mpol_put(old); 778 779 return 0; 780 err_out: 781 mpol_put(new); 782 return err; 783 } 784 785 /* Step 2: apply policy to a range and do splits. */ 786 static int mbind_range(struct mm_struct *mm, unsigned long start, 787 unsigned long end, struct mempolicy *new_pol) 788 { 789 struct vm_area_struct *next; 790 struct vm_area_struct *prev; 791 struct vm_area_struct *vma; 792 int err = 0; 793 pgoff_t pgoff; 794 unsigned long vmstart; 795 unsigned long vmend; 796 797 vma = find_vma(mm, start); 798 VM_BUG_ON(!vma); 799 800 prev = vma->vm_prev; 801 if (start > vma->vm_start) 802 prev = vma; 803 804 for (; vma && vma->vm_start < end; prev = vma, vma = next) { 805 next = vma->vm_next; 806 vmstart = max(start, vma->vm_start); 807 vmend = min(end, vma->vm_end); 808 809 if (mpol_equal(vma_policy(vma), new_pol)) 810 continue; 811 812 pgoff = vma->vm_pgoff + 813 ((vmstart - vma->vm_start) >> PAGE_SHIFT); 814 prev = vma_merge(mm, prev, vmstart, vmend, vma->vm_flags, 815 vma->anon_vma, vma->vm_file, pgoff, 816 new_pol, vma->vm_userfaultfd_ctx, 817 vma_anon_name(vma)); 818 if (prev) { 819 vma = prev; 820 next = vma->vm_next; 821 if (mpol_equal(vma_policy(vma), new_pol)) 822 continue; 823 /* vma_merge() joined vma && vma->next, case 8 */ 824 goto replace; 825 } 826 if (vma->vm_start != vmstart) { 827 err = split_vma(vma->vm_mm, vma, vmstart, 1); 828 if (err) 829 goto out; 830 } 831 if (vma->vm_end != vmend) { 832 err = split_vma(vma->vm_mm, vma, vmend, 0); 833 if (err) 834 goto out; 835 } 836 replace: 837 err = vma_replace_policy(vma, new_pol); 838 if (err) 839 goto out; 840 } 841 842 out: 843 return err; 844 } 845 846 /* Set the process memory policy */ 847 static long do_set_mempolicy(unsigned short mode, unsigned short flags, 848 nodemask_t *nodes) 849 { 850 struct mempolicy *new, *old; 851 NODEMASK_SCRATCH(scratch); 852 int ret; 853 854 if (!scratch) 855 return -ENOMEM; 856 857 new = mpol_new(mode, flags, nodes); 858 if (IS_ERR(new)) { 859 ret = PTR_ERR(new); 860 goto out; 861 } 862 863 ret = mpol_set_nodemask(new, nodes, scratch); 864 if (ret) { 865 mpol_put(new); 866 goto out; 867 } 868 task_lock(current); 869 old = current->mempolicy; 870 current->mempolicy = new; 871 if (new && new->mode == MPOL_INTERLEAVE) 872 current->il_prev = MAX_NUMNODES-1; 873 task_unlock(current); 874 mpol_put(old); 875 ret = 0; 876 out: 877 NODEMASK_SCRATCH_FREE(scratch); 878 return ret; 879 } 880 881 /* 882 * Return nodemask for policy for get_mempolicy() query 883 * 884 * Called with task's alloc_lock held 885 */ 886 static void get_policy_nodemask(struct mempolicy *p, nodemask_t *nodes) 887 { 888 nodes_clear(*nodes); 889 if (p == &default_policy) 890 return; 891 892 switch (p->mode) { 893 case MPOL_BIND: 894 case MPOL_INTERLEAVE: 895 case MPOL_PREFERRED: 896 case MPOL_PREFERRED_MANY: 897 *nodes = p->nodes; 898 break; 899 case MPOL_LOCAL: 900 /* return empty node mask for local allocation */ 901 break; 902 default: 903 BUG(); 904 } 905 } 906 907 static int lookup_node(struct mm_struct *mm, unsigned long addr) 908 { 909 struct page *p = NULL; 910 int err; 911 912 int locked = 1; 913 err = get_user_pages_locked(addr & PAGE_MASK, 1, 0, &p, &locked); 914 if (err > 0) { 915 err = page_to_nid(p); 916 put_page(p); 917 } 918 if (locked) 919 mmap_read_unlock(mm); 920 return err; 921 } 922 923 /* Retrieve NUMA policy */ 924 static long do_get_mempolicy(int *policy, nodemask_t *nmask, 925 unsigned long addr, unsigned long flags) 926 { 927 int err; 928 struct mm_struct *mm = current->mm; 929 struct vm_area_struct *vma = NULL; 930 struct mempolicy *pol = current->mempolicy, *pol_refcount = NULL; 931 932 if (flags & 933 ~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR|MPOL_F_MEMS_ALLOWED)) 934 return -EINVAL; 935 936 if (flags & MPOL_F_MEMS_ALLOWED) { 937 if (flags & (MPOL_F_NODE|MPOL_F_ADDR)) 938 return -EINVAL; 939 *policy = 0; /* just so it's initialized */ 940 task_lock(current); 941 *nmask = cpuset_current_mems_allowed; 942 task_unlock(current); 943 return 0; 944 } 945 946 if (flags & MPOL_F_ADDR) { 947 /* 948 * Do NOT fall back to task policy if the 949 * vma/shared policy at addr is NULL. We 950 * want to return MPOL_DEFAULT in this case. 951 */ 952 mmap_read_lock(mm); 953 vma = vma_lookup(mm, addr); 954 if (!vma) { 955 mmap_read_unlock(mm); 956 return -EFAULT; 957 } 958 if (vma->vm_ops && vma->vm_ops->get_policy) 959 pol = vma->vm_ops->get_policy(vma, addr); 960 else 961 pol = vma->vm_policy; 962 } else if (addr) 963 return -EINVAL; 964 965 if (!pol) 966 pol = &default_policy; /* indicates default behavior */ 967 968 if (flags & MPOL_F_NODE) { 969 if (flags & MPOL_F_ADDR) { 970 /* 971 * Take a refcount on the mpol, lookup_node() 972 * will drop the mmap_lock, so after calling 973 * lookup_node() only "pol" remains valid, "vma" 974 * is stale. 975 */ 976 pol_refcount = pol; 977 vma = NULL; 978 mpol_get(pol); 979 err = lookup_node(mm, addr); 980 if (err < 0) 981 goto out; 982 *policy = err; 983 } else if (pol == current->mempolicy && 984 pol->mode == MPOL_INTERLEAVE) { 985 *policy = next_node_in(current->il_prev, pol->nodes); 986 } else { 987 err = -EINVAL; 988 goto out; 989 } 990 } else { 991 *policy = pol == &default_policy ? MPOL_DEFAULT : 992 pol->mode; 993 /* 994 * Internal mempolicy flags must be masked off before exposing 995 * the policy to userspace. 996 */ 997 *policy |= (pol->flags & MPOL_MODE_FLAGS); 998 } 999 1000 err = 0; 1001 if (nmask) { 1002 if (mpol_store_user_nodemask(pol)) { 1003 *nmask = pol->w.user_nodemask; 1004 } else { 1005 task_lock(current); 1006 get_policy_nodemask(pol, nmask); 1007 task_unlock(current); 1008 } 1009 } 1010 1011 out: 1012 mpol_cond_put(pol); 1013 if (vma) 1014 mmap_read_unlock(mm); 1015 if (pol_refcount) 1016 mpol_put(pol_refcount); 1017 return err; 1018 } 1019 1020 #ifdef CONFIG_MIGRATION 1021 /* 1022 * page migration, thp tail pages can be passed. 1023 */ 1024 static int migrate_page_add(struct page *page, struct list_head *pagelist, 1025 unsigned long flags) 1026 { 1027 struct page *head = compound_head(page); 1028 /* 1029 * Avoid migrating a page that is shared with others. 1030 */ 1031 if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(head) == 1) { 1032 if (!isolate_lru_page(head)) { 1033 list_add_tail(&head->lru, pagelist); 1034 mod_node_page_state(page_pgdat(head), 1035 NR_ISOLATED_ANON + page_is_file_lru(head), 1036 thp_nr_pages(head)); 1037 } else if (flags & MPOL_MF_STRICT) { 1038 /* 1039 * Non-movable page may reach here. And, there may be 1040 * temporary off LRU pages or non-LRU movable pages. 1041 * Treat them as unmovable pages since they can't be 1042 * isolated, so they can't be moved at the moment. It 1043 * should return -EIO for this case too. 1044 */ 1045 return -EIO; 1046 } 1047 } 1048 1049 return 0; 1050 } 1051 1052 /* 1053 * Migrate pages from one node to a target node. 1054 * Returns error or the number of pages not migrated. 1055 */ 1056 static int migrate_to_node(struct mm_struct *mm, int source, int dest, 1057 int flags) 1058 { 1059 nodemask_t nmask; 1060 LIST_HEAD(pagelist); 1061 int err = 0; 1062 struct migration_target_control mtc = { 1063 .nid = dest, 1064 .gfp_mask = GFP_HIGHUSER_MOVABLE | __GFP_THISNODE, 1065 }; 1066 1067 nodes_clear(nmask); 1068 node_set(source, nmask); 1069 1070 /* 1071 * This does not "check" the range but isolates all pages that 1072 * need migration. Between passing in the full user address 1073 * space range and MPOL_MF_DISCONTIG_OK, this call can not fail. 1074 */ 1075 VM_BUG_ON(!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))); 1076 queue_pages_range(mm, mm->mmap->vm_start, mm->task_size, &nmask, 1077 flags | MPOL_MF_DISCONTIG_OK, &pagelist); 1078 1079 if (!list_empty(&pagelist)) { 1080 err = migrate_pages(&pagelist, alloc_migration_target, NULL, 1081 (unsigned long)&mtc, MIGRATE_SYNC, MR_SYSCALL, NULL); 1082 if (err) 1083 putback_movable_pages(&pagelist); 1084 } 1085 1086 return err; 1087 } 1088 1089 /* 1090 * Move pages between the two nodesets so as to preserve the physical 1091 * layout as much as possible. 1092 * 1093 * Returns the number of page that could not be moved. 1094 */ 1095 int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from, 1096 const nodemask_t *to, int flags) 1097 { 1098 int busy = 0; 1099 int err = 0; 1100 nodemask_t tmp; 1101 1102 lru_cache_disable(); 1103 1104 mmap_read_lock(mm); 1105 1106 /* 1107 * Find a 'source' bit set in 'tmp' whose corresponding 'dest' 1108 * bit in 'to' is not also set in 'tmp'. Clear the found 'source' 1109 * bit in 'tmp', and return that <source, dest> pair for migration. 1110 * The pair of nodemasks 'to' and 'from' define the map. 1111 * 1112 * If no pair of bits is found that way, fallback to picking some 1113 * pair of 'source' and 'dest' bits that are not the same. If the 1114 * 'source' and 'dest' bits are the same, this represents a node 1115 * that will be migrating to itself, so no pages need move. 1116 * 1117 * If no bits are left in 'tmp', or if all remaining bits left 1118 * in 'tmp' correspond to the same bit in 'to', return false 1119 * (nothing left to migrate). 1120 * 1121 * This lets us pick a pair of nodes to migrate between, such that 1122 * if possible the dest node is not already occupied by some other 1123 * source node, minimizing the risk of overloading the memory on a 1124 * node that would happen if we migrated incoming memory to a node 1125 * before migrating outgoing memory source that same node. 1126 * 1127 * A single scan of tmp is sufficient. As we go, we remember the 1128 * most recent <s, d> pair that moved (s != d). If we find a pair 1129 * that not only moved, but what's better, moved to an empty slot 1130 * (d is not set in tmp), then we break out then, with that pair. 1131 * Otherwise when we finish scanning from_tmp, we at least have the 1132 * most recent <s, d> pair that moved. If we get all the way through 1133 * the scan of tmp without finding any node that moved, much less 1134 * moved to an empty node, then there is nothing left worth migrating. 1135 */ 1136 1137 tmp = *from; 1138 while (!nodes_empty(tmp)) { 1139 int s, d; 1140 int source = NUMA_NO_NODE; 1141 int dest = 0; 1142 1143 for_each_node_mask(s, tmp) { 1144 1145 /* 1146 * do_migrate_pages() tries to maintain the relative 1147 * node relationship of the pages established between 1148 * threads and memory areas. 1149 * 1150 * However if the number of source nodes is not equal to 1151 * the number of destination nodes we can not preserve 1152 * this node relative relationship. In that case, skip 1153 * copying memory from a node that is in the destination 1154 * mask. 1155 * 1156 * Example: [2,3,4] -> [3,4,5] moves everything. 1157 * [0-7] - > [3,4,5] moves only 0,1,2,6,7. 1158 */ 1159 1160 if ((nodes_weight(*from) != nodes_weight(*to)) && 1161 (node_isset(s, *to))) 1162 continue; 1163 1164 d = node_remap(s, *from, *to); 1165 if (s == d) 1166 continue; 1167 1168 source = s; /* Node moved. Memorize */ 1169 dest = d; 1170 1171 /* dest not in remaining from nodes? */ 1172 if (!node_isset(dest, tmp)) 1173 break; 1174 } 1175 if (source == NUMA_NO_NODE) 1176 break; 1177 1178 node_clear(source, tmp); 1179 err = migrate_to_node(mm, source, dest, flags); 1180 if (err > 0) 1181 busy += err; 1182 if (err < 0) 1183 break; 1184 } 1185 mmap_read_unlock(mm); 1186 1187 lru_cache_enable(); 1188 if (err < 0) 1189 return err; 1190 return busy; 1191 1192 } 1193 1194 /* 1195 * Allocate a new page for page migration based on vma policy. 1196 * Start by assuming the page is mapped by the same vma as contains @start. 1197 * Search forward from there, if not. N.B., this assumes that the 1198 * list of pages handed to migrate_pages()--which is how we get here-- 1199 * is in virtual address order. 1200 */ 1201 static struct page *new_page(struct page *page, unsigned long start) 1202 { 1203 struct vm_area_struct *vma; 1204 unsigned long address; 1205 1206 vma = find_vma(current->mm, start); 1207 while (vma) { 1208 address = page_address_in_vma(page, vma); 1209 if (address != -EFAULT) 1210 break; 1211 vma = vma->vm_next; 1212 } 1213 1214 if (PageHuge(page)) { 1215 return alloc_huge_page_vma(page_hstate(compound_head(page)), 1216 vma, address); 1217 } else if (PageTransHuge(page)) { 1218 struct page *thp; 1219 1220 thp = alloc_hugepage_vma(GFP_TRANSHUGE, vma, address, 1221 HPAGE_PMD_ORDER); 1222 if (!thp) 1223 return NULL; 1224 prep_transhuge_page(thp); 1225 return thp; 1226 } 1227 /* 1228 * if !vma, alloc_page_vma() will use task or system default policy 1229 */ 1230 return alloc_page_vma(GFP_HIGHUSER_MOVABLE | __GFP_RETRY_MAYFAIL, 1231 vma, address); 1232 } 1233 #else 1234 1235 static int migrate_page_add(struct page *page, struct list_head *pagelist, 1236 unsigned long flags) 1237 { 1238 return -EIO; 1239 } 1240 1241 int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from, 1242 const nodemask_t *to, int flags) 1243 { 1244 return -ENOSYS; 1245 } 1246 1247 static struct page *new_page(struct page *page, unsigned long start) 1248 { 1249 return NULL; 1250 } 1251 #endif 1252 1253 static long do_mbind(unsigned long start, unsigned long len, 1254 unsigned short mode, unsigned short mode_flags, 1255 nodemask_t *nmask, unsigned long flags) 1256 { 1257 struct mm_struct *mm = current->mm; 1258 struct mempolicy *new; 1259 unsigned long end; 1260 int err; 1261 int ret; 1262 LIST_HEAD(pagelist); 1263 1264 if (flags & ~(unsigned long)MPOL_MF_VALID) 1265 return -EINVAL; 1266 if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE)) 1267 return -EPERM; 1268 1269 if (start & ~PAGE_MASK) 1270 return -EINVAL; 1271 1272 if (mode == MPOL_DEFAULT) 1273 flags &= ~MPOL_MF_STRICT; 1274 1275 len = (len + PAGE_SIZE - 1) & PAGE_MASK; 1276 end = start + len; 1277 1278 if (end < start) 1279 return -EINVAL; 1280 if (end == start) 1281 return 0; 1282 1283 new = mpol_new(mode, mode_flags, nmask); 1284 if (IS_ERR(new)) 1285 return PTR_ERR(new); 1286 1287 if (flags & MPOL_MF_LAZY) 1288 new->flags |= MPOL_F_MOF; 1289 1290 /* 1291 * If we are using the default policy then operation 1292 * on discontinuous address spaces is okay after all 1293 */ 1294 if (!new) 1295 flags |= MPOL_MF_DISCONTIG_OK; 1296 1297 pr_debug("mbind %lx-%lx mode:%d flags:%d nodes:%lx\n", 1298 start, start + len, mode, mode_flags, 1299 nmask ? nodes_addr(*nmask)[0] : NUMA_NO_NODE); 1300 1301 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) { 1302 1303 lru_cache_disable(); 1304 } 1305 { 1306 NODEMASK_SCRATCH(scratch); 1307 if (scratch) { 1308 mmap_write_lock(mm); 1309 err = mpol_set_nodemask(new, nmask, scratch); 1310 if (err) 1311 mmap_write_unlock(mm); 1312 } else 1313 err = -ENOMEM; 1314 NODEMASK_SCRATCH_FREE(scratch); 1315 } 1316 if (err) 1317 goto mpol_out; 1318 1319 ret = queue_pages_range(mm, start, end, nmask, 1320 flags | MPOL_MF_INVERT, &pagelist); 1321 1322 if (ret < 0) { 1323 err = ret; 1324 goto up_out; 1325 } 1326 1327 err = mbind_range(mm, start, end, new); 1328 1329 if (!err) { 1330 int nr_failed = 0; 1331 1332 if (!list_empty(&pagelist)) { 1333 WARN_ON_ONCE(flags & MPOL_MF_LAZY); 1334 nr_failed = migrate_pages(&pagelist, new_page, NULL, 1335 start, MIGRATE_SYNC, MR_MEMPOLICY_MBIND, NULL); 1336 if (nr_failed) 1337 putback_movable_pages(&pagelist); 1338 } 1339 1340 if ((ret > 0) || (nr_failed && (flags & MPOL_MF_STRICT))) 1341 err = -EIO; 1342 } else { 1343 up_out: 1344 if (!list_empty(&pagelist)) 1345 putback_movable_pages(&pagelist); 1346 } 1347 1348 mmap_write_unlock(mm); 1349 mpol_out: 1350 mpol_put(new); 1351 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) 1352 lru_cache_enable(); 1353 return err; 1354 } 1355 1356 /* 1357 * User space interface with variable sized bitmaps for nodelists. 1358 */ 1359 static int get_bitmap(unsigned long *mask, const unsigned long __user *nmask, 1360 unsigned long maxnode) 1361 { 1362 unsigned long nlongs = BITS_TO_LONGS(maxnode); 1363 int ret; 1364 1365 if (in_compat_syscall()) 1366 ret = compat_get_bitmap(mask, 1367 (const compat_ulong_t __user *)nmask, 1368 maxnode); 1369 else 1370 ret = copy_from_user(mask, nmask, 1371 nlongs * sizeof(unsigned long)); 1372 1373 if (ret) 1374 return -EFAULT; 1375 1376 if (maxnode % BITS_PER_LONG) 1377 mask[nlongs - 1] &= (1UL << (maxnode % BITS_PER_LONG)) - 1; 1378 1379 return 0; 1380 } 1381 1382 /* Copy a node mask from user space. */ 1383 static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask, 1384 unsigned long maxnode) 1385 { 1386 --maxnode; 1387 nodes_clear(*nodes); 1388 if (maxnode == 0 || !nmask) 1389 return 0; 1390 if (maxnode > PAGE_SIZE*BITS_PER_BYTE) 1391 return -EINVAL; 1392 1393 /* 1394 * When the user specified more nodes than supported just check 1395 * if the non supported part is all zero, one word at a time, 1396 * starting at the end. 1397 */ 1398 while (maxnode > MAX_NUMNODES) { 1399 unsigned long bits = min_t(unsigned long, maxnode, BITS_PER_LONG); 1400 unsigned long t; 1401 1402 if (get_bitmap(&t, &nmask[maxnode / BITS_PER_LONG], bits)) 1403 return -EFAULT; 1404 1405 if (maxnode - bits >= MAX_NUMNODES) { 1406 maxnode -= bits; 1407 } else { 1408 maxnode = MAX_NUMNODES; 1409 t &= ~((1UL << (MAX_NUMNODES % BITS_PER_LONG)) - 1); 1410 } 1411 if (t) 1412 return -EINVAL; 1413 } 1414 1415 return get_bitmap(nodes_addr(*nodes), nmask, maxnode); 1416 } 1417 1418 /* Copy a kernel node mask to user space */ 1419 static int copy_nodes_to_user(unsigned long __user *mask, unsigned long maxnode, 1420 nodemask_t *nodes) 1421 { 1422 unsigned long copy = ALIGN(maxnode-1, 64) / 8; 1423 unsigned int nbytes = BITS_TO_LONGS(nr_node_ids) * sizeof(long); 1424 bool compat = in_compat_syscall(); 1425 1426 if (compat) 1427 nbytes = BITS_TO_COMPAT_LONGS(nr_node_ids) * sizeof(compat_long_t); 1428 1429 if (copy > nbytes) { 1430 if (copy > PAGE_SIZE) 1431 return -EINVAL; 1432 if (clear_user((char __user *)mask + nbytes, copy - nbytes)) 1433 return -EFAULT; 1434 copy = nbytes; 1435 maxnode = nr_node_ids; 1436 } 1437 1438 if (compat) 1439 return compat_put_bitmap((compat_ulong_t __user *)mask, 1440 nodes_addr(*nodes), maxnode); 1441 1442 return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0; 1443 } 1444 1445 /* Basic parameter sanity check used by both mbind() and set_mempolicy() */ 1446 static inline int sanitize_mpol_flags(int *mode, unsigned short *flags) 1447 { 1448 *flags = *mode & MPOL_MODE_FLAGS; 1449 *mode &= ~MPOL_MODE_FLAGS; 1450 1451 if ((unsigned int)(*mode) >= MPOL_MAX) 1452 return -EINVAL; 1453 if ((*flags & MPOL_F_STATIC_NODES) && (*flags & MPOL_F_RELATIVE_NODES)) 1454 return -EINVAL; 1455 if (*flags & MPOL_F_NUMA_BALANCING) { 1456 if (*mode != MPOL_BIND) 1457 return -EINVAL; 1458 *flags |= (MPOL_F_MOF | MPOL_F_MORON); 1459 } 1460 return 0; 1461 } 1462 1463 static long kernel_mbind(unsigned long start, unsigned long len, 1464 unsigned long mode, const unsigned long __user *nmask, 1465 unsigned long maxnode, unsigned int flags) 1466 { 1467 unsigned short mode_flags; 1468 nodemask_t nodes; 1469 int lmode = mode; 1470 int err; 1471 1472 start = untagged_addr(start); 1473 err = sanitize_mpol_flags(&lmode, &mode_flags); 1474 if (err) 1475 return err; 1476 1477 err = get_nodes(&nodes, nmask, maxnode); 1478 if (err) 1479 return err; 1480 1481 return do_mbind(start, len, lmode, mode_flags, &nodes, flags); 1482 } 1483 1484 SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, len, 1485 unsigned long, home_node, unsigned long, flags) 1486 { 1487 struct mm_struct *mm = current->mm; 1488 struct vm_area_struct *vma; 1489 struct mempolicy *new; 1490 unsigned long vmstart; 1491 unsigned long vmend; 1492 unsigned long end; 1493 int err = -ENOENT; 1494 1495 start = untagged_addr(start); 1496 if (start & ~PAGE_MASK) 1497 return -EINVAL; 1498 /* 1499 * flags is used for future extension if any. 1500 */ 1501 if (flags != 0) 1502 return -EINVAL; 1503 1504 /* 1505 * Check home_node is online to avoid accessing uninitialized 1506 * NODE_DATA. 1507 */ 1508 if (home_node >= MAX_NUMNODES || !node_online(home_node)) 1509 return -EINVAL; 1510 1511 len = (len + PAGE_SIZE - 1) & PAGE_MASK; 1512 end = start + len; 1513 1514 if (end < start) 1515 return -EINVAL; 1516 if (end == start) 1517 return 0; 1518 mmap_write_lock(mm); 1519 vma = find_vma(mm, start); 1520 for (; vma && vma->vm_start < end; vma = vma->vm_next) { 1521 1522 vmstart = max(start, vma->vm_start); 1523 vmend = min(end, vma->vm_end); 1524 new = mpol_dup(vma_policy(vma)); 1525 if (IS_ERR(new)) { 1526 err = PTR_ERR(new); 1527 break; 1528 } 1529 /* 1530 * Only update home node if there is an existing vma policy 1531 */ 1532 if (!new) 1533 continue; 1534 1535 /* 1536 * If any vma in the range got policy other than MPOL_BIND 1537 * or MPOL_PREFERRED_MANY we return error. We don't reset 1538 * the home node for vmas we already updated before. 1539 */ 1540 if (new->mode != MPOL_BIND && new->mode != MPOL_PREFERRED_MANY) { 1541 err = -EOPNOTSUPP; 1542 break; 1543 } 1544 1545 new->home_node = home_node; 1546 err = mbind_range(mm, vmstart, vmend, new); 1547 mpol_put(new); 1548 if (err) 1549 break; 1550 } 1551 mmap_write_unlock(mm); 1552 return err; 1553 } 1554 1555 SYSCALL_DEFINE6(mbind, unsigned long, start, unsigned long, len, 1556 unsigned long, mode, const unsigned long __user *, nmask, 1557 unsigned long, maxnode, unsigned int, flags) 1558 { 1559 return kernel_mbind(start, len, mode, nmask, maxnode, flags); 1560 } 1561 1562 /* Set the process memory policy */ 1563 static long kernel_set_mempolicy(int mode, const unsigned long __user *nmask, 1564 unsigned long maxnode) 1565 { 1566 unsigned short mode_flags; 1567 nodemask_t nodes; 1568 int lmode = mode; 1569 int err; 1570 1571 err = sanitize_mpol_flags(&lmode, &mode_flags); 1572 if (err) 1573 return err; 1574 1575 err = get_nodes(&nodes, nmask, maxnode); 1576 if (err) 1577 return err; 1578 1579 return do_set_mempolicy(lmode, mode_flags, &nodes); 1580 } 1581 1582 SYSCALL_DEFINE3(set_mempolicy, int, mode, const unsigned long __user *, nmask, 1583 unsigned long, maxnode) 1584 { 1585 return kernel_set_mempolicy(mode, nmask, maxnode); 1586 } 1587 1588 static int kernel_migrate_pages(pid_t pid, unsigned long maxnode, 1589 const unsigned long __user *old_nodes, 1590 const unsigned long __user *new_nodes) 1591 { 1592 struct mm_struct *mm = NULL; 1593 struct task_struct *task; 1594 nodemask_t task_nodes; 1595 int err; 1596 nodemask_t *old; 1597 nodemask_t *new; 1598 NODEMASK_SCRATCH(scratch); 1599 1600 if (!scratch) 1601 return -ENOMEM; 1602 1603 old = &scratch->mask1; 1604 new = &scratch->mask2; 1605 1606 err = get_nodes(old, old_nodes, maxnode); 1607 if (err) 1608 goto out; 1609 1610 err = get_nodes(new, new_nodes, maxnode); 1611 if (err) 1612 goto out; 1613 1614 /* Find the mm_struct */ 1615 rcu_read_lock(); 1616 task = pid ? find_task_by_vpid(pid) : current; 1617 if (!task) { 1618 rcu_read_unlock(); 1619 err = -ESRCH; 1620 goto out; 1621 } 1622 get_task_struct(task); 1623 1624 err = -EINVAL; 1625 1626 /* 1627 * Check if this process has the right to modify the specified process. 1628 * Use the regular "ptrace_may_access()" checks. 1629 */ 1630 if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) { 1631 rcu_read_unlock(); 1632 err = -EPERM; 1633 goto out_put; 1634 } 1635 rcu_read_unlock(); 1636 1637 task_nodes = cpuset_mems_allowed(task); 1638 /* Is the user allowed to access the target nodes? */ 1639 if (!nodes_subset(*new, task_nodes) && !capable(CAP_SYS_NICE)) { 1640 err = -EPERM; 1641 goto out_put; 1642 } 1643 1644 task_nodes = cpuset_mems_allowed(current); 1645 nodes_and(*new, *new, task_nodes); 1646 if (nodes_empty(*new)) 1647 goto out_put; 1648 1649 err = security_task_movememory(task); 1650 if (err) 1651 goto out_put; 1652 1653 mm = get_task_mm(task); 1654 put_task_struct(task); 1655 1656 if (!mm) { 1657 err = -EINVAL; 1658 goto out; 1659 } 1660 1661 err = do_migrate_pages(mm, old, new, 1662 capable(CAP_SYS_NICE) ? MPOL_MF_MOVE_ALL : MPOL_MF_MOVE); 1663 1664 mmput(mm); 1665 out: 1666 NODEMASK_SCRATCH_FREE(scratch); 1667 1668 return err; 1669 1670 out_put: 1671 put_task_struct(task); 1672 goto out; 1673 1674 } 1675 1676 SYSCALL_DEFINE4(migrate_pages, pid_t, pid, unsigned long, maxnode, 1677 const unsigned long __user *, old_nodes, 1678 const unsigned long __user *, new_nodes) 1679 { 1680 return kernel_migrate_pages(pid, maxnode, old_nodes, new_nodes); 1681 } 1682 1683 1684 /* Retrieve NUMA policy */ 1685 static int kernel_get_mempolicy(int __user *policy, 1686 unsigned long __user *nmask, 1687 unsigned long maxnode, 1688 unsigned long addr, 1689 unsigned long flags) 1690 { 1691 int err; 1692 int pval; 1693 nodemask_t nodes; 1694 1695 if (nmask != NULL && maxnode < nr_node_ids) 1696 return -EINVAL; 1697 1698 addr = untagged_addr(addr); 1699 1700 err = do_get_mempolicy(&pval, &nodes, addr, flags); 1701 1702 if (err) 1703 return err; 1704 1705 if (policy && put_user(pval, policy)) 1706 return -EFAULT; 1707 1708 if (nmask) 1709 err = copy_nodes_to_user(nmask, maxnode, &nodes); 1710 1711 return err; 1712 } 1713 1714 SYSCALL_DEFINE5(get_mempolicy, int __user *, policy, 1715 unsigned long __user *, nmask, unsigned long, maxnode, 1716 unsigned long, addr, unsigned long, flags) 1717 { 1718 return kernel_get_mempolicy(policy, nmask, maxnode, addr, flags); 1719 } 1720 1721 bool vma_migratable(struct vm_area_struct *vma) 1722 { 1723 if (vma->vm_flags & (VM_IO | VM_PFNMAP)) 1724 return false; 1725 1726 /* 1727 * DAX device mappings require predictable access latency, so avoid 1728 * incurring periodic faults. 1729 */ 1730 if (vma_is_dax(vma)) 1731 return false; 1732 1733 if (is_vm_hugetlb_page(vma) && 1734 !hugepage_migration_supported(hstate_vma(vma))) 1735 return false; 1736 1737 /* 1738 * Migration allocates pages in the highest zone. If we cannot 1739 * do so then migration (at least from node to node) is not 1740 * possible. 1741 */ 1742 if (vma->vm_file && 1743 gfp_zone(mapping_gfp_mask(vma->vm_file->f_mapping)) 1744 < policy_zone) 1745 return false; 1746 return true; 1747 } 1748 1749 struct mempolicy *__get_vma_policy(struct vm_area_struct *vma, 1750 unsigned long addr) 1751 { 1752 struct mempolicy *pol = NULL; 1753 1754 if (vma) { 1755 if (vma->vm_ops && vma->vm_ops->get_policy) { 1756 pol = vma->vm_ops->get_policy(vma, addr); 1757 } else if (vma->vm_policy) { 1758 pol = vma->vm_policy; 1759 1760 /* 1761 * shmem_alloc_page() passes MPOL_F_SHARED policy with 1762 * a pseudo vma whose vma->vm_ops=NULL. Take a reference 1763 * count on these policies which will be dropped by 1764 * mpol_cond_put() later 1765 */ 1766 if (mpol_needs_cond_ref(pol)) 1767 mpol_get(pol); 1768 } 1769 } 1770 1771 return pol; 1772 } 1773 1774 /* 1775 * get_vma_policy(@vma, @addr) 1776 * @vma: virtual memory area whose policy is sought 1777 * @addr: address in @vma for shared policy lookup 1778 * 1779 * Returns effective policy for a VMA at specified address. 1780 * Falls back to current->mempolicy or system default policy, as necessary. 1781 * Shared policies [those marked as MPOL_F_SHARED] require an extra reference 1782 * count--added by the get_policy() vm_op, as appropriate--to protect against 1783 * freeing by another task. It is the caller's responsibility to free the 1784 * extra reference for shared policies. 1785 */ 1786 static struct mempolicy *get_vma_policy(struct vm_area_struct *vma, 1787 unsigned long addr) 1788 { 1789 struct mempolicy *pol = __get_vma_policy(vma, addr); 1790 1791 if (!pol) 1792 pol = get_task_policy(current); 1793 1794 return pol; 1795 } 1796 1797 bool vma_policy_mof(struct vm_area_struct *vma) 1798 { 1799 struct mempolicy *pol; 1800 1801 if (vma->vm_ops && vma->vm_ops->get_policy) { 1802 bool ret = false; 1803 1804 pol = vma->vm_ops->get_policy(vma, vma->vm_start); 1805 if (pol && (pol->flags & MPOL_F_MOF)) 1806 ret = true; 1807 mpol_cond_put(pol); 1808 1809 return ret; 1810 } 1811 1812 pol = vma->vm_policy; 1813 if (!pol) 1814 pol = get_task_policy(current); 1815 1816 return pol->flags & MPOL_F_MOF; 1817 } 1818 1819 static int apply_policy_zone(struct mempolicy *policy, enum zone_type zone) 1820 { 1821 enum zone_type dynamic_policy_zone = policy_zone; 1822 1823 BUG_ON(dynamic_policy_zone == ZONE_MOVABLE); 1824 1825 /* 1826 * if policy->nodes has movable memory only, 1827 * we apply policy when gfp_zone(gfp) = ZONE_MOVABLE only. 1828 * 1829 * policy->nodes is intersect with node_states[N_MEMORY]. 1830 * so if the following test fails, it implies 1831 * policy->nodes has movable memory only. 1832 */ 1833 if (!nodes_intersects(policy->nodes, node_states[N_HIGH_MEMORY])) 1834 dynamic_policy_zone = ZONE_MOVABLE; 1835 1836 return zone >= dynamic_policy_zone; 1837 } 1838 1839 /* 1840 * Return a nodemask representing a mempolicy for filtering nodes for 1841 * page allocation 1842 */ 1843 nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *policy) 1844 { 1845 int mode = policy->mode; 1846 1847 /* Lower zones don't get a nodemask applied for MPOL_BIND */ 1848 if (unlikely(mode == MPOL_BIND) && 1849 apply_policy_zone(policy, gfp_zone(gfp)) && 1850 cpuset_nodemask_valid_mems_allowed(&policy->nodes)) 1851 return &policy->nodes; 1852 1853 if (mode == MPOL_PREFERRED_MANY) 1854 return &policy->nodes; 1855 1856 return NULL; 1857 } 1858 1859 /* 1860 * Return the preferred node id for 'prefer' mempolicy, and return 1861 * the given id for all other policies. 1862 * 1863 * policy_node() is always coupled with policy_nodemask(), which 1864 * secures the nodemask limit for 'bind' and 'prefer-many' policy. 1865 */ 1866 static int policy_node(gfp_t gfp, struct mempolicy *policy, int nd) 1867 { 1868 if (policy->mode == MPOL_PREFERRED) { 1869 nd = first_node(policy->nodes); 1870 } else { 1871 /* 1872 * __GFP_THISNODE shouldn't even be used with the bind policy 1873 * because we might easily break the expectation to stay on the 1874 * requested node and not break the policy. 1875 */ 1876 WARN_ON_ONCE(policy->mode == MPOL_BIND && (gfp & __GFP_THISNODE)); 1877 } 1878 1879 if ((policy->mode == MPOL_BIND || 1880 policy->mode == MPOL_PREFERRED_MANY) && 1881 policy->home_node != NUMA_NO_NODE) 1882 return policy->home_node; 1883 1884 return nd; 1885 } 1886 1887 /* Do dynamic interleaving for a process */ 1888 static unsigned interleave_nodes(struct mempolicy *policy) 1889 { 1890 unsigned next; 1891 struct task_struct *me = current; 1892 1893 next = next_node_in(me->il_prev, policy->nodes); 1894 if (next < MAX_NUMNODES) 1895 me->il_prev = next; 1896 return next; 1897 } 1898 1899 /* 1900 * Depending on the memory policy provide a node from which to allocate the 1901 * next slab entry. 1902 */ 1903 unsigned int mempolicy_slab_node(void) 1904 { 1905 struct mempolicy *policy; 1906 int node = numa_mem_id(); 1907 1908 if (!in_task()) 1909 return node; 1910 1911 policy = current->mempolicy; 1912 if (!policy) 1913 return node; 1914 1915 switch (policy->mode) { 1916 case MPOL_PREFERRED: 1917 return first_node(policy->nodes); 1918 1919 case MPOL_INTERLEAVE: 1920 return interleave_nodes(policy); 1921 1922 case MPOL_BIND: 1923 case MPOL_PREFERRED_MANY: 1924 { 1925 struct zoneref *z; 1926 1927 /* 1928 * Follow bind policy behavior and start allocation at the 1929 * first node. 1930 */ 1931 struct zonelist *zonelist; 1932 enum zone_type highest_zoneidx = gfp_zone(GFP_KERNEL); 1933 zonelist = &NODE_DATA(node)->node_zonelists[ZONELIST_FALLBACK]; 1934 z = first_zones_zonelist(zonelist, highest_zoneidx, 1935 &policy->nodes); 1936 return z->zone ? zone_to_nid(z->zone) : node; 1937 } 1938 case MPOL_LOCAL: 1939 return node; 1940 1941 default: 1942 BUG(); 1943 } 1944 } 1945 1946 /* 1947 * Do static interleaving for a VMA with known offset @n. Returns the n'th 1948 * node in pol->nodes (starting from n=0), wrapping around if n exceeds the 1949 * number of present nodes. 1950 */ 1951 static unsigned offset_il_node(struct mempolicy *pol, unsigned long n) 1952 { 1953 nodemask_t nodemask = pol->nodes; 1954 unsigned int target, nnodes; 1955 int i; 1956 int nid; 1957 /* 1958 * The barrier will stabilize the nodemask in a register or on 1959 * the stack so that it will stop changing under the code. 1960 * 1961 * Between first_node() and next_node(), pol->nodes could be changed 1962 * by other threads. So we put pol->nodes in a local stack. 1963 */ 1964 barrier(); 1965 1966 nnodes = nodes_weight(nodemask); 1967 if (!nnodes) 1968 return numa_node_id(); 1969 target = (unsigned int)n % nnodes; 1970 nid = first_node(nodemask); 1971 for (i = 0; i < target; i++) 1972 nid = next_node(nid, nodemask); 1973 return nid; 1974 } 1975 1976 /* Determine a node number for interleave */ 1977 static inline unsigned interleave_nid(struct mempolicy *pol, 1978 struct vm_area_struct *vma, unsigned long addr, int shift) 1979 { 1980 if (vma) { 1981 unsigned long off; 1982 1983 /* 1984 * for small pages, there is no difference between 1985 * shift and PAGE_SHIFT, so the bit-shift is safe. 1986 * for huge pages, since vm_pgoff is in units of small 1987 * pages, we need to shift off the always 0 bits to get 1988 * a useful offset. 1989 */ 1990 BUG_ON(shift < PAGE_SHIFT); 1991 off = vma->vm_pgoff >> (shift - PAGE_SHIFT); 1992 off += (addr - vma->vm_start) >> shift; 1993 return offset_il_node(pol, off); 1994 } else 1995 return interleave_nodes(pol); 1996 } 1997 1998 #ifdef CONFIG_HUGETLBFS 1999 /* 2000 * huge_node(@vma, @addr, @gfp_flags, @mpol) 2001 * @vma: virtual memory area whose policy is sought 2002 * @addr: address in @vma for shared policy lookup and interleave policy 2003 * @gfp_flags: for requested zone 2004 * @mpol: pointer to mempolicy pointer for reference counted mempolicy 2005 * @nodemask: pointer to nodemask pointer for 'bind' and 'prefer-many' policy 2006 * 2007 * Returns a nid suitable for a huge page allocation and a pointer 2008 * to the struct mempolicy for conditional unref after allocation. 2009 * If the effective policy is 'bind' or 'prefer-many', returns a pointer 2010 * to the mempolicy's @nodemask for filtering the zonelist. 2011 * 2012 * Must be protected by read_mems_allowed_begin() 2013 */ 2014 int huge_node(struct vm_area_struct *vma, unsigned long addr, gfp_t gfp_flags, 2015 struct mempolicy **mpol, nodemask_t **nodemask) 2016 { 2017 int nid; 2018 int mode; 2019 2020 *mpol = get_vma_policy(vma, addr); 2021 *nodemask = NULL; 2022 mode = (*mpol)->mode; 2023 2024 if (unlikely(mode == MPOL_INTERLEAVE)) { 2025 nid = interleave_nid(*mpol, vma, addr, 2026 huge_page_shift(hstate_vma(vma))); 2027 } else { 2028 nid = policy_node(gfp_flags, *mpol, numa_node_id()); 2029 if (mode == MPOL_BIND || mode == MPOL_PREFERRED_MANY) 2030 *nodemask = &(*mpol)->nodes; 2031 } 2032 return nid; 2033 } 2034 2035 /* 2036 * init_nodemask_of_mempolicy 2037 * 2038 * If the current task's mempolicy is "default" [NULL], return 'false' 2039 * to indicate default policy. Otherwise, extract the policy nodemask 2040 * for 'bind' or 'interleave' policy into the argument nodemask, or 2041 * initialize the argument nodemask to contain the single node for 2042 * 'preferred' or 'local' policy and return 'true' to indicate presence 2043 * of non-default mempolicy. 2044 * 2045 * We don't bother with reference counting the mempolicy [mpol_get/put] 2046 * because the current task is examining it's own mempolicy and a task's 2047 * mempolicy is only ever changed by the task itself. 2048 * 2049 * N.B., it is the caller's responsibility to free a returned nodemask. 2050 */ 2051 bool init_nodemask_of_mempolicy(nodemask_t *mask) 2052 { 2053 struct mempolicy *mempolicy; 2054 2055 if (!(mask && current->mempolicy)) 2056 return false; 2057 2058 task_lock(current); 2059 mempolicy = current->mempolicy; 2060 switch (mempolicy->mode) { 2061 case MPOL_PREFERRED: 2062 case MPOL_PREFERRED_MANY: 2063 case MPOL_BIND: 2064 case MPOL_INTERLEAVE: 2065 *mask = mempolicy->nodes; 2066 break; 2067 2068 case MPOL_LOCAL: 2069 init_nodemask_of_node(mask, numa_node_id()); 2070 break; 2071 2072 default: 2073 BUG(); 2074 } 2075 task_unlock(current); 2076 2077 return true; 2078 } 2079 #endif 2080 2081 /* 2082 * mempolicy_in_oom_domain 2083 * 2084 * If tsk's mempolicy is "bind", check for intersection between mask and 2085 * the policy nodemask. Otherwise, return true for all other policies 2086 * including "interleave", as a tsk with "interleave" policy may have 2087 * memory allocated from all nodes in system. 2088 * 2089 * Takes task_lock(tsk) to prevent freeing of its mempolicy. 2090 */ 2091 bool mempolicy_in_oom_domain(struct task_struct *tsk, 2092 const nodemask_t *mask) 2093 { 2094 struct mempolicy *mempolicy; 2095 bool ret = true; 2096 2097 if (!mask) 2098 return ret; 2099 2100 task_lock(tsk); 2101 mempolicy = tsk->mempolicy; 2102 if (mempolicy && mempolicy->mode == MPOL_BIND) 2103 ret = nodes_intersects(mempolicy->nodes, *mask); 2104 task_unlock(tsk); 2105 2106 return ret; 2107 } 2108 2109 /* Allocate a page in interleaved policy. 2110 Own path because it needs to do special accounting. */ 2111 static struct page *alloc_page_interleave(gfp_t gfp, unsigned order, 2112 unsigned nid) 2113 { 2114 struct page *page; 2115 2116 page = __alloc_pages(gfp, order, nid, NULL); 2117 /* skip NUMA_INTERLEAVE_HIT counter update if numa stats is disabled */ 2118 if (!static_branch_likely(&vm_numa_stat_key)) 2119 return page; 2120 if (page && page_to_nid(page) == nid) { 2121 preempt_disable(); 2122 __count_numa_event(page_zone(page), NUMA_INTERLEAVE_HIT); 2123 preempt_enable(); 2124 } 2125 return page; 2126 } 2127 2128 static struct page *alloc_pages_preferred_many(gfp_t gfp, unsigned int order, 2129 int nid, struct mempolicy *pol) 2130 { 2131 struct page *page; 2132 gfp_t preferred_gfp; 2133 2134 /* 2135 * This is a two pass approach. The first pass will only try the 2136 * preferred nodes but skip the direct reclaim and allow the 2137 * allocation to fail, while the second pass will try all the 2138 * nodes in system. 2139 */ 2140 preferred_gfp = gfp | __GFP_NOWARN; 2141 preferred_gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL); 2142 page = __alloc_pages(preferred_gfp, order, nid, &pol->nodes); 2143 if (!page) 2144 page = __alloc_pages(gfp, order, nid, NULL); 2145 2146 return page; 2147 } 2148 2149 /** 2150 * alloc_pages_vma - Allocate a page for a VMA. 2151 * @gfp: GFP flags. 2152 * @order: Order of the GFP allocation. 2153 * @vma: Pointer to VMA or NULL if not available. 2154 * @addr: Virtual address of the allocation. Must be inside @vma. 2155 * @hugepage: For hugepages try only the preferred node if possible. 2156 * 2157 * Allocate a page for a specific address in @vma, using the appropriate 2158 * NUMA policy. When @vma is not NULL the caller must hold the mmap_lock 2159 * of the mm_struct of the VMA to prevent it from going away. Should be 2160 * used for all allocations for pages that will be mapped into user space. 2161 * 2162 * Return: The page on success or NULL if allocation fails. 2163 */ 2164 struct page *alloc_pages_vma(gfp_t gfp, int order, struct vm_area_struct *vma, 2165 unsigned long addr, bool hugepage) 2166 { 2167 struct mempolicy *pol; 2168 int node = numa_node_id(); 2169 struct page *page; 2170 int preferred_nid; 2171 nodemask_t *nmask; 2172 2173 pol = get_vma_policy(vma, addr); 2174 2175 if (pol->mode == MPOL_INTERLEAVE) { 2176 unsigned nid; 2177 2178 nid = interleave_nid(pol, vma, addr, PAGE_SHIFT + order); 2179 mpol_cond_put(pol); 2180 page = alloc_page_interleave(gfp, order, nid); 2181 goto out; 2182 } 2183 2184 if (pol->mode == MPOL_PREFERRED_MANY) { 2185 node = policy_node(gfp, pol, node); 2186 page = alloc_pages_preferred_many(gfp, order, node, pol); 2187 mpol_cond_put(pol); 2188 goto out; 2189 } 2190 2191 if (unlikely(IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && hugepage)) { 2192 int hpage_node = node; 2193 2194 /* 2195 * For hugepage allocation and non-interleave policy which 2196 * allows the current node (or other explicitly preferred 2197 * node) we only try to allocate from the current/preferred 2198 * node and don't fall back to other nodes, as the cost of 2199 * remote accesses would likely offset THP benefits. 2200 * 2201 * If the policy is interleave or does not allow the current 2202 * node in its nodemask, we allocate the standard way. 2203 */ 2204 if (pol->mode == MPOL_PREFERRED) 2205 hpage_node = first_node(pol->nodes); 2206 2207 nmask = policy_nodemask(gfp, pol); 2208 if (!nmask || node_isset(hpage_node, *nmask)) { 2209 mpol_cond_put(pol); 2210 /* 2211 * First, try to allocate THP only on local node, but 2212 * don't reclaim unnecessarily, just compact. 2213 */ 2214 page = __alloc_pages_node(hpage_node, 2215 gfp | __GFP_THISNODE | __GFP_NORETRY, order); 2216 2217 /* 2218 * If hugepage allocations are configured to always 2219 * synchronous compact or the vma has been madvised 2220 * to prefer hugepage backing, retry allowing remote 2221 * memory with both reclaim and compact as well. 2222 */ 2223 if (!page && (gfp & __GFP_DIRECT_RECLAIM)) 2224 page = __alloc_pages(gfp, order, hpage_node, nmask); 2225 2226 goto out; 2227 } 2228 } 2229 2230 nmask = policy_nodemask(gfp, pol); 2231 preferred_nid = policy_node(gfp, pol, node); 2232 page = __alloc_pages(gfp, order, preferred_nid, nmask); 2233 mpol_cond_put(pol); 2234 out: 2235 return page; 2236 } 2237 EXPORT_SYMBOL(alloc_pages_vma); 2238 2239 /** 2240 * alloc_pages - Allocate pages. 2241 * @gfp: GFP flags. 2242 * @order: Power of two of number of pages to allocate. 2243 * 2244 * Allocate 1 << @order contiguous pages. The physical address of the 2245 * first page is naturally aligned (eg an order-3 allocation will be aligned 2246 * to a multiple of 8 * PAGE_SIZE bytes). The NUMA policy of the current 2247 * process is honoured when in process context. 2248 * 2249 * Context: Can be called from any context, providing the appropriate GFP 2250 * flags are used. 2251 * Return: The page on success or NULL if allocation fails. 2252 */ 2253 struct page *alloc_pages(gfp_t gfp, unsigned order) 2254 { 2255 struct mempolicy *pol = &default_policy; 2256 struct page *page; 2257 2258 if (!in_interrupt() && !(gfp & __GFP_THISNODE)) 2259 pol = get_task_policy(current); 2260 2261 /* 2262 * No reference counting needed for current->mempolicy 2263 * nor system default_policy 2264 */ 2265 if (pol->mode == MPOL_INTERLEAVE) 2266 page = alloc_page_interleave(gfp, order, interleave_nodes(pol)); 2267 else if (pol->mode == MPOL_PREFERRED_MANY) 2268 page = alloc_pages_preferred_many(gfp, order, 2269 policy_node(gfp, pol, numa_node_id()), pol); 2270 else 2271 page = __alloc_pages(gfp, order, 2272 policy_node(gfp, pol, numa_node_id()), 2273 policy_nodemask(gfp, pol)); 2274 2275 return page; 2276 } 2277 EXPORT_SYMBOL(alloc_pages); 2278 2279 struct folio *folio_alloc(gfp_t gfp, unsigned order) 2280 { 2281 struct page *page = alloc_pages(gfp | __GFP_COMP, order); 2282 2283 if (page && order > 1) 2284 prep_transhuge_page(page); 2285 return (struct folio *)page; 2286 } 2287 EXPORT_SYMBOL(folio_alloc); 2288 2289 static unsigned long alloc_pages_bulk_array_interleave(gfp_t gfp, 2290 struct mempolicy *pol, unsigned long nr_pages, 2291 struct page **page_array) 2292 { 2293 int nodes; 2294 unsigned long nr_pages_per_node; 2295 int delta; 2296 int i; 2297 unsigned long nr_allocated; 2298 unsigned long total_allocated = 0; 2299 2300 nodes = nodes_weight(pol->nodes); 2301 nr_pages_per_node = nr_pages / nodes; 2302 delta = nr_pages - nodes * nr_pages_per_node; 2303 2304 for (i = 0; i < nodes; i++) { 2305 if (delta) { 2306 nr_allocated = __alloc_pages_bulk(gfp, 2307 interleave_nodes(pol), NULL, 2308 nr_pages_per_node + 1, NULL, 2309 page_array); 2310 delta--; 2311 } else { 2312 nr_allocated = __alloc_pages_bulk(gfp, 2313 interleave_nodes(pol), NULL, 2314 nr_pages_per_node, NULL, page_array); 2315 } 2316 2317 page_array += nr_allocated; 2318 total_allocated += nr_allocated; 2319 } 2320 2321 return total_allocated; 2322 } 2323 2324 static unsigned long alloc_pages_bulk_array_preferred_many(gfp_t gfp, int nid, 2325 struct mempolicy *pol, unsigned long nr_pages, 2326 struct page **page_array) 2327 { 2328 gfp_t preferred_gfp; 2329 unsigned long nr_allocated = 0; 2330 2331 preferred_gfp = gfp | __GFP_NOWARN; 2332 preferred_gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL); 2333 2334 nr_allocated = __alloc_pages_bulk(preferred_gfp, nid, &pol->nodes, 2335 nr_pages, NULL, page_array); 2336 2337 if (nr_allocated < nr_pages) 2338 nr_allocated += __alloc_pages_bulk(gfp, numa_node_id(), NULL, 2339 nr_pages - nr_allocated, NULL, 2340 page_array + nr_allocated); 2341 return nr_allocated; 2342 } 2343 2344 /* alloc pages bulk and mempolicy should be considered at the 2345 * same time in some situation such as vmalloc. 2346 * 2347 * It can accelerate memory allocation especially interleaving 2348 * allocate memory. 2349 */ 2350 unsigned long alloc_pages_bulk_array_mempolicy(gfp_t gfp, 2351 unsigned long nr_pages, struct page **page_array) 2352 { 2353 struct mempolicy *pol = &default_policy; 2354 2355 if (!in_interrupt() && !(gfp & __GFP_THISNODE)) 2356 pol = get_task_policy(current); 2357 2358 if (pol->mode == MPOL_INTERLEAVE) 2359 return alloc_pages_bulk_array_interleave(gfp, pol, 2360 nr_pages, page_array); 2361 2362 if (pol->mode == MPOL_PREFERRED_MANY) 2363 return alloc_pages_bulk_array_preferred_many(gfp, 2364 numa_node_id(), pol, nr_pages, page_array); 2365 2366 return __alloc_pages_bulk(gfp, policy_node(gfp, pol, numa_node_id()), 2367 policy_nodemask(gfp, pol), nr_pages, NULL, 2368 page_array); 2369 } 2370 2371 int vma_dup_policy(struct vm_area_struct *src, struct vm_area_struct *dst) 2372 { 2373 struct mempolicy *pol = mpol_dup(vma_policy(src)); 2374 2375 if (IS_ERR(pol)) 2376 return PTR_ERR(pol); 2377 dst->vm_policy = pol; 2378 return 0; 2379 } 2380 2381 /* 2382 * If mpol_dup() sees current->cpuset == cpuset_being_rebound, then it 2383 * rebinds the mempolicy its copying by calling mpol_rebind_policy() 2384 * with the mems_allowed returned by cpuset_mems_allowed(). This 2385 * keeps mempolicies cpuset relative after its cpuset moves. See 2386 * further kernel/cpuset.c update_nodemask(). 2387 * 2388 * current's mempolicy may be rebinded by the other task(the task that changes 2389 * cpuset's mems), so we needn't do rebind work for current task. 2390 */ 2391 2392 /* Slow path of a mempolicy duplicate */ 2393 struct mempolicy *__mpol_dup(struct mempolicy *old) 2394 { 2395 struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL); 2396 2397 if (!new) 2398 return ERR_PTR(-ENOMEM); 2399 2400 /* task's mempolicy is protected by alloc_lock */ 2401 if (old == current->mempolicy) { 2402 task_lock(current); 2403 *new = *old; 2404 task_unlock(current); 2405 } else 2406 *new = *old; 2407 2408 if (current_cpuset_is_being_rebound()) { 2409 nodemask_t mems = cpuset_mems_allowed(current); 2410 mpol_rebind_policy(new, &mems); 2411 } 2412 atomic_set(&new->refcnt, 1); 2413 return new; 2414 } 2415 2416 /* Slow path of a mempolicy comparison */ 2417 bool __mpol_equal(struct mempolicy *a, struct mempolicy *b) 2418 { 2419 if (!a || !b) 2420 return false; 2421 if (a->mode != b->mode) 2422 return false; 2423 if (a->flags != b->flags) 2424 return false; 2425 if (a->home_node != b->home_node) 2426 return false; 2427 if (mpol_store_user_nodemask(a)) 2428 if (!nodes_equal(a->w.user_nodemask, b->w.user_nodemask)) 2429 return false; 2430 2431 switch (a->mode) { 2432 case MPOL_BIND: 2433 case MPOL_INTERLEAVE: 2434 case MPOL_PREFERRED: 2435 case MPOL_PREFERRED_MANY: 2436 return !!nodes_equal(a->nodes, b->nodes); 2437 case MPOL_LOCAL: 2438 return true; 2439 default: 2440 BUG(); 2441 return false; 2442 } 2443 } 2444 2445 /* 2446 * Shared memory backing store policy support. 2447 * 2448 * Remember policies even when nobody has shared memory mapped. 2449 * The policies are kept in Red-Black tree linked from the inode. 2450 * They are protected by the sp->lock rwlock, which should be held 2451 * for any accesses to the tree. 2452 */ 2453 2454 /* 2455 * lookup first element intersecting start-end. Caller holds sp->lock for 2456 * reading or for writing 2457 */ 2458 static struct sp_node * 2459 sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end) 2460 { 2461 struct rb_node *n = sp->root.rb_node; 2462 2463 while (n) { 2464 struct sp_node *p = rb_entry(n, struct sp_node, nd); 2465 2466 if (start >= p->end) 2467 n = n->rb_right; 2468 else if (end <= p->start) 2469 n = n->rb_left; 2470 else 2471 break; 2472 } 2473 if (!n) 2474 return NULL; 2475 for (;;) { 2476 struct sp_node *w = NULL; 2477 struct rb_node *prev = rb_prev(n); 2478 if (!prev) 2479 break; 2480 w = rb_entry(prev, struct sp_node, nd); 2481 if (w->end <= start) 2482 break; 2483 n = prev; 2484 } 2485 return rb_entry(n, struct sp_node, nd); 2486 } 2487 2488 /* 2489 * Insert a new shared policy into the list. Caller holds sp->lock for 2490 * writing. 2491 */ 2492 static void sp_insert(struct shared_policy *sp, struct sp_node *new) 2493 { 2494 struct rb_node **p = &sp->root.rb_node; 2495 struct rb_node *parent = NULL; 2496 struct sp_node *nd; 2497 2498 while (*p) { 2499 parent = *p; 2500 nd = rb_entry(parent, struct sp_node, nd); 2501 if (new->start < nd->start) 2502 p = &(*p)->rb_left; 2503 else if (new->end > nd->end) 2504 p = &(*p)->rb_right; 2505 else 2506 BUG(); 2507 } 2508 rb_link_node(&new->nd, parent, p); 2509 rb_insert_color(&new->nd, &sp->root); 2510 pr_debug("inserting %lx-%lx: %d\n", new->start, new->end, 2511 new->policy ? new->policy->mode : 0); 2512 } 2513 2514 /* Find shared policy intersecting idx */ 2515 struct mempolicy * 2516 mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx) 2517 { 2518 struct mempolicy *pol = NULL; 2519 struct sp_node *sn; 2520 2521 if (!sp->root.rb_node) 2522 return NULL; 2523 read_lock(&sp->lock); 2524 sn = sp_lookup(sp, idx, idx+1); 2525 if (sn) { 2526 mpol_get(sn->policy); 2527 pol = sn->policy; 2528 } 2529 read_unlock(&sp->lock); 2530 return pol; 2531 } 2532 2533 static void sp_free(struct sp_node *n) 2534 { 2535 mpol_put(n->policy); 2536 kmem_cache_free(sn_cache, n); 2537 } 2538 2539 /** 2540 * mpol_misplaced - check whether current page node is valid in policy 2541 * 2542 * @page: page to be checked 2543 * @vma: vm area where page mapped 2544 * @addr: virtual address where page mapped 2545 * 2546 * Lookup current policy node id for vma,addr and "compare to" page's 2547 * node id. Policy determination "mimics" alloc_page_vma(). 2548 * Called from fault path where we know the vma and faulting address. 2549 * 2550 * Return: NUMA_NO_NODE if the page is in a node that is valid for this 2551 * policy, or a suitable node ID to allocate a replacement page from. 2552 */ 2553 int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long addr) 2554 { 2555 struct mempolicy *pol; 2556 struct zoneref *z; 2557 int curnid = page_to_nid(page); 2558 unsigned long pgoff; 2559 int thiscpu = raw_smp_processor_id(); 2560 int thisnid = cpu_to_node(thiscpu); 2561 int polnid = NUMA_NO_NODE; 2562 int ret = NUMA_NO_NODE; 2563 2564 pol = get_vma_policy(vma, addr); 2565 if (!(pol->flags & MPOL_F_MOF)) 2566 goto out; 2567 2568 switch (pol->mode) { 2569 case MPOL_INTERLEAVE: 2570 pgoff = vma->vm_pgoff; 2571 pgoff += (addr - vma->vm_start) >> PAGE_SHIFT; 2572 polnid = offset_il_node(pol, pgoff); 2573 break; 2574 2575 case MPOL_PREFERRED: 2576 if (node_isset(curnid, pol->nodes)) 2577 goto out; 2578 polnid = first_node(pol->nodes); 2579 break; 2580 2581 case MPOL_LOCAL: 2582 polnid = numa_node_id(); 2583 break; 2584 2585 case MPOL_BIND: 2586 /* Optimize placement among multiple nodes via NUMA balancing */ 2587 if (pol->flags & MPOL_F_MORON) { 2588 if (node_isset(thisnid, pol->nodes)) 2589 break; 2590 goto out; 2591 } 2592 fallthrough; 2593 2594 case MPOL_PREFERRED_MANY: 2595 /* 2596 * use current page if in policy nodemask, 2597 * else select nearest allowed node, if any. 2598 * If no allowed nodes, use current [!misplaced]. 2599 */ 2600 if (node_isset(curnid, pol->nodes)) 2601 goto out; 2602 z = first_zones_zonelist( 2603 node_zonelist(numa_node_id(), GFP_HIGHUSER), 2604 gfp_zone(GFP_HIGHUSER), 2605 &pol->nodes); 2606 polnid = zone_to_nid(z->zone); 2607 break; 2608 2609 default: 2610 BUG(); 2611 } 2612 2613 /* Migrate the page towards the node whose CPU is referencing it */ 2614 if (pol->flags & MPOL_F_MORON) { 2615 polnid = thisnid; 2616 2617 if (!should_numa_migrate_memory(current, page, curnid, thiscpu)) 2618 goto out; 2619 } 2620 2621 if (curnid != polnid) 2622 ret = polnid; 2623 out: 2624 mpol_cond_put(pol); 2625 2626 return ret; 2627 } 2628 2629 /* 2630 * Drop the (possibly final) reference to task->mempolicy. It needs to be 2631 * dropped after task->mempolicy is set to NULL so that any allocation done as 2632 * part of its kmem_cache_free(), such as by KASAN, doesn't reference a freed 2633 * policy. 2634 */ 2635 void mpol_put_task_policy(struct task_struct *task) 2636 { 2637 struct mempolicy *pol; 2638 2639 task_lock(task); 2640 pol = task->mempolicy; 2641 task->mempolicy = NULL; 2642 task_unlock(task); 2643 mpol_put(pol); 2644 } 2645 2646 static void sp_delete(struct shared_policy *sp, struct sp_node *n) 2647 { 2648 pr_debug("deleting %lx-l%lx\n", n->start, n->end); 2649 rb_erase(&n->nd, &sp->root); 2650 sp_free(n); 2651 } 2652 2653 static void sp_node_init(struct sp_node *node, unsigned long start, 2654 unsigned long end, struct mempolicy *pol) 2655 { 2656 node->start = start; 2657 node->end = end; 2658 node->policy = pol; 2659 } 2660 2661 static struct sp_node *sp_alloc(unsigned long start, unsigned long end, 2662 struct mempolicy *pol) 2663 { 2664 struct sp_node *n; 2665 struct mempolicy *newpol; 2666 2667 n = kmem_cache_alloc(sn_cache, GFP_KERNEL); 2668 if (!n) 2669 return NULL; 2670 2671 newpol = mpol_dup(pol); 2672 if (IS_ERR(newpol)) { 2673 kmem_cache_free(sn_cache, n); 2674 return NULL; 2675 } 2676 newpol->flags |= MPOL_F_SHARED; 2677 sp_node_init(n, start, end, newpol); 2678 2679 return n; 2680 } 2681 2682 /* Replace a policy range. */ 2683 static int shared_policy_replace(struct shared_policy *sp, unsigned long start, 2684 unsigned long end, struct sp_node *new) 2685 { 2686 struct sp_node *n; 2687 struct sp_node *n_new = NULL; 2688 struct mempolicy *mpol_new = NULL; 2689 int ret = 0; 2690 2691 restart: 2692 write_lock(&sp->lock); 2693 n = sp_lookup(sp, start, end); 2694 /* Take care of old policies in the same range. */ 2695 while (n && n->start < end) { 2696 struct rb_node *next = rb_next(&n->nd); 2697 if (n->start >= start) { 2698 if (n->end <= end) 2699 sp_delete(sp, n); 2700 else 2701 n->start = end; 2702 } else { 2703 /* Old policy spanning whole new range. */ 2704 if (n->end > end) { 2705 if (!n_new) 2706 goto alloc_new; 2707 2708 *mpol_new = *n->policy; 2709 atomic_set(&mpol_new->refcnt, 1); 2710 sp_node_init(n_new, end, n->end, mpol_new); 2711 n->end = start; 2712 sp_insert(sp, n_new); 2713 n_new = NULL; 2714 mpol_new = NULL; 2715 break; 2716 } else 2717 n->end = start; 2718 } 2719 if (!next) 2720 break; 2721 n = rb_entry(next, struct sp_node, nd); 2722 } 2723 if (new) 2724 sp_insert(sp, new); 2725 write_unlock(&sp->lock); 2726 ret = 0; 2727 2728 err_out: 2729 if (mpol_new) 2730 mpol_put(mpol_new); 2731 if (n_new) 2732 kmem_cache_free(sn_cache, n_new); 2733 2734 return ret; 2735 2736 alloc_new: 2737 write_unlock(&sp->lock); 2738 ret = -ENOMEM; 2739 n_new = kmem_cache_alloc(sn_cache, GFP_KERNEL); 2740 if (!n_new) 2741 goto err_out; 2742 mpol_new = kmem_cache_alloc(policy_cache, GFP_KERNEL); 2743 if (!mpol_new) 2744 goto err_out; 2745 goto restart; 2746 } 2747 2748 /** 2749 * mpol_shared_policy_init - initialize shared policy for inode 2750 * @sp: pointer to inode shared policy 2751 * @mpol: struct mempolicy to install 2752 * 2753 * Install non-NULL @mpol in inode's shared policy rb-tree. 2754 * On entry, the current task has a reference on a non-NULL @mpol. 2755 * This must be released on exit. 2756 * This is called at get_inode() calls and we can use GFP_KERNEL. 2757 */ 2758 void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol) 2759 { 2760 int ret; 2761 2762 sp->root = RB_ROOT; /* empty tree == default mempolicy */ 2763 rwlock_init(&sp->lock); 2764 2765 if (mpol) { 2766 struct vm_area_struct pvma; 2767 struct mempolicy *new; 2768 NODEMASK_SCRATCH(scratch); 2769 2770 if (!scratch) 2771 goto put_mpol; 2772 /* contextualize the tmpfs mount point mempolicy */ 2773 new = mpol_new(mpol->mode, mpol->flags, &mpol->w.user_nodemask); 2774 if (IS_ERR(new)) 2775 goto free_scratch; /* no valid nodemask intersection */ 2776 2777 task_lock(current); 2778 ret = mpol_set_nodemask(new, &mpol->w.user_nodemask, scratch); 2779 task_unlock(current); 2780 if (ret) 2781 goto put_new; 2782 2783 /* Create pseudo-vma that contains just the policy */ 2784 vma_init(&pvma, NULL); 2785 pvma.vm_end = TASK_SIZE; /* policy covers entire file */ 2786 mpol_set_shared_policy(sp, &pvma, new); /* adds ref */ 2787 2788 put_new: 2789 mpol_put(new); /* drop initial ref */ 2790 free_scratch: 2791 NODEMASK_SCRATCH_FREE(scratch); 2792 put_mpol: 2793 mpol_put(mpol); /* drop our incoming ref on sb mpol */ 2794 } 2795 } 2796 2797 int mpol_set_shared_policy(struct shared_policy *info, 2798 struct vm_area_struct *vma, struct mempolicy *npol) 2799 { 2800 int err; 2801 struct sp_node *new = NULL; 2802 unsigned long sz = vma_pages(vma); 2803 2804 pr_debug("set_shared_policy %lx sz %lu %d %d %lx\n", 2805 vma->vm_pgoff, 2806 sz, npol ? npol->mode : -1, 2807 npol ? npol->flags : -1, 2808 npol ? nodes_addr(npol->nodes)[0] : NUMA_NO_NODE); 2809 2810 if (npol) { 2811 new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, npol); 2812 if (!new) 2813 return -ENOMEM; 2814 } 2815 err = shared_policy_replace(info, vma->vm_pgoff, vma->vm_pgoff+sz, new); 2816 if (err && new) 2817 sp_free(new); 2818 return err; 2819 } 2820 2821 /* Free a backing policy store on inode delete. */ 2822 void mpol_free_shared_policy(struct shared_policy *p) 2823 { 2824 struct sp_node *n; 2825 struct rb_node *next; 2826 2827 if (!p->root.rb_node) 2828 return; 2829 write_lock(&p->lock); 2830 next = rb_first(&p->root); 2831 while (next) { 2832 n = rb_entry(next, struct sp_node, nd); 2833 next = rb_next(&n->nd); 2834 sp_delete(p, n); 2835 } 2836 write_unlock(&p->lock); 2837 } 2838 2839 #ifdef CONFIG_NUMA_BALANCING 2840 static int __initdata numabalancing_override; 2841 2842 static void __init check_numabalancing_enable(void) 2843 { 2844 bool numabalancing_default = false; 2845 2846 if (IS_ENABLED(CONFIG_NUMA_BALANCING_DEFAULT_ENABLED)) 2847 numabalancing_default = true; 2848 2849 /* Parsed by setup_numabalancing. override == 1 enables, -1 disables */ 2850 if (numabalancing_override) 2851 set_numabalancing_state(numabalancing_override == 1); 2852 2853 if (num_online_nodes() > 1 && !numabalancing_override) { 2854 pr_info("%s automatic NUMA balancing. Configure with numa_balancing= or the kernel.numa_balancing sysctl\n", 2855 numabalancing_default ? "Enabling" : "Disabling"); 2856 set_numabalancing_state(numabalancing_default); 2857 } 2858 } 2859 2860 static int __init setup_numabalancing(char *str) 2861 { 2862 int ret = 0; 2863 if (!str) 2864 goto out; 2865 2866 if (!strcmp(str, "enable")) { 2867 numabalancing_override = 1; 2868 ret = 1; 2869 } else if (!strcmp(str, "disable")) { 2870 numabalancing_override = -1; 2871 ret = 1; 2872 } 2873 out: 2874 if (!ret) 2875 pr_warn("Unable to parse numa_balancing=\n"); 2876 2877 return ret; 2878 } 2879 __setup("numa_balancing=", setup_numabalancing); 2880 #else 2881 static inline void __init check_numabalancing_enable(void) 2882 { 2883 } 2884 #endif /* CONFIG_NUMA_BALANCING */ 2885 2886 /* assumes fs == KERNEL_DS */ 2887 void __init numa_policy_init(void) 2888 { 2889 nodemask_t interleave_nodes; 2890 unsigned long largest = 0; 2891 int nid, prefer = 0; 2892 2893 policy_cache = kmem_cache_create("numa_policy", 2894 sizeof(struct mempolicy), 2895 0, SLAB_PANIC, NULL); 2896 2897 sn_cache = kmem_cache_create("shared_policy_node", 2898 sizeof(struct sp_node), 2899 0, SLAB_PANIC, NULL); 2900 2901 for_each_node(nid) { 2902 preferred_node_policy[nid] = (struct mempolicy) { 2903 .refcnt = ATOMIC_INIT(1), 2904 .mode = MPOL_PREFERRED, 2905 .flags = MPOL_F_MOF | MPOL_F_MORON, 2906 .nodes = nodemask_of_node(nid), 2907 }; 2908 } 2909 2910 /* 2911 * Set interleaving policy for system init. Interleaving is only 2912 * enabled across suitably sized nodes (default is >= 16MB), or 2913 * fall back to the largest node if they're all smaller. 2914 */ 2915 nodes_clear(interleave_nodes); 2916 for_each_node_state(nid, N_MEMORY) { 2917 unsigned long total_pages = node_present_pages(nid); 2918 2919 /* Preserve the largest node */ 2920 if (largest < total_pages) { 2921 largest = total_pages; 2922 prefer = nid; 2923 } 2924 2925 /* Interleave this node? */ 2926 if ((total_pages << PAGE_SHIFT) >= (16 << 20)) 2927 node_set(nid, interleave_nodes); 2928 } 2929 2930 /* All too small, use the largest */ 2931 if (unlikely(nodes_empty(interleave_nodes))) 2932 node_set(prefer, interleave_nodes); 2933 2934 if (do_set_mempolicy(MPOL_INTERLEAVE, 0, &interleave_nodes)) 2935 pr_err("%s: interleaving failed\n", __func__); 2936 2937 check_numabalancing_enable(); 2938 } 2939 2940 /* Reset policy of current process to default */ 2941 void numa_default_policy(void) 2942 { 2943 do_set_mempolicy(MPOL_DEFAULT, 0, NULL); 2944 } 2945 2946 /* 2947 * Parse and format mempolicy from/to strings 2948 */ 2949 2950 static const char * const policy_modes[] = 2951 { 2952 [MPOL_DEFAULT] = "default", 2953 [MPOL_PREFERRED] = "prefer", 2954 [MPOL_BIND] = "bind", 2955 [MPOL_INTERLEAVE] = "interleave", 2956 [MPOL_LOCAL] = "local", 2957 [MPOL_PREFERRED_MANY] = "prefer (many)", 2958 }; 2959 2960 2961 #ifdef CONFIG_TMPFS 2962 /** 2963 * mpol_parse_str - parse string to mempolicy, for tmpfs mpol mount option. 2964 * @str: string containing mempolicy to parse 2965 * @mpol: pointer to struct mempolicy pointer, returned on success. 2966 * 2967 * Format of input: 2968 * <mode>[=<flags>][:<nodelist>] 2969 * 2970 * Return: %0 on success, else %1 2971 */ 2972 int mpol_parse_str(char *str, struct mempolicy **mpol) 2973 { 2974 struct mempolicy *new = NULL; 2975 unsigned short mode_flags; 2976 nodemask_t nodes; 2977 char *nodelist = strchr(str, ':'); 2978 char *flags = strchr(str, '='); 2979 int err = 1, mode; 2980 2981 if (flags) 2982 *flags++ = '\0'; /* terminate mode string */ 2983 2984 if (nodelist) { 2985 /* NUL-terminate mode or flags string */ 2986 *nodelist++ = '\0'; 2987 if (nodelist_parse(nodelist, nodes)) 2988 goto out; 2989 if (!nodes_subset(nodes, node_states[N_MEMORY])) 2990 goto out; 2991 } else 2992 nodes_clear(nodes); 2993 2994 mode = match_string(policy_modes, MPOL_MAX, str); 2995 if (mode < 0) 2996 goto out; 2997 2998 switch (mode) { 2999 case MPOL_PREFERRED: 3000 /* 3001 * Insist on a nodelist of one node only, although later 3002 * we use first_node(nodes) to grab a single node, so here 3003 * nodelist (or nodes) cannot be empty. 3004 */ 3005 if (nodelist) { 3006 char *rest = nodelist; 3007 while (isdigit(*rest)) 3008 rest++; 3009 if (*rest) 3010 goto out; 3011 if (nodes_empty(nodes)) 3012 goto out; 3013 } 3014 break; 3015 case MPOL_INTERLEAVE: 3016 /* 3017 * Default to online nodes with memory if no nodelist 3018 */ 3019 if (!nodelist) 3020 nodes = node_states[N_MEMORY]; 3021 break; 3022 case MPOL_LOCAL: 3023 /* 3024 * Don't allow a nodelist; mpol_new() checks flags 3025 */ 3026 if (nodelist) 3027 goto out; 3028 break; 3029 case MPOL_DEFAULT: 3030 /* 3031 * Insist on a empty nodelist 3032 */ 3033 if (!nodelist) 3034 err = 0; 3035 goto out; 3036 case MPOL_PREFERRED_MANY: 3037 case MPOL_BIND: 3038 /* 3039 * Insist on a nodelist 3040 */ 3041 if (!nodelist) 3042 goto out; 3043 } 3044 3045 mode_flags = 0; 3046 if (flags) { 3047 /* 3048 * Currently, we only support two mutually exclusive 3049 * mode flags. 3050 */ 3051 if (!strcmp(flags, "static")) 3052 mode_flags |= MPOL_F_STATIC_NODES; 3053 else if (!strcmp(flags, "relative")) 3054 mode_flags |= MPOL_F_RELATIVE_NODES; 3055 else 3056 goto out; 3057 } 3058 3059 new = mpol_new(mode, mode_flags, &nodes); 3060 if (IS_ERR(new)) 3061 goto out; 3062 3063 /* 3064 * Save nodes for mpol_to_str() to show the tmpfs mount options 3065 * for /proc/mounts, /proc/pid/mounts and /proc/pid/mountinfo. 3066 */ 3067 if (mode != MPOL_PREFERRED) { 3068 new->nodes = nodes; 3069 } else if (nodelist) { 3070 nodes_clear(new->nodes); 3071 node_set(first_node(nodes), new->nodes); 3072 } else { 3073 new->mode = MPOL_LOCAL; 3074 } 3075 3076 /* 3077 * Save nodes for contextualization: this will be used to "clone" 3078 * the mempolicy in a specific context [cpuset] at a later time. 3079 */ 3080 new->w.user_nodemask = nodes; 3081 3082 err = 0; 3083 3084 out: 3085 /* Restore string for error message */ 3086 if (nodelist) 3087 *--nodelist = ':'; 3088 if (flags) 3089 *--flags = '='; 3090 if (!err) 3091 *mpol = new; 3092 return err; 3093 } 3094 #endif /* CONFIG_TMPFS */ 3095 3096 /** 3097 * mpol_to_str - format a mempolicy structure for printing 3098 * @buffer: to contain formatted mempolicy string 3099 * @maxlen: length of @buffer 3100 * @pol: pointer to mempolicy to be formatted 3101 * 3102 * Convert @pol into a string. If @buffer is too short, truncate the string. 3103 * Recommend a @maxlen of at least 32 for the longest mode, "interleave", the 3104 * longest flag, "relative", and to display at least a few node ids. 3105 */ 3106 void mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol) 3107 { 3108 char *p = buffer; 3109 nodemask_t nodes = NODE_MASK_NONE; 3110 unsigned short mode = MPOL_DEFAULT; 3111 unsigned short flags = 0; 3112 3113 if (pol && pol != &default_policy && !(pol->flags & MPOL_F_MORON)) { 3114 mode = pol->mode; 3115 flags = pol->flags; 3116 } 3117 3118 switch (mode) { 3119 case MPOL_DEFAULT: 3120 case MPOL_LOCAL: 3121 break; 3122 case MPOL_PREFERRED: 3123 case MPOL_PREFERRED_MANY: 3124 case MPOL_BIND: 3125 case MPOL_INTERLEAVE: 3126 nodes = pol->nodes; 3127 break; 3128 default: 3129 WARN_ON_ONCE(1); 3130 snprintf(p, maxlen, "unknown"); 3131 return; 3132 } 3133 3134 p += snprintf(p, maxlen, "%s", policy_modes[mode]); 3135 3136 if (flags & MPOL_MODE_FLAGS) { 3137 p += snprintf(p, buffer + maxlen - p, "="); 3138 3139 /* 3140 * Currently, the only defined flags are mutually exclusive 3141 */ 3142 if (flags & MPOL_F_STATIC_NODES) 3143 p += snprintf(p, buffer + maxlen - p, "static"); 3144 else if (flags & MPOL_F_RELATIVE_NODES) 3145 p += snprintf(p, buffer + maxlen - p, "relative"); 3146 } 3147 3148 if (!nodes_empty(nodes)) 3149 p += scnprintf(p, buffer + maxlen - p, ":%*pbl", 3150 nodemask_pr_args(&nodes)); 3151 } 3152