1 /* 2 * kernel/cpuset.c 3 * 4 * Processor and Memory placement constraints for sets of tasks. 5 * 6 * Copyright (C) 2003 BULL SA. 7 * Copyright (C) 2004-2007 Silicon Graphics, Inc. 8 * Copyright (C) 2006 Google, Inc 9 * 10 * Portions derived from Patrick Mochel's sysfs code. 11 * sysfs is Copyright (c) 2001-3 Patrick Mochel 12 * 13 * 2003-10-10 Written by Simon Derr. 14 * 2003-10-22 Updates by Stephen Hemminger. 15 * 2004 May-July Rework by Paul Jackson. 16 * 2006 Rework by Paul Menage to use generic cgroups 17 * 2008 Rework of the scheduler domains and CPU hotplug handling 18 * by Max Krasnyansky 19 * 20 * This file is subject to the terms and conditions of the GNU General Public 21 * License. See the file COPYING in the main directory of the Linux 22 * distribution for more details. 23 */ 24 25 #include <linux/cpu.h> 26 #include <linux/cpumask.h> 27 #include <linux/cpuset.h> 28 #include <linux/err.h> 29 #include <linux/errno.h> 30 #include <linux/file.h> 31 #include <linux/fs.h> 32 #include <linux/init.h> 33 #include <linux/interrupt.h> 34 #include <linux/kernel.h> 35 #include <linux/kmod.h> 36 #include <linux/kthread.h> 37 #include <linux/list.h> 38 #include <linux/mempolicy.h> 39 #include <linux/mm.h> 40 #include <linux/memory.h> 41 #include <linux/export.h> 42 #include <linux/mount.h> 43 #include <linux/fs_context.h> 44 #include <linux/namei.h> 45 #include <linux/pagemap.h> 46 #include <linux/proc_fs.h> 47 #include <linux/rcupdate.h> 48 #include <linux/sched.h> 49 #include <linux/sched/deadline.h> 50 #include <linux/sched/mm.h> 51 #include <linux/sched/task.h> 52 #include <linux/seq_file.h> 53 #include <linux/security.h> 54 #include <linux/slab.h> 55 #include <linux/spinlock.h> 56 #include <linux/stat.h> 57 #include <linux/string.h> 58 #include <linux/time.h> 59 #include <linux/time64.h> 60 #include <linux/backing-dev.h> 61 #include <linux/sort.h> 62 #include <linux/oom.h> 63 #include <linux/sched/isolation.h> 64 #include <linux/uaccess.h> 65 #include <linux/atomic.h> 66 #include <linux/mutex.h> 67 #include <linux/cgroup.h> 68 #include <linux/wait.h> 69 70 DEFINE_STATIC_KEY_FALSE(cpusets_pre_enable_key); 71 DEFINE_STATIC_KEY_FALSE(cpusets_enabled_key); 72 73 /* 74 * There could be abnormal cpuset configurations for cpu or memory 75 * node binding, add this key to provide a quick low-cost judgment 76 * of the situation. 77 */ 78 DEFINE_STATIC_KEY_FALSE(cpusets_insane_config_key); 79 80 /* See "Frequency meter" comments, below. */ 81 82 struct fmeter { 83 int cnt; /* unprocessed events count */ 84 int val; /* most recent output value */ 85 time64_t time; /* clock (secs) when val computed */ 86 spinlock_t lock; /* guards read or write of above */ 87 }; 88 89 /* 90 * Invalid partition error code 91 */ 92 enum prs_errcode { 93 PERR_NONE = 0, 94 PERR_INVCPUS, 95 PERR_INVPARENT, 96 PERR_NOTPART, 97 PERR_NOTEXCL, 98 PERR_NOCPUS, 99 PERR_HOTPLUG, 100 PERR_CPUSEMPTY, 101 }; 102 103 static const char * const perr_strings[] = { 104 [PERR_INVCPUS] = "Invalid cpu list in cpuset.cpus", 105 [PERR_INVPARENT] = "Parent is an invalid partition root", 106 [PERR_NOTPART] = "Parent is not a partition root", 107 [PERR_NOTEXCL] = "Cpu list in cpuset.cpus not exclusive", 108 [PERR_NOCPUS] = "Parent unable to distribute cpu downstream", 109 [PERR_HOTPLUG] = "No cpu available due to hotplug", 110 [PERR_CPUSEMPTY] = "cpuset.cpus is empty", 111 }; 112 113 struct cpuset { 114 struct cgroup_subsys_state css; 115 116 unsigned long flags; /* "unsigned long" so bitops work */ 117 118 /* 119 * On default hierarchy: 120 * 121 * The user-configured masks can only be changed by writing to 122 * cpuset.cpus and cpuset.mems, and won't be limited by the 123 * parent masks. 124 * 125 * The effective masks is the real masks that apply to the tasks 126 * in the cpuset. They may be changed if the configured masks are 127 * changed or hotplug happens. 128 * 129 * effective_mask == configured_mask & parent's effective_mask, 130 * and if it ends up empty, it will inherit the parent's mask. 131 * 132 * 133 * On legacy hierarchy: 134 * 135 * The user-configured masks are always the same with effective masks. 136 */ 137 138 /* user-configured CPUs and Memory Nodes allow to tasks */ 139 cpumask_var_t cpus_allowed; 140 nodemask_t mems_allowed; 141 142 /* effective CPUs and Memory Nodes allow to tasks */ 143 cpumask_var_t effective_cpus; 144 nodemask_t effective_mems; 145 146 /* 147 * CPUs allocated to child sub-partitions (default hierarchy only) 148 * - CPUs granted by the parent = effective_cpus U subparts_cpus 149 * - effective_cpus and subparts_cpus are mutually exclusive. 150 * 151 * effective_cpus contains only onlined CPUs, but subparts_cpus 152 * may have offlined ones. 153 */ 154 cpumask_var_t subparts_cpus; 155 156 /* 157 * This is old Memory Nodes tasks took on. 158 * 159 * - top_cpuset.old_mems_allowed is initialized to mems_allowed. 160 * - A new cpuset's old_mems_allowed is initialized when some 161 * task is moved into it. 162 * - old_mems_allowed is used in cpuset_migrate_mm() when we change 163 * cpuset.mems_allowed and have tasks' nodemask updated, and 164 * then old_mems_allowed is updated to mems_allowed. 165 */ 166 nodemask_t old_mems_allowed; 167 168 struct fmeter fmeter; /* memory_pressure filter */ 169 170 /* 171 * Tasks are being attached to this cpuset. Used to prevent 172 * zeroing cpus/mems_allowed between ->can_attach() and ->attach(). 173 */ 174 int attach_in_progress; 175 176 /* partition number for rebuild_sched_domains() */ 177 int pn; 178 179 /* for custom sched domain */ 180 int relax_domain_level; 181 182 /* number of CPUs in subparts_cpus */ 183 int nr_subparts_cpus; 184 185 /* partition root state */ 186 int partition_root_state; 187 188 /* 189 * Default hierarchy only: 190 * use_parent_ecpus - set if using parent's effective_cpus 191 * child_ecpus_count - # of children with use_parent_ecpus set 192 */ 193 int use_parent_ecpus; 194 int child_ecpus_count; 195 196 /* Invalid partition error code, not lock protected */ 197 enum prs_errcode prs_err; 198 199 /* Handle for cpuset.cpus.partition */ 200 struct cgroup_file partition_file; 201 }; 202 203 /* 204 * Partition root states: 205 * 206 * 0 - member (not a partition root) 207 * 1 - partition root 208 * 2 - partition root without load balancing (isolated) 209 * -1 - invalid partition root 210 * -2 - invalid isolated partition root 211 */ 212 #define PRS_MEMBER 0 213 #define PRS_ROOT 1 214 #define PRS_ISOLATED 2 215 #define PRS_INVALID_ROOT -1 216 #define PRS_INVALID_ISOLATED -2 217 218 static inline bool is_prs_invalid(int prs_state) 219 { 220 return prs_state < 0; 221 } 222 223 /* 224 * Temporary cpumasks for working with partitions that are passed among 225 * functions to avoid memory allocation in inner functions. 226 */ 227 struct tmpmasks { 228 cpumask_var_t addmask, delmask; /* For partition root */ 229 cpumask_var_t new_cpus; /* For update_cpumasks_hier() */ 230 }; 231 232 static inline struct cpuset *css_cs(struct cgroup_subsys_state *css) 233 { 234 return css ? container_of(css, struct cpuset, css) : NULL; 235 } 236 237 /* Retrieve the cpuset for a task */ 238 static inline struct cpuset *task_cs(struct task_struct *task) 239 { 240 return css_cs(task_css(task, cpuset_cgrp_id)); 241 } 242 243 static inline struct cpuset *parent_cs(struct cpuset *cs) 244 { 245 return css_cs(cs->css.parent); 246 } 247 248 /* bits in struct cpuset flags field */ 249 typedef enum { 250 CS_ONLINE, 251 CS_CPU_EXCLUSIVE, 252 CS_MEM_EXCLUSIVE, 253 CS_MEM_HARDWALL, 254 CS_MEMORY_MIGRATE, 255 CS_SCHED_LOAD_BALANCE, 256 CS_SPREAD_PAGE, 257 CS_SPREAD_SLAB, 258 } cpuset_flagbits_t; 259 260 /* convenient tests for these bits */ 261 static inline bool is_cpuset_online(struct cpuset *cs) 262 { 263 return test_bit(CS_ONLINE, &cs->flags) && !css_is_dying(&cs->css); 264 } 265 266 static inline int is_cpu_exclusive(const struct cpuset *cs) 267 { 268 return test_bit(CS_CPU_EXCLUSIVE, &cs->flags); 269 } 270 271 static inline int is_mem_exclusive(const struct cpuset *cs) 272 { 273 return test_bit(CS_MEM_EXCLUSIVE, &cs->flags); 274 } 275 276 static inline int is_mem_hardwall(const struct cpuset *cs) 277 { 278 return test_bit(CS_MEM_HARDWALL, &cs->flags); 279 } 280 281 static inline int is_sched_load_balance(const struct cpuset *cs) 282 { 283 return test_bit(CS_SCHED_LOAD_BALANCE, &cs->flags); 284 } 285 286 static inline int is_memory_migrate(const struct cpuset *cs) 287 { 288 return test_bit(CS_MEMORY_MIGRATE, &cs->flags); 289 } 290 291 static inline int is_spread_page(const struct cpuset *cs) 292 { 293 return test_bit(CS_SPREAD_PAGE, &cs->flags); 294 } 295 296 static inline int is_spread_slab(const struct cpuset *cs) 297 { 298 return test_bit(CS_SPREAD_SLAB, &cs->flags); 299 } 300 301 static inline int is_partition_valid(const struct cpuset *cs) 302 { 303 return cs->partition_root_state > 0; 304 } 305 306 static inline int is_partition_invalid(const struct cpuset *cs) 307 { 308 return cs->partition_root_state < 0; 309 } 310 311 /* 312 * Callers should hold callback_lock to modify partition_root_state. 313 */ 314 static inline void make_partition_invalid(struct cpuset *cs) 315 { 316 if (is_partition_valid(cs)) 317 cs->partition_root_state = -cs->partition_root_state; 318 } 319 320 /* 321 * Send notification event of whenever partition_root_state changes. 322 */ 323 static inline void notify_partition_change(struct cpuset *cs, int old_prs) 324 { 325 if (old_prs == cs->partition_root_state) 326 return; 327 cgroup_file_notify(&cs->partition_file); 328 329 /* Reset prs_err if not invalid */ 330 if (is_partition_valid(cs)) 331 WRITE_ONCE(cs->prs_err, PERR_NONE); 332 } 333 334 static struct cpuset top_cpuset = { 335 .flags = ((1 << CS_ONLINE) | (1 << CS_CPU_EXCLUSIVE) | 336 (1 << CS_MEM_EXCLUSIVE)), 337 .partition_root_state = PRS_ROOT, 338 }; 339 340 /** 341 * cpuset_for_each_child - traverse online children of a cpuset 342 * @child_cs: loop cursor pointing to the current child 343 * @pos_css: used for iteration 344 * @parent_cs: target cpuset to walk children of 345 * 346 * Walk @child_cs through the online children of @parent_cs. Must be used 347 * with RCU read locked. 348 */ 349 #define cpuset_for_each_child(child_cs, pos_css, parent_cs) \ 350 css_for_each_child((pos_css), &(parent_cs)->css) \ 351 if (is_cpuset_online(((child_cs) = css_cs((pos_css))))) 352 353 /** 354 * cpuset_for_each_descendant_pre - pre-order walk of a cpuset's descendants 355 * @des_cs: loop cursor pointing to the current descendant 356 * @pos_css: used for iteration 357 * @root_cs: target cpuset to walk ancestor of 358 * 359 * Walk @des_cs through the online descendants of @root_cs. Must be used 360 * with RCU read locked. The caller may modify @pos_css by calling 361 * css_rightmost_descendant() to skip subtree. @root_cs is included in the 362 * iteration and the first node to be visited. 363 */ 364 #define cpuset_for_each_descendant_pre(des_cs, pos_css, root_cs) \ 365 css_for_each_descendant_pre((pos_css), &(root_cs)->css) \ 366 if (is_cpuset_online(((des_cs) = css_cs((pos_css))))) 367 368 /* 369 * There are two global locks guarding cpuset structures - cpuset_rwsem and 370 * callback_lock. We also require taking task_lock() when dereferencing a 371 * task's cpuset pointer. See "The task_lock() exception", at the end of this 372 * comment. The cpuset code uses only cpuset_rwsem write lock. Other 373 * kernel subsystems can use cpuset_read_lock()/cpuset_read_unlock() to 374 * prevent change to cpuset structures. 375 * 376 * A task must hold both locks to modify cpusets. If a task holds 377 * cpuset_rwsem, it blocks others wanting that rwsem, ensuring that it 378 * is the only task able to also acquire callback_lock and be able to 379 * modify cpusets. It can perform various checks on the cpuset structure 380 * first, knowing nothing will change. It can also allocate memory while 381 * just holding cpuset_rwsem. While it is performing these checks, various 382 * callback routines can briefly acquire callback_lock to query cpusets. 383 * Once it is ready to make the changes, it takes callback_lock, blocking 384 * everyone else. 385 * 386 * Calls to the kernel memory allocator can not be made while holding 387 * callback_lock, as that would risk double tripping on callback_lock 388 * from one of the callbacks into the cpuset code from within 389 * __alloc_pages(). 390 * 391 * If a task is only holding callback_lock, then it has read-only 392 * access to cpusets. 393 * 394 * Now, the task_struct fields mems_allowed and mempolicy may be changed 395 * by other task, we use alloc_lock in the task_struct fields to protect 396 * them. 397 * 398 * The cpuset_common_file_read() handlers only hold callback_lock across 399 * small pieces of code, such as when reading out possibly multi-word 400 * cpumasks and nodemasks. 401 * 402 * Accessing a task's cpuset should be done in accordance with the 403 * guidelines for accessing subsystem state in kernel/cgroup.c 404 */ 405 406 DEFINE_STATIC_PERCPU_RWSEM(cpuset_rwsem); 407 408 void cpuset_read_lock(void) 409 { 410 percpu_down_read(&cpuset_rwsem); 411 } 412 413 void cpuset_read_unlock(void) 414 { 415 percpu_up_read(&cpuset_rwsem); 416 } 417 418 static DEFINE_SPINLOCK(callback_lock); 419 420 static struct workqueue_struct *cpuset_migrate_mm_wq; 421 422 /* 423 * CPU / memory hotplug is handled asynchronously. 424 */ 425 static void cpuset_hotplug_workfn(struct work_struct *work); 426 static DECLARE_WORK(cpuset_hotplug_work, cpuset_hotplug_workfn); 427 428 static DECLARE_WAIT_QUEUE_HEAD(cpuset_attach_wq); 429 430 static inline void check_insane_mems_config(nodemask_t *nodes) 431 { 432 if (!cpusets_insane_config() && 433 movable_only_nodes(nodes)) { 434 static_branch_enable(&cpusets_insane_config_key); 435 pr_info("Unsupported (movable nodes only) cpuset configuration detected (nmask=%*pbl)!\n" 436 "Cpuset allocations might fail even with a lot of memory available.\n", 437 nodemask_pr_args(nodes)); 438 } 439 } 440 441 /* 442 * Cgroup v2 behavior is used on the "cpus" and "mems" control files when 443 * on default hierarchy or when the cpuset_v2_mode flag is set by mounting 444 * the v1 cpuset cgroup filesystem with the "cpuset_v2_mode" mount option. 445 * With v2 behavior, "cpus" and "mems" are always what the users have 446 * requested and won't be changed by hotplug events. Only the effective 447 * cpus or mems will be affected. 448 */ 449 static inline bool is_in_v2_mode(void) 450 { 451 return cgroup_subsys_on_dfl(cpuset_cgrp_subsys) || 452 (cpuset_cgrp_subsys.root->flags & CGRP_ROOT_CPUSET_V2_MODE); 453 } 454 455 /** 456 * partition_is_populated - check if partition has tasks 457 * @cs: partition root to be checked 458 * @excluded_child: a child cpuset to be excluded in task checking 459 * Return: true if there are tasks, false otherwise 460 * 461 * It is assumed that @cs is a valid partition root. @excluded_child should 462 * be non-NULL when this cpuset is going to become a partition itself. 463 */ 464 static inline bool partition_is_populated(struct cpuset *cs, 465 struct cpuset *excluded_child) 466 { 467 struct cgroup_subsys_state *css; 468 struct cpuset *child; 469 470 if (cs->css.cgroup->nr_populated_csets) 471 return true; 472 if (!excluded_child && !cs->nr_subparts_cpus) 473 return cgroup_is_populated(cs->css.cgroup); 474 475 rcu_read_lock(); 476 cpuset_for_each_child(child, css, cs) { 477 if (child == excluded_child) 478 continue; 479 if (is_partition_valid(child)) 480 continue; 481 if (cgroup_is_populated(child->css.cgroup)) { 482 rcu_read_unlock(); 483 return true; 484 } 485 } 486 rcu_read_unlock(); 487 return false; 488 } 489 490 /* 491 * Return in pmask the portion of a task's cpusets's cpus_allowed that 492 * are online and are capable of running the task. If none are found, 493 * walk up the cpuset hierarchy until we find one that does have some 494 * appropriate cpus. 495 * 496 * One way or another, we guarantee to return some non-empty subset 497 * of cpu_online_mask. 498 * 499 * Call with callback_lock or cpuset_rwsem held. 500 */ 501 static void guarantee_online_cpus(struct task_struct *tsk, 502 struct cpumask *pmask) 503 { 504 const struct cpumask *possible_mask = task_cpu_possible_mask(tsk); 505 struct cpuset *cs; 506 507 if (WARN_ON(!cpumask_and(pmask, possible_mask, cpu_online_mask))) 508 cpumask_copy(pmask, cpu_online_mask); 509 510 rcu_read_lock(); 511 cs = task_cs(tsk); 512 513 while (!cpumask_intersects(cs->effective_cpus, pmask)) { 514 cs = parent_cs(cs); 515 if (unlikely(!cs)) { 516 /* 517 * The top cpuset doesn't have any online cpu as a 518 * consequence of a race between cpuset_hotplug_work 519 * and cpu hotplug notifier. But we know the top 520 * cpuset's effective_cpus is on its way to be 521 * identical to cpu_online_mask. 522 */ 523 goto out_unlock; 524 } 525 } 526 cpumask_and(pmask, pmask, cs->effective_cpus); 527 528 out_unlock: 529 rcu_read_unlock(); 530 } 531 532 /* 533 * Return in *pmask the portion of a cpusets's mems_allowed that 534 * are online, with memory. If none are online with memory, walk 535 * up the cpuset hierarchy until we find one that does have some 536 * online mems. The top cpuset always has some mems online. 537 * 538 * One way or another, we guarantee to return some non-empty subset 539 * of node_states[N_MEMORY]. 540 * 541 * Call with callback_lock or cpuset_rwsem held. 542 */ 543 static void guarantee_online_mems(struct cpuset *cs, nodemask_t *pmask) 544 { 545 while (!nodes_intersects(cs->effective_mems, node_states[N_MEMORY])) 546 cs = parent_cs(cs); 547 nodes_and(*pmask, cs->effective_mems, node_states[N_MEMORY]); 548 } 549 550 /* 551 * update task's spread flag if cpuset's page/slab spread flag is set 552 * 553 * Call with callback_lock or cpuset_rwsem held. The check can be skipped 554 * if on default hierarchy. 555 */ 556 static void cpuset_update_task_spread_flags(struct cpuset *cs, 557 struct task_struct *tsk) 558 { 559 if (cgroup_subsys_on_dfl(cpuset_cgrp_subsys)) 560 return; 561 562 if (is_spread_page(cs)) 563 task_set_spread_page(tsk); 564 else 565 task_clear_spread_page(tsk); 566 567 if (is_spread_slab(cs)) 568 task_set_spread_slab(tsk); 569 else 570 task_clear_spread_slab(tsk); 571 } 572 573 /* 574 * is_cpuset_subset(p, q) - Is cpuset p a subset of cpuset q? 575 * 576 * One cpuset is a subset of another if all its allowed CPUs and 577 * Memory Nodes are a subset of the other, and its exclusive flags 578 * are only set if the other's are set. Call holding cpuset_rwsem. 579 */ 580 581 static int is_cpuset_subset(const struct cpuset *p, const struct cpuset *q) 582 { 583 return cpumask_subset(p->cpus_allowed, q->cpus_allowed) && 584 nodes_subset(p->mems_allowed, q->mems_allowed) && 585 is_cpu_exclusive(p) <= is_cpu_exclusive(q) && 586 is_mem_exclusive(p) <= is_mem_exclusive(q); 587 } 588 589 /** 590 * alloc_cpumasks - allocate three cpumasks for cpuset 591 * @cs: the cpuset that have cpumasks to be allocated. 592 * @tmp: the tmpmasks structure pointer 593 * Return: 0 if successful, -ENOMEM otherwise. 594 * 595 * Only one of the two input arguments should be non-NULL. 596 */ 597 static inline int alloc_cpumasks(struct cpuset *cs, struct tmpmasks *tmp) 598 { 599 cpumask_var_t *pmask1, *pmask2, *pmask3; 600 601 if (cs) { 602 pmask1 = &cs->cpus_allowed; 603 pmask2 = &cs->effective_cpus; 604 pmask3 = &cs->subparts_cpus; 605 } else { 606 pmask1 = &tmp->new_cpus; 607 pmask2 = &tmp->addmask; 608 pmask3 = &tmp->delmask; 609 } 610 611 if (!zalloc_cpumask_var(pmask1, GFP_KERNEL)) 612 return -ENOMEM; 613 614 if (!zalloc_cpumask_var(pmask2, GFP_KERNEL)) 615 goto free_one; 616 617 if (!zalloc_cpumask_var(pmask3, GFP_KERNEL)) 618 goto free_two; 619 620 return 0; 621 622 free_two: 623 free_cpumask_var(*pmask2); 624 free_one: 625 free_cpumask_var(*pmask1); 626 return -ENOMEM; 627 } 628 629 /** 630 * free_cpumasks - free cpumasks in a tmpmasks structure 631 * @cs: the cpuset that have cpumasks to be free. 632 * @tmp: the tmpmasks structure pointer 633 */ 634 static inline void free_cpumasks(struct cpuset *cs, struct tmpmasks *tmp) 635 { 636 if (cs) { 637 free_cpumask_var(cs->cpus_allowed); 638 free_cpumask_var(cs->effective_cpus); 639 free_cpumask_var(cs->subparts_cpus); 640 } 641 if (tmp) { 642 free_cpumask_var(tmp->new_cpus); 643 free_cpumask_var(tmp->addmask); 644 free_cpumask_var(tmp->delmask); 645 } 646 } 647 648 /** 649 * alloc_trial_cpuset - allocate a trial cpuset 650 * @cs: the cpuset that the trial cpuset duplicates 651 */ 652 static struct cpuset *alloc_trial_cpuset(struct cpuset *cs) 653 { 654 struct cpuset *trial; 655 656 trial = kmemdup(cs, sizeof(*cs), GFP_KERNEL); 657 if (!trial) 658 return NULL; 659 660 if (alloc_cpumasks(trial, NULL)) { 661 kfree(trial); 662 return NULL; 663 } 664 665 cpumask_copy(trial->cpus_allowed, cs->cpus_allowed); 666 cpumask_copy(trial->effective_cpus, cs->effective_cpus); 667 return trial; 668 } 669 670 /** 671 * free_cpuset - free the cpuset 672 * @cs: the cpuset to be freed 673 */ 674 static inline void free_cpuset(struct cpuset *cs) 675 { 676 free_cpumasks(cs, NULL); 677 kfree(cs); 678 } 679 680 /* 681 * validate_change_legacy() - Validate conditions specific to legacy (v1) 682 * behavior. 683 */ 684 static int validate_change_legacy(struct cpuset *cur, struct cpuset *trial) 685 { 686 struct cgroup_subsys_state *css; 687 struct cpuset *c, *par; 688 int ret; 689 690 WARN_ON_ONCE(!rcu_read_lock_held()); 691 692 /* Each of our child cpusets must be a subset of us */ 693 ret = -EBUSY; 694 cpuset_for_each_child(c, css, cur) 695 if (!is_cpuset_subset(c, trial)) 696 goto out; 697 698 /* On legacy hierarchy, we must be a subset of our parent cpuset. */ 699 ret = -EACCES; 700 par = parent_cs(cur); 701 if (par && !is_cpuset_subset(trial, par)) 702 goto out; 703 704 ret = 0; 705 out: 706 return ret; 707 } 708 709 /* 710 * validate_change() - Used to validate that any proposed cpuset change 711 * follows the structural rules for cpusets. 712 * 713 * If we replaced the flag and mask values of the current cpuset 714 * (cur) with those values in the trial cpuset (trial), would 715 * our various subset and exclusive rules still be valid? Presumes 716 * cpuset_rwsem held. 717 * 718 * 'cur' is the address of an actual, in-use cpuset. Operations 719 * such as list traversal that depend on the actual address of the 720 * cpuset in the list must use cur below, not trial. 721 * 722 * 'trial' is the address of bulk structure copy of cur, with 723 * perhaps one or more of the fields cpus_allowed, mems_allowed, 724 * or flags changed to new, trial values. 725 * 726 * Return 0 if valid, -errno if not. 727 */ 728 729 static int validate_change(struct cpuset *cur, struct cpuset *trial) 730 { 731 struct cgroup_subsys_state *css; 732 struct cpuset *c, *par; 733 int ret = 0; 734 735 rcu_read_lock(); 736 737 if (!is_in_v2_mode()) 738 ret = validate_change_legacy(cur, trial); 739 if (ret) 740 goto out; 741 742 /* Remaining checks don't apply to root cpuset */ 743 if (cur == &top_cpuset) 744 goto out; 745 746 par = parent_cs(cur); 747 748 /* 749 * Cpusets with tasks - existing or newly being attached - can't 750 * be changed to have empty cpus_allowed or mems_allowed. 751 */ 752 ret = -ENOSPC; 753 if ((cgroup_is_populated(cur->css.cgroup) || cur->attach_in_progress)) { 754 if (!cpumask_empty(cur->cpus_allowed) && 755 cpumask_empty(trial->cpus_allowed)) 756 goto out; 757 if (!nodes_empty(cur->mems_allowed) && 758 nodes_empty(trial->mems_allowed)) 759 goto out; 760 } 761 762 /* 763 * We can't shrink if we won't have enough room for SCHED_DEADLINE 764 * tasks. 765 */ 766 ret = -EBUSY; 767 if (is_cpu_exclusive(cur) && 768 !cpuset_cpumask_can_shrink(cur->cpus_allowed, 769 trial->cpus_allowed)) 770 goto out; 771 772 /* 773 * If either I or some sibling (!= me) is exclusive, we can't 774 * overlap 775 */ 776 ret = -EINVAL; 777 cpuset_for_each_child(c, css, par) { 778 if ((is_cpu_exclusive(trial) || is_cpu_exclusive(c)) && 779 c != cur && 780 cpumask_intersects(trial->cpus_allowed, c->cpus_allowed)) 781 goto out; 782 if ((is_mem_exclusive(trial) || is_mem_exclusive(c)) && 783 c != cur && 784 nodes_intersects(trial->mems_allowed, c->mems_allowed)) 785 goto out; 786 } 787 788 ret = 0; 789 out: 790 rcu_read_unlock(); 791 return ret; 792 } 793 794 #ifdef CONFIG_SMP 795 /* 796 * Helper routine for generate_sched_domains(). 797 * Do cpusets a, b have overlapping effective cpus_allowed masks? 798 */ 799 static int cpusets_overlap(struct cpuset *a, struct cpuset *b) 800 { 801 return cpumask_intersects(a->effective_cpus, b->effective_cpus); 802 } 803 804 static void 805 update_domain_attr(struct sched_domain_attr *dattr, struct cpuset *c) 806 { 807 if (dattr->relax_domain_level < c->relax_domain_level) 808 dattr->relax_domain_level = c->relax_domain_level; 809 return; 810 } 811 812 static void update_domain_attr_tree(struct sched_domain_attr *dattr, 813 struct cpuset *root_cs) 814 { 815 struct cpuset *cp; 816 struct cgroup_subsys_state *pos_css; 817 818 rcu_read_lock(); 819 cpuset_for_each_descendant_pre(cp, pos_css, root_cs) { 820 /* skip the whole subtree if @cp doesn't have any CPU */ 821 if (cpumask_empty(cp->cpus_allowed)) { 822 pos_css = css_rightmost_descendant(pos_css); 823 continue; 824 } 825 826 if (is_sched_load_balance(cp)) 827 update_domain_attr(dattr, cp); 828 } 829 rcu_read_unlock(); 830 } 831 832 /* Must be called with cpuset_rwsem held. */ 833 static inline int nr_cpusets(void) 834 { 835 /* jump label reference count + the top-level cpuset */ 836 return static_key_count(&cpusets_enabled_key.key) + 1; 837 } 838 839 /* 840 * generate_sched_domains() 841 * 842 * This function builds a partial partition of the systems CPUs 843 * A 'partial partition' is a set of non-overlapping subsets whose 844 * union is a subset of that set. 845 * The output of this function needs to be passed to kernel/sched/core.c 846 * partition_sched_domains() routine, which will rebuild the scheduler's 847 * load balancing domains (sched domains) as specified by that partial 848 * partition. 849 * 850 * See "What is sched_load_balance" in Documentation/admin-guide/cgroup-v1/cpusets.rst 851 * for a background explanation of this. 852 * 853 * Does not return errors, on the theory that the callers of this 854 * routine would rather not worry about failures to rebuild sched 855 * domains when operating in the severe memory shortage situations 856 * that could cause allocation failures below. 857 * 858 * Must be called with cpuset_rwsem held. 859 * 860 * The three key local variables below are: 861 * cp - cpuset pointer, used (together with pos_css) to perform a 862 * top-down scan of all cpusets. For our purposes, rebuilding 863 * the schedulers sched domains, we can ignore !is_sched_load_ 864 * balance cpusets. 865 * csa - (for CpuSet Array) Array of pointers to all the cpusets 866 * that need to be load balanced, for convenient iterative 867 * access by the subsequent code that finds the best partition, 868 * i.e the set of domains (subsets) of CPUs such that the 869 * cpus_allowed of every cpuset marked is_sched_load_balance 870 * is a subset of one of these domains, while there are as 871 * many such domains as possible, each as small as possible. 872 * doms - Conversion of 'csa' to an array of cpumasks, for passing to 873 * the kernel/sched/core.c routine partition_sched_domains() in a 874 * convenient format, that can be easily compared to the prior 875 * value to determine what partition elements (sched domains) 876 * were changed (added or removed.) 877 * 878 * Finding the best partition (set of domains): 879 * The triple nested loops below over i, j, k scan over the 880 * load balanced cpusets (using the array of cpuset pointers in 881 * csa[]) looking for pairs of cpusets that have overlapping 882 * cpus_allowed, but which don't have the same 'pn' partition 883 * number and gives them in the same partition number. It keeps 884 * looping on the 'restart' label until it can no longer find 885 * any such pairs. 886 * 887 * The union of the cpus_allowed masks from the set of 888 * all cpusets having the same 'pn' value then form the one 889 * element of the partition (one sched domain) to be passed to 890 * partition_sched_domains(). 891 */ 892 static int generate_sched_domains(cpumask_var_t **domains, 893 struct sched_domain_attr **attributes) 894 { 895 struct cpuset *cp; /* top-down scan of cpusets */ 896 struct cpuset **csa; /* array of all cpuset ptrs */ 897 int csn; /* how many cpuset ptrs in csa so far */ 898 int i, j, k; /* indices for partition finding loops */ 899 cpumask_var_t *doms; /* resulting partition; i.e. sched domains */ 900 struct sched_domain_attr *dattr; /* attributes for custom domains */ 901 int ndoms = 0; /* number of sched domains in result */ 902 int nslot; /* next empty doms[] struct cpumask slot */ 903 struct cgroup_subsys_state *pos_css; 904 bool root_load_balance = is_sched_load_balance(&top_cpuset); 905 906 doms = NULL; 907 dattr = NULL; 908 csa = NULL; 909 910 /* Special case for the 99% of systems with one, full, sched domain */ 911 if (root_load_balance && !top_cpuset.nr_subparts_cpus) { 912 ndoms = 1; 913 doms = alloc_sched_domains(ndoms); 914 if (!doms) 915 goto done; 916 917 dattr = kmalloc(sizeof(struct sched_domain_attr), GFP_KERNEL); 918 if (dattr) { 919 *dattr = SD_ATTR_INIT; 920 update_domain_attr_tree(dattr, &top_cpuset); 921 } 922 cpumask_and(doms[0], top_cpuset.effective_cpus, 923 housekeeping_cpumask(HK_TYPE_DOMAIN)); 924 925 goto done; 926 } 927 928 csa = kmalloc_array(nr_cpusets(), sizeof(cp), GFP_KERNEL); 929 if (!csa) 930 goto done; 931 csn = 0; 932 933 rcu_read_lock(); 934 if (root_load_balance) 935 csa[csn++] = &top_cpuset; 936 cpuset_for_each_descendant_pre(cp, pos_css, &top_cpuset) { 937 if (cp == &top_cpuset) 938 continue; 939 /* 940 * Continue traversing beyond @cp iff @cp has some CPUs and 941 * isn't load balancing. The former is obvious. The 942 * latter: All child cpusets contain a subset of the 943 * parent's cpus, so just skip them, and then we call 944 * update_domain_attr_tree() to calc relax_domain_level of 945 * the corresponding sched domain. 946 * 947 * If root is load-balancing, we can skip @cp if it 948 * is a subset of the root's effective_cpus. 949 */ 950 if (!cpumask_empty(cp->cpus_allowed) && 951 !(is_sched_load_balance(cp) && 952 cpumask_intersects(cp->cpus_allowed, 953 housekeeping_cpumask(HK_TYPE_DOMAIN)))) 954 continue; 955 956 if (root_load_balance && 957 cpumask_subset(cp->cpus_allowed, top_cpuset.effective_cpus)) 958 continue; 959 960 if (is_sched_load_balance(cp) && 961 !cpumask_empty(cp->effective_cpus)) 962 csa[csn++] = cp; 963 964 /* skip @cp's subtree if not a partition root */ 965 if (!is_partition_valid(cp)) 966 pos_css = css_rightmost_descendant(pos_css); 967 } 968 rcu_read_unlock(); 969 970 for (i = 0; i < csn; i++) 971 csa[i]->pn = i; 972 ndoms = csn; 973 974 restart: 975 /* Find the best partition (set of sched domains) */ 976 for (i = 0; i < csn; i++) { 977 struct cpuset *a = csa[i]; 978 int apn = a->pn; 979 980 for (j = 0; j < csn; j++) { 981 struct cpuset *b = csa[j]; 982 int bpn = b->pn; 983 984 if (apn != bpn && cpusets_overlap(a, b)) { 985 for (k = 0; k < csn; k++) { 986 struct cpuset *c = csa[k]; 987 988 if (c->pn == bpn) 989 c->pn = apn; 990 } 991 ndoms--; /* one less element */ 992 goto restart; 993 } 994 } 995 } 996 997 /* 998 * Now we know how many domains to create. 999 * Convert <csn, csa> to <ndoms, doms> and populate cpu masks. 1000 */ 1001 doms = alloc_sched_domains(ndoms); 1002 if (!doms) 1003 goto done; 1004 1005 /* 1006 * The rest of the code, including the scheduler, can deal with 1007 * dattr==NULL case. No need to abort if alloc fails. 1008 */ 1009 dattr = kmalloc_array(ndoms, sizeof(struct sched_domain_attr), 1010 GFP_KERNEL); 1011 1012 for (nslot = 0, i = 0; i < csn; i++) { 1013 struct cpuset *a = csa[i]; 1014 struct cpumask *dp; 1015 int apn = a->pn; 1016 1017 if (apn < 0) { 1018 /* Skip completed partitions */ 1019 continue; 1020 } 1021 1022 dp = doms[nslot]; 1023 1024 if (nslot == ndoms) { 1025 static int warnings = 10; 1026 if (warnings) { 1027 pr_warn("rebuild_sched_domains confused: nslot %d, ndoms %d, csn %d, i %d, apn %d\n", 1028 nslot, ndoms, csn, i, apn); 1029 warnings--; 1030 } 1031 continue; 1032 } 1033 1034 cpumask_clear(dp); 1035 if (dattr) 1036 *(dattr + nslot) = SD_ATTR_INIT; 1037 for (j = i; j < csn; j++) { 1038 struct cpuset *b = csa[j]; 1039 1040 if (apn == b->pn) { 1041 cpumask_or(dp, dp, b->effective_cpus); 1042 cpumask_and(dp, dp, housekeeping_cpumask(HK_TYPE_DOMAIN)); 1043 if (dattr) 1044 update_domain_attr_tree(dattr + nslot, b); 1045 1046 /* Done with this partition */ 1047 b->pn = -1; 1048 } 1049 } 1050 nslot++; 1051 } 1052 BUG_ON(nslot != ndoms); 1053 1054 done: 1055 kfree(csa); 1056 1057 /* 1058 * Fallback to the default domain if kmalloc() failed. 1059 * See comments in partition_sched_domains(). 1060 */ 1061 if (doms == NULL) 1062 ndoms = 1; 1063 1064 *domains = doms; 1065 *attributes = dattr; 1066 return ndoms; 1067 } 1068 1069 static void update_tasks_root_domain(struct cpuset *cs) 1070 { 1071 struct css_task_iter it; 1072 struct task_struct *task; 1073 1074 css_task_iter_start(&cs->css, 0, &it); 1075 1076 while ((task = css_task_iter_next(&it))) 1077 dl_add_task_root_domain(task); 1078 1079 css_task_iter_end(&it); 1080 } 1081 1082 static void rebuild_root_domains(void) 1083 { 1084 struct cpuset *cs = NULL; 1085 struct cgroup_subsys_state *pos_css; 1086 1087 percpu_rwsem_assert_held(&cpuset_rwsem); 1088 lockdep_assert_cpus_held(); 1089 lockdep_assert_held(&sched_domains_mutex); 1090 1091 rcu_read_lock(); 1092 1093 /* 1094 * Clear default root domain DL accounting, it will be computed again 1095 * if a task belongs to it. 1096 */ 1097 dl_clear_root_domain(&def_root_domain); 1098 1099 cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) { 1100 1101 if (cpumask_empty(cs->effective_cpus)) { 1102 pos_css = css_rightmost_descendant(pos_css); 1103 continue; 1104 } 1105 1106 css_get(&cs->css); 1107 1108 rcu_read_unlock(); 1109 1110 update_tasks_root_domain(cs); 1111 1112 rcu_read_lock(); 1113 css_put(&cs->css); 1114 } 1115 rcu_read_unlock(); 1116 } 1117 1118 static void 1119 partition_and_rebuild_sched_domains(int ndoms_new, cpumask_var_t doms_new[], 1120 struct sched_domain_attr *dattr_new) 1121 { 1122 mutex_lock(&sched_domains_mutex); 1123 partition_sched_domains_locked(ndoms_new, doms_new, dattr_new); 1124 rebuild_root_domains(); 1125 mutex_unlock(&sched_domains_mutex); 1126 } 1127 1128 /* 1129 * Rebuild scheduler domains. 1130 * 1131 * If the flag 'sched_load_balance' of any cpuset with non-empty 1132 * 'cpus' changes, or if the 'cpus' allowed changes in any cpuset 1133 * which has that flag enabled, or if any cpuset with a non-empty 1134 * 'cpus' is removed, then call this routine to rebuild the 1135 * scheduler's dynamic sched domains. 1136 * 1137 * Call with cpuset_rwsem held. Takes cpus_read_lock(). 1138 */ 1139 static void rebuild_sched_domains_locked(void) 1140 { 1141 struct cgroup_subsys_state *pos_css; 1142 struct sched_domain_attr *attr; 1143 cpumask_var_t *doms; 1144 struct cpuset *cs; 1145 int ndoms; 1146 1147 lockdep_assert_cpus_held(); 1148 percpu_rwsem_assert_held(&cpuset_rwsem); 1149 1150 /* 1151 * If we have raced with CPU hotplug, return early to avoid 1152 * passing doms with offlined cpu to partition_sched_domains(). 1153 * Anyways, cpuset_hotplug_workfn() will rebuild sched domains. 1154 * 1155 * With no CPUs in any subpartitions, top_cpuset's effective CPUs 1156 * should be the same as the active CPUs, so checking only top_cpuset 1157 * is enough to detect racing CPU offlines. 1158 */ 1159 if (!top_cpuset.nr_subparts_cpus && 1160 !cpumask_equal(top_cpuset.effective_cpus, cpu_active_mask)) 1161 return; 1162 1163 /* 1164 * With subpartition CPUs, however, the effective CPUs of a partition 1165 * root should be only a subset of the active CPUs. Since a CPU in any 1166 * partition root could be offlined, all must be checked. 1167 */ 1168 if (top_cpuset.nr_subparts_cpus) { 1169 rcu_read_lock(); 1170 cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) { 1171 if (!is_partition_valid(cs)) { 1172 pos_css = css_rightmost_descendant(pos_css); 1173 continue; 1174 } 1175 if (!cpumask_subset(cs->effective_cpus, 1176 cpu_active_mask)) { 1177 rcu_read_unlock(); 1178 return; 1179 } 1180 } 1181 rcu_read_unlock(); 1182 } 1183 1184 /* Generate domain masks and attrs */ 1185 ndoms = generate_sched_domains(&doms, &attr); 1186 1187 /* Have scheduler rebuild the domains */ 1188 partition_and_rebuild_sched_domains(ndoms, doms, attr); 1189 } 1190 #else /* !CONFIG_SMP */ 1191 static void rebuild_sched_domains_locked(void) 1192 { 1193 } 1194 #endif /* CONFIG_SMP */ 1195 1196 void rebuild_sched_domains(void) 1197 { 1198 cpus_read_lock(); 1199 percpu_down_write(&cpuset_rwsem); 1200 rebuild_sched_domains_locked(); 1201 percpu_up_write(&cpuset_rwsem); 1202 cpus_read_unlock(); 1203 } 1204 1205 /** 1206 * update_tasks_cpumask - Update the cpumasks of tasks in the cpuset. 1207 * @cs: the cpuset in which each task's cpus_allowed mask needs to be changed 1208 * 1209 * Iterate through each task of @cs updating its cpus_allowed to the 1210 * effective cpuset's. As this function is called with cpuset_rwsem held, 1211 * cpuset membership stays stable. 1212 */ 1213 static void update_tasks_cpumask(struct cpuset *cs) 1214 { 1215 struct css_task_iter it; 1216 struct task_struct *task; 1217 bool top_cs = cs == &top_cpuset; 1218 1219 css_task_iter_start(&cs->css, 0, &it); 1220 while ((task = css_task_iter_next(&it))) { 1221 /* 1222 * Percpu kthreads in top_cpuset are ignored 1223 */ 1224 if (top_cs && (task->flags & PF_KTHREAD) && 1225 kthread_is_per_cpu(task)) 1226 continue; 1227 set_cpus_allowed_ptr(task, cs->effective_cpus); 1228 } 1229 css_task_iter_end(&it); 1230 } 1231 1232 /** 1233 * compute_effective_cpumask - Compute the effective cpumask of the cpuset 1234 * @new_cpus: the temp variable for the new effective_cpus mask 1235 * @cs: the cpuset the need to recompute the new effective_cpus mask 1236 * @parent: the parent cpuset 1237 * 1238 * If the parent has subpartition CPUs, include them in the list of 1239 * allowable CPUs in computing the new effective_cpus mask. Since offlined 1240 * CPUs are not removed from subparts_cpus, we have to use cpu_active_mask 1241 * to mask those out. 1242 */ 1243 static void compute_effective_cpumask(struct cpumask *new_cpus, 1244 struct cpuset *cs, struct cpuset *parent) 1245 { 1246 if (parent->nr_subparts_cpus) { 1247 cpumask_or(new_cpus, parent->effective_cpus, 1248 parent->subparts_cpus); 1249 cpumask_and(new_cpus, new_cpus, cs->cpus_allowed); 1250 cpumask_and(new_cpus, new_cpus, cpu_active_mask); 1251 } else { 1252 cpumask_and(new_cpus, cs->cpus_allowed, parent->effective_cpus); 1253 } 1254 } 1255 1256 /* 1257 * Commands for update_parent_subparts_cpumask 1258 */ 1259 enum subparts_cmd { 1260 partcmd_enable, /* Enable partition root */ 1261 partcmd_disable, /* Disable partition root */ 1262 partcmd_update, /* Update parent's subparts_cpus */ 1263 partcmd_invalidate, /* Make partition invalid */ 1264 }; 1265 1266 static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs, 1267 int turning_on); 1268 /** 1269 * update_parent_subparts_cpumask - update subparts_cpus mask of parent cpuset 1270 * @cpuset: The cpuset that requests change in partition root state 1271 * @cmd: Partition root state change command 1272 * @newmask: Optional new cpumask for partcmd_update 1273 * @tmp: Temporary addmask and delmask 1274 * Return: 0 or a partition root state error code 1275 * 1276 * For partcmd_enable, the cpuset is being transformed from a non-partition 1277 * root to a partition root. The cpus_allowed mask of the given cpuset will 1278 * be put into parent's subparts_cpus and taken away from parent's 1279 * effective_cpus. The function will return 0 if all the CPUs listed in 1280 * cpus_allowed can be granted or an error code will be returned. 1281 * 1282 * For partcmd_disable, the cpuset is being transformed from a partition 1283 * root back to a non-partition root. Any CPUs in cpus_allowed that are in 1284 * parent's subparts_cpus will be taken away from that cpumask and put back 1285 * into parent's effective_cpus. 0 will always be returned. 1286 * 1287 * For partcmd_update, if the optional newmask is specified, the cpu list is 1288 * to be changed from cpus_allowed to newmask. Otherwise, cpus_allowed is 1289 * assumed to remain the same. The cpuset should either be a valid or invalid 1290 * partition root. The partition root state may change from valid to invalid 1291 * or vice versa. An error code will only be returned if transitioning from 1292 * invalid to valid violates the exclusivity rule. 1293 * 1294 * For partcmd_invalidate, the current partition will be made invalid. 1295 * 1296 * The partcmd_enable and partcmd_disable commands are used by 1297 * update_prstate(). An error code may be returned and the caller will check 1298 * for error. 1299 * 1300 * The partcmd_update command is used by update_cpumasks_hier() with newmask 1301 * NULL and update_cpumask() with newmask set. The partcmd_invalidate is used 1302 * by update_cpumask() with NULL newmask. In both cases, the callers won't 1303 * check for error and so partition_root_state and prs_error will be updated 1304 * directly. 1305 */ 1306 static int update_parent_subparts_cpumask(struct cpuset *cs, int cmd, 1307 struct cpumask *newmask, 1308 struct tmpmasks *tmp) 1309 { 1310 struct cpuset *parent = parent_cs(cs); 1311 int adding; /* Moving cpus from effective_cpus to subparts_cpus */ 1312 int deleting; /* Moving cpus from subparts_cpus to effective_cpus */ 1313 int old_prs, new_prs; 1314 int part_error = PERR_NONE; /* Partition error? */ 1315 1316 percpu_rwsem_assert_held(&cpuset_rwsem); 1317 1318 /* 1319 * The parent must be a partition root. 1320 * The new cpumask, if present, or the current cpus_allowed must 1321 * not be empty. 1322 */ 1323 if (!is_partition_valid(parent)) { 1324 return is_partition_invalid(parent) 1325 ? PERR_INVPARENT : PERR_NOTPART; 1326 } 1327 if ((newmask && cpumask_empty(newmask)) || 1328 (!newmask && cpumask_empty(cs->cpus_allowed))) 1329 return PERR_CPUSEMPTY; 1330 1331 /* 1332 * new_prs will only be changed for the partcmd_update and 1333 * partcmd_invalidate commands. 1334 */ 1335 adding = deleting = false; 1336 old_prs = new_prs = cs->partition_root_state; 1337 if (cmd == partcmd_enable) { 1338 /* 1339 * Enabling partition root is not allowed if cpus_allowed 1340 * doesn't overlap parent's cpus_allowed. 1341 */ 1342 if (!cpumask_intersects(cs->cpus_allowed, parent->cpus_allowed)) 1343 return PERR_INVCPUS; 1344 1345 /* 1346 * A parent can be left with no CPU as long as there is no 1347 * task directly associated with the parent partition. 1348 */ 1349 if (cpumask_subset(parent->effective_cpus, cs->cpus_allowed) && 1350 partition_is_populated(parent, cs)) 1351 return PERR_NOCPUS; 1352 1353 cpumask_copy(tmp->addmask, cs->cpus_allowed); 1354 adding = true; 1355 } else if (cmd == partcmd_disable) { 1356 /* 1357 * Need to remove cpus from parent's subparts_cpus for valid 1358 * partition root. 1359 */ 1360 deleting = !is_prs_invalid(old_prs) && 1361 cpumask_and(tmp->delmask, cs->cpus_allowed, 1362 parent->subparts_cpus); 1363 } else if (cmd == partcmd_invalidate) { 1364 if (is_prs_invalid(old_prs)) 1365 return 0; 1366 1367 /* 1368 * Make the current partition invalid. It is assumed that 1369 * invalidation is caused by violating cpu exclusivity rule. 1370 */ 1371 deleting = cpumask_and(tmp->delmask, cs->cpus_allowed, 1372 parent->subparts_cpus); 1373 if (old_prs > 0) { 1374 new_prs = -old_prs; 1375 part_error = PERR_NOTEXCL; 1376 } 1377 } else if (newmask) { 1378 /* 1379 * partcmd_update with newmask: 1380 * 1381 * Compute add/delete mask to/from subparts_cpus 1382 * 1383 * delmask = cpus_allowed & ~newmask & parent->subparts_cpus 1384 * addmask = newmask & parent->cpus_allowed 1385 * & ~parent->subparts_cpus 1386 */ 1387 cpumask_andnot(tmp->delmask, cs->cpus_allowed, newmask); 1388 deleting = cpumask_and(tmp->delmask, tmp->delmask, 1389 parent->subparts_cpus); 1390 1391 cpumask_and(tmp->addmask, newmask, parent->cpus_allowed); 1392 adding = cpumask_andnot(tmp->addmask, tmp->addmask, 1393 parent->subparts_cpus); 1394 /* 1395 * Make partition invalid if parent's effective_cpus could 1396 * become empty and there are tasks in the parent. 1397 */ 1398 if (adding && 1399 cpumask_subset(parent->effective_cpus, tmp->addmask) && 1400 !cpumask_intersects(tmp->delmask, cpu_active_mask) && 1401 partition_is_populated(parent, cs)) { 1402 part_error = PERR_NOCPUS; 1403 adding = false; 1404 deleting = cpumask_and(tmp->delmask, cs->cpus_allowed, 1405 parent->subparts_cpus); 1406 } 1407 } else { 1408 /* 1409 * partcmd_update w/o newmask: 1410 * 1411 * delmask = cpus_allowed & parent->subparts_cpus 1412 * addmask = cpus_allowed & parent->cpus_allowed 1413 * & ~parent->subparts_cpus 1414 * 1415 * This gets invoked either due to a hotplug event or from 1416 * update_cpumasks_hier(). This can cause the state of a 1417 * partition root to transition from valid to invalid or vice 1418 * versa. So we still need to compute the addmask and delmask. 1419 1420 * A partition error happens when: 1421 * 1) Cpuset is valid partition, but parent does not distribute 1422 * out any CPUs. 1423 * 2) Parent has tasks and all its effective CPUs will have 1424 * to be distributed out. 1425 */ 1426 cpumask_and(tmp->addmask, cs->cpus_allowed, 1427 parent->cpus_allowed); 1428 adding = cpumask_andnot(tmp->addmask, tmp->addmask, 1429 parent->subparts_cpus); 1430 1431 if ((is_partition_valid(cs) && !parent->nr_subparts_cpus) || 1432 (adding && 1433 cpumask_subset(parent->effective_cpus, tmp->addmask) && 1434 partition_is_populated(parent, cs))) { 1435 part_error = PERR_NOCPUS; 1436 adding = false; 1437 } 1438 1439 if (part_error && is_partition_valid(cs) && 1440 parent->nr_subparts_cpus) 1441 deleting = cpumask_and(tmp->delmask, cs->cpus_allowed, 1442 parent->subparts_cpus); 1443 } 1444 if (part_error) 1445 WRITE_ONCE(cs->prs_err, part_error); 1446 1447 if (cmd == partcmd_update) { 1448 /* 1449 * Check for possible transition between valid and invalid 1450 * partition root. 1451 */ 1452 switch (cs->partition_root_state) { 1453 case PRS_ROOT: 1454 case PRS_ISOLATED: 1455 if (part_error) 1456 new_prs = -old_prs; 1457 break; 1458 case PRS_INVALID_ROOT: 1459 case PRS_INVALID_ISOLATED: 1460 if (!part_error) 1461 new_prs = -old_prs; 1462 break; 1463 } 1464 } 1465 1466 if (!adding && !deleting && (new_prs == old_prs)) 1467 return 0; 1468 1469 /* 1470 * Transitioning between invalid to valid or vice versa may require 1471 * changing CS_CPU_EXCLUSIVE and CS_SCHED_LOAD_BALANCE. 1472 */ 1473 if (old_prs != new_prs) { 1474 if (is_prs_invalid(old_prs) && !is_cpu_exclusive(cs) && 1475 (update_flag(CS_CPU_EXCLUSIVE, cs, 1) < 0)) 1476 return PERR_NOTEXCL; 1477 if (is_prs_invalid(new_prs) && is_cpu_exclusive(cs)) 1478 update_flag(CS_CPU_EXCLUSIVE, cs, 0); 1479 } 1480 1481 /* 1482 * Change the parent's subparts_cpus. 1483 * Newly added CPUs will be removed from effective_cpus and 1484 * newly deleted ones will be added back to effective_cpus. 1485 */ 1486 spin_lock_irq(&callback_lock); 1487 if (adding) { 1488 cpumask_or(parent->subparts_cpus, 1489 parent->subparts_cpus, tmp->addmask); 1490 cpumask_andnot(parent->effective_cpus, 1491 parent->effective_cpus, tmp->addmask); 1492 } 1493 if (deleting) { 1494 cpumask_andnot(parent->subparts_cpus, 1495 parent->subparts_cpus, tmp->delmask); 1496 /* 1497 * Some of the CPUs in subparts_cpus might have been offlined. 1498 */ 1499 cpumask_and(tmp->delmask, tmp->delmask, cpu_active_mask); 1500 cpumask_or(parent->effective_cpus, 1501 parent->effective_cpus, tmp->delmask); 1502 } 1503 1504 parent->nr_subparts_cpus = cpumask_weight(parent->subparts_cpus); 1505 1506 if (old_prs != new_prs) 1507 cs->partition_root_state = new_prs; 1508 1509 spin_unlock_irq(&callback_lock); 1510 1511 if (adding || deleting) 1512 update_tasks_cpumask(parent); 1513 1514 /* 1515 * Set or clear CS_SCHED_LOAD_BALANCE when partcmd_update, if necessary. 1516 * rebuild_sched_domains_locked() may be called. 1517 */ 1518 if (old_prs != new_prs) { 1519 if (old_prs == PRS_ISOLATED) 1520 update_flag(CS_SCHED_LOAD_BALANCE, cs, 1); 1521 else if (new_prs == PRS_ISOLATED) 1522 update_flag(CS_SCHED_LOAD_BALANCE, cs, 0); 1523 } 1524 notify_partition_change(cs, old_prs); 1525 return 0; 1526 } 1527 1528 /* 1529 * update_cpumasks_hier - Update effective cpumasks and tasks in the subtree 1530 * @cs: the cpuset to consider 1531 * @tmp: temp variables for calculating effective_cpus & partition setup 1532 * @force: don't skip any descendant cpusets if set 1533 * 1534 * When configured cpumask is changed, the effective cpumasks of this cpuset 1535 * and all its descendants need to be updated. 1536 * 1537 * On legacy hierarchy, effective_cpus will be the same with cpu_allowed. 1538 * 1539 * Called with cpuset_rwsem held 1540 */ 1541 static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp, 1542 bool force) 1543 { 1544 struct cpuset *cp; 1545 struct cgroup_subsys_state *pos_css; 1546 bool need_rebuild_sched_domains = false; 1547 int old_prs, new_prs; 1548 1549 rcu_read_lock(); 1550 cpuset_for_each_descendant_pre(cp, pos_css, cs) { 1551 struct cpuset *parent = parent_cs(cp); 1552 bool update_parent = false; 1553 1554 compute_effective_cpumask(tmp->new_cpus, cp, parent); 1555 1556 /* 1557 * If it becomes empty, inherit the effective mask of the 1558 * parent, which is guaranteed to have some CPUs unless 1559 * it is a partition root that has explicitly distributed 1560 * out all its CPUs. 1561 */ 1562 if (is_in_v2_mode() && cpumask_empty(tmp->new_cpus)) { 1563 if (is_partition_valid(cp) && 1564 cpumask_equal(cp->cpus_allowed, cp->subparts_cpus)) 1565 goto update_parent_subparts; 1566 1567 cpumask_copy(tmp->new_cpus, parent->effective_cpus); 1568 if (!cp->use_parent_ecpus) { 1569 cp->use_parent_ecpus = true; 1570 parent->child_ecpus_count++; 1571 } 1572 } else if (cp->use_parent_ecpus) { 1573 cp->use_parent_ecpus = false; 1574 WARN_ON_ONCE(!parent->child_ecpus_count); 1575 parent->child_ecpus_count--; 1576 } 1577 1578 /* 1579 * Skip the whole subtree if the cpumask remains the same 1580 * and has no partition root state and force flag not set. 1581 */ 1582 if (!cp->partition_root_state && !force && 1583 cpumask_equal(tmp->new_cpus, cp->effective_cpus)) { 1584 pos_css = css_rightmost_descendant(pos_css); 1585 continue; 1586 } 1587 1588 update_parent_subparts: 1589 /* 1590 * update_parent_subparts_cpumask() should have been called 1591 * for cs already in update_cpumask(). We should also call 1592 * update_tasks_cpumask() again for tasks in the parent 1593 * cpuset if the parent's subparts_cpus changes. 1594 */ 1595 old_prs = new_prs = cp->partition_root_state; 1596 if ((cp != cs) && old_prs) { 1597 switch (parent->partition_root_state) { 1598 case PRS_ROOT: 1599 case PRS_ISOLATED: 1600 update_parent = true; 1601 break; 1602 1603 default: 1604 /* 1605 * When parent is not a partition root or is 1606 * invalid, child partition roots become 1607 * invalid too. 1608 */ 1609 if (is_partition_valid(cp)) 1610 new_prs = -cp->partition_root_state; 1611 WRITE_ONCE(cp->prs_err, 1612 is_partition_invalid(parent) 1613 ? PERR_INVPARENT : PERR_NOTPART); 1614 break; 1615 } 1616 } 1617 1618 if (!css_tryget_online(&cp->css)) 1619 continue; 1620 rcu_read_unlock(); 1621 1622 if (update_parent) { 1623 update_parent_subparts_cpumask(cp, partcmd_update, NULL, 1624 tmp); 1625 /* 1626 * The cpuset partition_root_state may become 1627 * invalid. Capture it. 1628 */ 1629 new_prs = cp->partition_root_state; 1630 } 1631 1632 spin_lock_irq(&callback_lock); 1633 1634 if (cp->nr_subparts_cpus && !is_partition_valid(cp)) { 1635 /* 1636 * Put all active subparts_cpus back to effective_cpus. 1637 */ 1638 cpumask_or(tmp->new_cpus, tmp->new_cpus, 1639 cp->subparts_cpus); 1640 cpumask_and(tmp->new_cpus, tmp->new_cpus, 1641 cpu_active_mask); 1642 cp->nr_subparts_cpus = 0; 1643 cpumask_clear(cp->subparts_cpus); 1644 } 1645 1646 cpumask_copy(cp->effective_cpus, tmp->new_cpus); 1647 if (cp->nr_subparts_cpus) { 1648 /* 1649 * Make sure that effective_cpus & subparts_cpus 1650 * are mutually exclusive. 1651 */ 1652 cpumask_andnot(cp->effective_cpus, cp->effective_cpus, 1653 cp->subparts_cpus); 1654 } 1655 1656 cp->partition_root_state = new_prs; 1657 spin_unlock_irq(&callback_lock); 1658 1659 notify_partition_change(cp, old_prs); 1660 1661 WARN_ON(!is_in_v2_mode() && 1662 !cpumask_equal(cp->cpus_allowed, cp->effective_cpus)); 1663 1664 update_tasks_cpumask(cp); 1665 1666 /* 1667 * On legacy hierarchy, if the effective cpumask of any non- 1668 * empty cpuset is changed, we need to rebuild sched domains. 1669 * On default hierarchy, the cpuset needs to be a partition 1670 * root as well. 1671 */ 1672 if (!cpumask_empty(cp->cpus_allowed) && 1673 is_sched_load_balance(cp) && 1674 (!cgroup_subsys_on_dfl(cpuset_cgrp_subsys) || 1675 is_partition_valid(cp))) 1676 need_rebuild_sched_domains = true; 1677 1678 rcu_read_lock(); 1679 css_put(&cp->css); 1680 } 1681 rcu_read_unlock(); 1682 1683 if (need_rebuild_sched_domains) 1684 rebuild_sched_domains_locked(); 1685 } 1686 1687 /** 1688 * update_sibling_cpumasks - Update siblings cpumasks 1689 * @parent: Parent cpuset 1690 * @cs: Current cpuset 1691 * @tmp: Temp variables 1692 */ 1693 static void update_sibling_cpumasks(struct cpuset *parent, struct cpuset *cs, 1694 struct tmpmasks *tmp) 1695 { 1696 struct cpuset *sibling; 1697 struct cgroup_subsys_state *pos_css; 1698 1699 percpu_rwsem_assert_held(&cpuset_rwsem); 1700 1701 /* 1702 * Check all its siblings and call update_cpumasks_hier() 1703 * if their use_parent_ecpus flag is set in order for them 1704 * to use the right effective_cpus value. 1705 * 1706 * The update_cpumasks_hier() function may sleep. So we have to 1707 * release the RCU read lock before calling it. 1708 */ 1709 rcu_read_lock(); 1710 cpuset_for_each_child(sibling, pos_css, parent) { 1711 if (sibling == cs) 1712 continue; 1713 if (!sibling->use_parent_ecpus) 1714 continue; 1715 if (!css_tryget_online(&sibling->css)) 1716 continue; 1717 1718 rcu_read_unlock(); 1719 update_cpumasks_hier(sibling, tmp, false); 1720 rcu_read_lock(); 1721 css_put(&sibling->css); 1722 } 1723 rcu_read_unlock(); 1724 } 1725 1726 /** 1727 * update_cpumask - update the cpus_allowed mask of a cpuset and all tasks in it 1728 * @cs: the cpuset to consider 1729 * @trialcs: trial cpuset 1730 * @buf: buffer of cpu numbers written to this cpuset 1731 */ 1732 static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs, 1733 const char *buf) 1734 { 1735 int retval; 1736 struct tmpmasks tmp; 1737 bool invalidate = false; 1738 1739 /* top_cpuset.cpus_allowed tracks cpu_online_mask; it's read-only */ 1740 if (cs == &top_cpuset) 1741 return -EACCES; 1742 1743 /* 1744 * An empty cpus_allowed is ok only if the cpuset has no tasks. 1745 * Since cpulist_parse() fails on an empty mask, we special case 1746 * that parsing. The validate_change() call ensures that cpusets 1747 * with tasks have cpus. 1748 */ 1749 if (!*buf) { 1750 cpumask_clear(trialcs->cpus_allowed); 1751 } else { 1752 retval = cpulist_parse(buf, trialcs->cpus_allowed); 1753 if (retval < 0) 1754 return retval; 1755 1756 if (!cpumask_subset(trialcs->cpus_allowed, 1757 top_cpuset.cpus_allowed)) 1758 return -EINVAL; 1759 } 1760 1761 /* Nothing to do if the cpus didn't change */ 1762 if (cpumask_equal(cs->cpus_allowed, trialcs->cpus_allowed)) 1763 return 0; 1764 1765 #ifdef CONFIG_CPUMASK_OFFSTACK 1766 /* 1767 * Use the cpumasks in trialcs for tmpmasks when they are pointers 1768 * to allocated cpumasks. 1769 */ 1770 tmp.addmask = trialcs->subparts_cpus; 1771 tmp.delmask = trialcs->effective_cpus; 1772 tmp.new_cpus = trialcs->cpus_allowed; 1773 #endif 1774 1775 retval = validate_change(cs, trialcs); 1776 1777 if ((retval == -EINVAL) && cgroup_subsys_on_dfl(cpuset_cgrp_subsys)) { 1778 struct cpuset *cp, *parent; 1779 struct cgroup_subsys_state *css; 1780 1781 /* 1782 * The -EINVAL error code indicates that partition sibling 1783 * CPU exclusivity rule has been violated. We still allow 1784 * the cpumask change to proceed while invalidating the 1785 * partition. However, any conflicting sibling partitions 1786 * have to be marked as invalid too. 1787 */ 1788 invalidate = true; 1789 rcu_read_lock(); 1790 parent = parent_cs(cs); 1791 cpuset_for_each_child(cp, css, parent) 1792 if (is_partition_valid(cp) && 1793 cpumask_intersects(trialcs->cpus_allowed, cp->cpus_allowed)) { 1794 rcu_read_unlock(); 1795 update_parent_subparts_cpumask(cp, partcmd_invalidate, NULL, &tmp); 1796 rcu_read_lock(); 1797 } 1798 rcu_read_unlock(); 1799 retval = 0; 1800 } 1801 if (retval < 0) 1802 return retval; 1803 1804 if (cs->partition_root_state) { 1805 if (invalidate) 1806 update_parent_subparts_cpumask(cs, partcmd_invalidate, 1807 NULL, &tmp); 1808 else 1809 update_parent_subparts_cpumask(cs, partcmd_update, 1810 trialcs->cpus_allowed, &tmp); 1811 } 1812 1813 compute_effective_cpumask(trialcs->effective_cpus, trialcs, 1814 parent_cs(cs)); 1815 spin_lock_irq(&callback_lock); 1816 cpumask_copy(cs->cpus_allowed, trialcs->cpus_allowed); 1817 1818 /* 1819 * Make sure that subparts_cpus, if not empty, is a subset of 1820 * cpus_allowed. Clear subparts_cpus if partition not valid or 1821 * empty effective cpus with tasks. 1822 */ 1823 if (cs->nr_subparts_cpus) { 1824 if (!is_partition_valid(cs) || 1825 (cpumask_subset(trialcs->effective_cpus, cs->subparts_cpus) && 1826 partition_is_populated(cs, NULL))) { 1827 cs->nr_subparts_cpus = 0; 1828 cpumask_clear(cs->subparts_cpus); 1829 } else { 1830 cpumask_and(cs->subparts_cpus, cs->subparts_cpus, 1831 cs->cpus_allowed); 1832 cs->nr_subparts_cpus = cpumask_weight(cs->subparts_cpus); 1833 } 1834 } 1835 spin_unlock_irq(&callback_lock); 1836 1837 /* effective_cpus will be updated here */ 1838 update_cpumasks_hier(cs, &tmp, false); 1839 1840 if (cs->partition_root_state) { 1841 struct cpuset *parent = parent_cs(cs); 1842 1843 /* 1844 * For partition root, update the cpumasks of sibling 1845 * cpusets if they use parent's effective_cpus. 1846 */ 1847 if (parent->child_ecpus_count) 1848 update_sibling_cpumasks(parent, cs, &tmp); 1849 } 1850 return 0; 1851 } 1852 1853 /* 1854 * Migrate memory region from one set of nodes to another. This is 1855 * performed asynchronously as it can be called from process migration path 1856 * holding locks involved in process management. All mm migrations are 1857 * performed in the queued order and can be waited for by flushing 1858 * cpuset_migrate_mm_wq. 1859 */ 1860 1861 struct cpuset_migrate_mm_work { 1862 struct work_struct work; 1863 struct mm_struct *mm; 1864 nodemask_t from; 1865 nodemask_t to; 1866 }; 1867 1868 static void cpuset_migrate_mm_workfn(struct work_struct *work) 1869 { 1870 struct cpuset_migrate_mm_work *mwork = 1871 container_of(work, struct cpuset_migrate_mm_work, work); 1872 1873 /* on a wq worker, no need to worry about %current's mems_allowed */ 1874 do_migrate_pages(mwork->mm, &mwork->from, &mwork->to, MPOL_MF_MOVE_ALL); 1875 mmput(mwork->mm); 1876 kfree(mwork); 1877 } 1878 1879 static void cpuset_migrate_mm(struct mm_struct *mm, const nodemask_t *from, 1880 const nodemask_t *to) 1881 { 1882 struct cpuset_migrate_mm_work *mwork; 1883 1884 if (nodes_equal(*from, *to)) { 1885 mmput(mm); 1886 return; 1887 } 1888 1889 mwork = kzalloc(sizeof(*mwork), GFP_KERNEL); 1890 if (mwork) { 1891 mwork->mm = mm; 1892 mwork->from = *from; 1893 mwork->to = *to; 1894 INIT_WORK(&mwork->work, cpuset_migrate_mm_workfn); 1895 queue_work(cpuset_migrate_mm_wq, &mwork->work); 1896 } else { 1897 mmput(mm); 1898 } 1899 } 1900 1901 static void cpuset_post_attach(void) 1902 { 1903 flush_workqueue(cpuset_migrate_mm_wq); 1904 } 1905 1906 /* 1907 * cpuset_change_task_nodemask - change task's mems_allowed and mempolicy 1908 * @tsk: the task to change 1909 * @newmems: new nodes that the task will be set 1910 * 1911 * We use the mems_allowed_seq seqlock to safely update both tsk->mems_allowed 1912 * and rebind an eventual tasks' mempolicy. If the task is allocating in 1913 * parallel, it might temporarily see an empty intersection, which results in 1914 * a seqlock check and retry before OOM or allocation failure. 1915 */ 1916 static void cpuset_change_task_nodemask(struct task_struct *tsk, 1917 nodemask_t *newmems) 1918 { 1919 task_lock(tsk); 1920 1921 local_irq_disable(); 1922 write_seqcount_begin(&tsk->mems_allowed_seq); 1923 1924 nodes_or(tsk->mems_allowed, tsk->mems_allowed, *newmems); 1925 mpol_rebind_task(tsk, newmems); 1926 tsk->mems_allowed = *newmems; 1927 1928 write_seqcount_end(&tsk->mems_allowed_seq); 1929 local_irq_enable(); 1930 1931 task_unlock(tsk); 1932 } 1933 1934 static void *cpuset_being_rebound; 1935 1936 /** 1937 * update_tasks_nodemask - Update the nodemasks of tasks in the cpuset. 1938 * @cs: the cpuset in which each task's mems_allowed mask needs to be changed 1939 * 1940 * Iterate through each task of @cs updating its mems_allowed to the 1941 * effective cpuset's. As this function is called with cpuset_rwsem held, 1942 * cpuset membership stays stable. 1943 */ 1944 static void update_tasks_nodemask(struct cpuset *cs) 1945 { 1946 static nodemask_t newmems; /* protected by cpuset_rwsem */ 1947 struct css_task_iter it; 1948 struct task_struct *task; 1949 1950 cpuset_being_rebound = cs; /* causes mpol_dup() rebind */ 1951 1952 guarantee_online_mems(cs, &newmems); 1953 1954 /* 1955 * The mpol_rebind_mm() call takes mmap_lock, which we couldn't 1956 * take while holding tasklist_lock. Forks can happen - the 1957 * mpol_dup() cpuset_being_rebound check will catch such forks, 1958 * and rebind their vma mempolicies too. Because we still hold 1959 * the global cpuset_rwsem, we know that no other rebind effort 1960 * will be contending for the global variable cpuset_being_rebound. 1961 * It's ok if we rebind the same mm twice; mpol_rebind_mm() 1962 * is idempotent. Also migrate pages in each mm to new nodes. 1963 */ 1964 css_task_iter_start(&cs->css, 0, &it); 1965 while ((task = css_task_iter_next(&it))) { 1966 struct mm_struct *mm; 1967 bool migrate; 1968 1969 cpuset_change_task_nodemask(task, &newmems); 1970 1971 mm = get_task_mm(task); 1972 if (!mm) 1973 continue; 1974 1975 migrate = is_memory_migrate(cs); 1976 1977 mpol_rebind_mm(mm, &cs->mems_allowed); 1978 if (migrate) 1979 cpuset_migrate_mm(mm, &cs->old_mems_allowed, &newmems); 1980 else 1981 mmput(mm); 1982 } 1983 css_task_iter_end(&it); 1984 1985 /* 1986 * All the tasks' nodemasks have been updated, update 1987 * cs->old_mems_allowed. 1988 */ 1989 cs->old_mems_allowed = newmems; 1990 1991 /* We're done rebinding vmas to this cpuset's new mems_allowed. */ 1992 cpuset_being_rebound = NULL; 1993 } 1994 1995 /* 1996 * update_nodemasks_hier - Update effective nodemasks and tasks in the subtree 1997 * @cs: the cpuset to consider 1998 * @new_mems: a temp variable for calculating new effective_mems 1999 * 2000 * When configured nodemask is changed, the effective nodemasks of this cpuset 2001 * and all its descendants need to be updated. 2002 * 2003 * On legacy hierarchy, effective_mems will be the same with mems_allowed. 2004 * 2005 * Called with cpuset_rwsem held 2006 */ 2007 static void update_nodemasks_hier(struct cpuset *cs, nodemask_t *new_mems) 2008 { 2009 struct cpuset *cp; 2010 struct cgroup_subsys_state *pos_css; 2011 2012 rcu_read_lock(); 2013 cpuset_for_each_descendant_pre(cp, pos_css, cs) { 2014 struct cpuset *parent = parent_cs(cp); 2015 2016 nodes_and(*new_mems, cp->mems_allowed, parent->effective_mems); 2017 2018 /* 2019 * If it becomes empty, inherit the effective mask of the 2020 * parent, which is guaranteed to have some MEMs. 2021 */ 2022 if (is_in_v2_mode() && nodes_empty(*new_mems)) 2023 *new_mems = parent->effective_mems; 2024 2025 /* Skip the whole subtree if the nodemask remains the same. */ 2026 if (nodes_equal(*new_mems, cp->effective_mems)) { 2027 pos_css = css_rightmost_descendant(pos_css); 2028 continue; 2029 } 2030 2031 if (!css_tryget_online(&cp->css)) 2032 continue; 2033 rcu_read_unlock(); 2034 2035 spin_lock_irq(&callback_lock); 2036 cp->effective_mems = *new_mems; 2037 spin_unlock_irq(&callback_lock); 2038 2039 WARN_ON(!is_in_v2_mode() && 2040 !nodes_equal(cp->mems_allowed, cp->effective_mems)); 2041 2042 update_tasks_nodemask(cp); 2043 2044 rcu_read_lock(); 2045 css_put(&cp->css); 2046 } 2047 rcu_read_unlock(); 2048 } 2049 2050 /* 2051 * Handle user request to change the 'mems' memory placement 2052 * of a cpuset. Needs to validate the request, update the 2053 * cpusets mems_allowed, and for each task in the cpuset, 2054 * update mems_allowed and rebind task's mempolicy and any vma 2055 * mempolicies and if the cpuset is marked 'memory_migrate', 2056 * migrate the tasks pages to the new memory. 2057 * 2058 * Call with cpuset_rwsem held. May take callback_lock during call. 2059 * Will take tasklist_lock, scan tasklist for tasks in cpuset cs, 2060 * lock each such tasks mm->mmap_lock, scan its vma's and rebind 2061 * their mempolicies to the cpusets new mems_allowed. 2062 */ 2063 static int update_nodemask(struct cpuset *cs, struct cpuset *trialcs, 2064 const char *buf) 2065 { 2066 int retval; 2067 2068 /* 2069 * top_cpuset.mems_allowed tracks node_stats[N_MEMORY]; 2070 * it's read-only 2071 */ 2072 if (cs == &top_cpuset) { 2073 retval = -EACCES; 2074 goto done; 2075 } 2076 2077 /* 2078 * An empty mems_allowed is ok iff there are no tasks in the cpuset. 2079 * Since nodelist_parse() fails on an empty mask, we special case 2080 * that parsing. The validate_change() call ensures that cpusets 2081 * with tasks have memory. 2082 */ 2083 if (!*buf) { 2084 nodes_clear(trialcs->mems_allowed); 2085 } else { 2086 retval = nodelist_parse(buf, trialcs->mems_allowed); 2087 if (retval < 0) 2088 goto done; 2089 2090 if (!nodes_subset(trialcs->mems_allowed, 2091 top_cpuset.mems_allowed)) { 2092 retval = -EINVAL; 2093 goto done; 2094 } 2095 } 2096 2097 if (nodes_equal(cs->mems_allowed, trialcs->mems_allowed)) { 2098 retval = 0; /* Too easy - nothing to do */ 2099 goto done; 2100 } 2101 retval = validate_change(cs, trialcs); 2102 if (retval < 0) 2103 goto done; 2104 2105 check_insane_mems_config(&trialcs->mems_allowed); 2106 2107 spin_lock_irq(&callback_lock); 2108 cs->mems_allowed = trialcs->mems_allowed; 2109 spin_unlock_irq(&callback_lock); 2110 2111 /* use trialcs->mems_allowed as a temp variable */ 2112 update_nodemasks_hier(cs, &trialcs->mems_allowed); 2113 done: 2114 return retval; 2115 } 2116 2117 bool current_cpuset_is_being_rebound(void) 2118 { 2119 bool ret; 2120 2121 rcu_read_lock(); 2122 ret = task_cs(current) == cpuset_being_rebound; 2123 rcu_read_unlock(); 2124 2125 return ret; 2126 } 2127 2128 static int update_relax_domain_level(struct cpuset *cs, s64 val) 2129 { 2130 #ifdef CONFIG_SMP 2131 if (val < -1 || val >= sched_domain_level_max) 2132 return -EINVAL; 2133 #endif 2134 2135 if (val != cs->relax_domain_level) { 2136 cs->relax_domain_level = val; 2137 if (!cpumask_empty(cs->cpus_allowed) && 2138 is_sched_load_balance(cs)) 2139 rebuild_sched_domains_locked(); 2140 } 2141 2142 return 0; 2143 } 2144 2145 /** 2146 * update_tasks_flags - update the spread flags of tasks in the cpuset. 2147 * @cs: the cpuset in which each task's spread flags needs to be changed 2148 * 2149 * Iterate through each task of @cs updating its spread flags. As this 2150 * function is called with cpuset_rwsem held, cpuset membership stays 2151 * stable. 2152 */ 2153 static void update_tasks_flags(struct cpuset *cs) 2154 { 2155 struct css_task_iter it; 2156 struct task_struct *task; 2157 2158 css_task_iter_start(&cs->css, 0, &it); 2159 while ((task = css_task_iter_next(&it))) 2160 cpuset_update_task_spread_flags(cs, task); 2161 css_task_iter_end(&it); 2162 } 2163 2164 /* 2165 * update_flag - read a 0 or a 1 in a file and update associated flag 2166 * bit: the bit to update (see cpuset_flagbits_t) 2167 * cs: the cpuset to update 2168 * turning_on: whether the flag is being set or cleared 2169 * 2170 * Call with cpuset_rwsem held. 2171 */ 2172 2173 static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs, 2174 int turning_on) 2175 { 2176 struct cpuset *trialcs; 2177 int balance_flag_changed; 2178 int spread_flag_changed; 2179 int err; 2180 2181 trialcs = alloc_trial_cpuset(cs); 2182 if (!trialcs) 2183 return -ENOMEM; 2184 2185 if (turning_on) 2186 set_bit(bit, &trialcs->flags); 2187 else 2188 clear_bit(bit, &trialcs->flags); 2189 2190 err = validate_change(cs, trialcs); 2191 if (err < 0) 2192 goto out; 2193 2194 balance_flag_changed = (is_sched_load_balance(cs) != 2195 is_sched_load_balance(trialcs)); 2196 2197 spread_flag_changed = ((is_spread_slab(cs) != is_spread_slab(trialcs)) 2198 || (is_spread_page(cs) != is_spread_page(trialcs))); 2199 2200 spin_lock_irq(&callback_lock); 2201 cs->flags = trialcs->flags; 2202 spin_unlock_irq(&callback_lock); 2203 2204 if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed) 2205 rebuild_sched_domains_locked(); 2206 2207 if (spread_flag_changed) 2208 update_tasks_flags(cs); 2209 out: 2210 free_cpuset(trialcs); 2211 return err; 2212 } 2213 2214 /** 2215 * update_prstate - update partition_root_state 2216 * @cs: the cpuset to update 2217 * @new_prs: new partition root state 2218 * Return: 0 if successful, != 0 if error 2219 * 2220 * Call with cpuset_rwsem held. 2221 */ 2222 static int update_prstate(struct cpuset *cs, int new_prs) 2223 { 2224 int err = PERR_NONE, old_prs = cs->partition_root_state; 2225 bool sched_domain_rebuilt = false; 2226 struct cpuset *parent = parent_cs(cs); 2227 struct tmpmasks tmpmask; 2228 2229 if (old_prs == new_prs) 2230 return 0; 2231 2232 /* 2233 * For a previously invalid partition root, leave it at being 2234 * invalid if new_prs is not "member". 2235 */ 2236 if (new_prs && is_prs_invalid(old_prs)) { 2237 cs->partition_root_state = -new_prs; 2238 return 0; 2239 } 2240 2241 if (alloc_cpumasks(NULL, &tmpmask)) 2242 return -ENOMEM; 2243 2244 if (!old_prs) { 2245 /* 2246 * Turning on partition root requires setting the 2247 * CS_CPU_EXCLUSIVE bit implicitly as well and cpus_allowed 2248 * cannot be empty. 2249 */ 2250 if (cpumask_empty(cs->cpus_allowed)) { 2251 err = PERR_CPUSEMPTY; 2252 goto out; 2253 } 2254 2255 err = update_flag(CS_CPU_EXCLUSIVE, cs, 1); 2256 if (err) { 2257 err = PERR_NOTEXCL; 2258 goto out; 2259 } 2260 2261 err = update_parent_subparts_cpumask(cs, partcmd_enable, 2262 NULL, &tmpmask); 2263 if (err) { 2264 update_flag(CS_CPU_EXCLUSIVE, cs, 0); 2265 goto out; 2266 } 2267 2268 if (new_prs == PRS_ISOLATED) { 2269 /* 2270 * Disable the load balance flag should not return an 2271 * error unless the system is running out of memory. 2272 */ 2273 update_flag(CS_SCHED_LOAD_BALANCE, cs, 0); 2274 sched_domain_rebuilt = true; 2275 } 2276 } else if (old_prs && new_prs) { 2277 /* 2278 * A change in load balance state only, no change in cpumasks. 2279 */ 2280 update_flag(CS_SCHED_LOAD_BALANCE, cs, (new_prs != PRS_ISOLATED)); 2281 sched_domain_rebuilt = true; 2282 goto out; /* Sched domain is rebuilt in update_flag() */ 2283 } else { 2284 /* 2285 * Switching back to member is always allowed even if it 2286 * disables child partitions. 2287 */ 2288 update_parent_subparts_cpumask(cs, partcmd_disable, NULL, 2289 &tmpmask); 2290 2291 /* 2292 * If there are child partitions, they will all become invalid. 2293 */ 2294 if (unlikely(cs->nr_subparts_cpus)) { 2295 spin_lock_irq(&callback_lock); 2296 cs->nr_subparts_cpus = 0; 2297 cpumask_clear(cs->subparts_cpus); 2298 compute_effective_cpumask(cs->effective_cpus, cs, parent); 2299 spin_unlock_irq(&callback_lock); 2300 } 2301 2302 /* Turning off CS_CPU_EXCLUSIVE will not return error */ 2303 update_flag(CS_CPU_EXCLUSIVE, cs, 0); 2304 2305 if (!is_sched_load_balance(cs)) { 2306 /* Make sure load balance is on */ 2307 update_flag(CS_SCHED_LOAD_BALANCE, cs, 1); 2308 sched_domain_rebuilt = true; 2309 } 2310 } 2311 2312 update_tasks_cpumask(parent); 2313 2314 if (parent->child_ecpus_count) 2315 update_sibling_cpumasks(parent, cs, &tmpmask); 2316 2317 if (!sched_domain_rebuilt) 2318 rebuild_sched_domains_locked(); 2319 out: 2320 /* 2321 * Make partition invalid if an error happen 2322 */ 2323 if (err) 2324 new_prs = -new_prs; 2325 spin_lock_irq(&callback_lock); 2326 cs->partition_root_state = new_prs; 2327 WRITE_ONCE(cs->prs_err, err); 2328 spin_unlock_irq(&callback_lock); 2329 /* 2330 * Update child cpusets, if present. 2331 * Force update if switching back to member. 2332 */ 2333 if (!list_empty(&cs->css.children)) 2334 update_cpumasks_hier(cs, &tmpmask, !new_prs); 2335 2336 notify_partition_change(cs, old_prs); 2337 free_cpumasks(NULL, &tmpmask); 2338 return 0; 2339 } 2340 2341 /* 2342 * Frequency meter - How fast is some event occurring? 2343 * 2344 * These routines manage a digitally filtered, constant time based, 2345 * event frequency meter. There are four routines: 2346 * fmeter_init() - initialize a frequency meter. 2347 * fmeter_markevent() - called each time the event happens. 2348 * fmeter_getrate() - returns the recent rate of such events. 2349 * fmeter_update() - internal routine used to update fmeter. 2350 * 2351 * A common data structure is passed to each of these routines, 2352 * which is used to keep track of the state required to manage the 2353 * frequency meter and its digital filter. 2354 * 2355 * The filter works on the number of events marked per unit time. 2356 * The filter is single-pole low-pass recursive (IIR). The time unit 2357 * is 1 second. Arithmetic is done using 32-bit integers scaled to 2358 * simulate 3 decimal digits of precision (multiplied by 1000). 2359 * 2360 * With an FM_COEF of 933, and a time base of 1 second, the filter 2361 * has a half-life of 10 seconds, meaning that if the events quit 2362 * happening, then the rate returned from the fmeter_getrate() 2363 * will be cut in half each 10 seconds, until it converges to zero. 2364 * 2365 * It is not worth doing a real infinitely recursive filter. If more 2366 * than FM_MAXTICKS ticks have elapsed since the last filter event, 2367 * just compute FM_MAXTICKS ticks worth, by which point the level 2368 * will be stable. 2369 * 2370 * Limit the count of unprocessed events to FM_MAXCNT, so as to avoid 2371 * arithmetic overflow in the fmeter_update() routine. 2372 * 2373 * Given the simple 32 bit integer arithmetic used, this meter works 2374 * best for reporting rates between one per millisecond (msec) and 2375 * one per 32 (approx) seconds. At constant rates faster than one 2376 * per msec it maxes out at values just under 1,000,000. At constant 2377 * rates between one per msec, and one per second it will stabilize 2378 * to a value N*1000, where N is the rate of events per second. 2379 * At constant rates between one per second and one per 32 seconds, 2380 * it will be choppy, moving up on the seconds that have an event, 2381 * and then decaying until the next event. At rates slower than 2382 * about one in 32 seconds, it decays all the way back to zero between 2383 * each event. 2384 */ 2385 2386 #define FM_COEF 933 /* coefficient for half-life of 10 secs */ 2387 #define FM_MAXTICKS ((u32)99) /* useless computing more ticks than this */ 2388 #define FM_MAXCNT 1000000 /* limit cnt to avoid overflow */ 2389 #define FM_SCALE 1000 /* faux fixed point scale */ 2390 2391 /* Initialize a frequency meter */ 2392 static void fmeter_init(struct fmeter *fmp) 2393 { 2394 fmp->cnt = 0; 2395 fmp->val = 0; 2396 fmp->time = 0; 2397 spin_lock_init(&fmp->lock); 2398 } 2399 2400 /* Internal meter update - process cnt events and update value */ 2401 static void fmeter_update(struct fmeter *fmp) 2402 { 2403 time64_t now; 2404 u32 ticks; 2405 2406 now = ktime_get_seconds(); 2407 ticks = now - fmp->time; 2408 2409 if (ticks == 0) 2410 return; 2411 2412 ticks = min(FM_MAXTICKS, ticks); 2413 while (ticks-- > 0) 2414 fmp->val = (FM_COEF * fmp->val) / FM_SCALE; 2415 fmp->time = now; 2416 2417 fmp->val += ((FM_SCALE - FM_COEF) * fmp->cnt) / FM_SCALE; 2418 fmp->cnt = 0; 2419 } 2420 2421 /* Process any previous ticks, then bump cnt by one (times scale). */ 2422 static void fmeter_markevent(struct fmeter *fmp) 2423 { 2424 spin_lock(&fmp->lock); 2425 fmeter_update(fmp); 2426 fmp->cnt = min(FM_MAXCNT, fmp->cnt + FM_SCALE); 2427 spin_unlock(&fmp->lock); 2428 } 2429 2430 /* Process any previous ticks, then return current value. */ 2431 static int fmeter_getrate(struct fmeter *fmp) 2432 { 2433 int val; 2434 2435 spin_lock(&fmp->lock); 2436 fmeter_update(fmp); 2437 val = fmp->val; 2438 spin_unlock(&fmp->lock); 2439 return val; 2440 } 2441 2442 static struct cpuset *cpuset_attach_old_cs; 2443 2444 /* Called by cgroups to determine if a cpuset is usable; cpuset_rwsem held */ 2445 static int cpuset_can_attach(struct cgroup_taskset *tset) 2446 { 2447 struct cgroup_subsys_state *css; 2448 struct cpuset *cs; 2449 struct task_struct *task; 2450 int ret; 2451 2452 /* used later by cpuset_attach() */ 2453 cpuset_attach_old_cs = task_cs(cgroup_taskset_first(tset, &css)); 2454 cs = css_cs(css); 2455 2456 percpu_down_write(&cpuset_rwsem); 2457 2458 /* allow moving tasks into an empty cpuset if on default hierarchy */ 2459 ret = -ENOSPC; 2460 if (!is_in_v2_mode() && 2461 (cpumask_empty(cs->cpus_allowed) || nodes_empty(cs->mems_allowed))) 2462 goto out_unlock; 2463 2464 /* 2465 * Task cannot be moved to a cpuset with empty effective cpus. 2466 */ 2467 if (cpumask_empty(cs->effective_cpus)) 2468 goto out_unlock; 2469 2470 cgroup_taskset_for_each(task, css, tset) { 2471 ret = task_can_attach(task, cs->effective_cpus); 2472 if (ret) 2473 goto out_unlock; 2474 ret = security_task_setscheduler(task); 2475 if (ret) 2476 goto out_unlock; 2477 } 2478 2479 /* 2480 * Mark attach is in progress. This makes validate_change() fail 2481 * changes which zero cpus/mems_allowed. 2482 */ 2483 cs->attach_in_progress++; 2484 ret = 0; 2485 out_unlock: 2486 percpu_up_write(&cpuset_rwsem); 2487 return ret; 2488 } 2489 2490 static void cpuset_cancel_attach(struct cgroup_taskset *tset) 2491 { 2492 struct cgroup_subsys_state *css; 2493 2494 cgroup_taskset_first(tset, &css); 2495 2496 percpu_down_write(&cpuset_rwsem); 2497 css_cs(css)->attach_in_progress--; 2498 percpu_up_write(&cpuset_rwsem); 2499 } 2500 2501 /* 2502 * Protected by cpuset_rwsem. cpus_attach is used only by cpuset_attach() 2503 * but we can't allocate it dynamically there. Define it global and 2504 * allocate from cpuset_init(). 2505 */ 2506 static cpumask_var_t cpus_attach; 2507 2508 static void cpuset_attach(struct cgroup_taskset *tset) 2509 { 2510 /* static buf protected by cpuset_rwsem */ 2511 static nodemask_t cpuset_attach_nodemask_to; 2512 struct task_struct *task; 2513 struct task_struct *leader; 2514 struct cgroup_subsys_state *css; 2515 struct cpuset *cs; 2516 struct cpuset *oldcs = cpuset_attach_old_cs; 2517 bool cpus_updated, mems_updated; 2518 2519 cgroup_taskset_first(tset, &css); 2520 cs = css_cs(css); 2521 2522 lockdep_assert_cpus_held(); /* see cgroup_attach_lock() */ 2523 percpu_down_write(&cpuset_rwsem); 2524 cpus_updated = !cpumask_equal(cs->effective_cpus, 2525 oldcs->effective_cpus); 2526 mems_updated = !nodes_equal(cs->effective_mems, oldcs->effective_mems); 2527 2528 /* 2529 * In the default hierarchy, enabling cpuset in the child cgroups 2530 * will trigger a number of cpuset_attach() calls with no change 2531 * in effective cpus and mems. In that case, we can optimize out 2532 * by skipping the task iteration and update. 2533 */ 2534 if (cgroup_subsys_on_dfl(cpuset_cgrp_subsys) && 2535 !cpus_updated && !mems_updated) { 2536 cpuset_attach_nodemask_to = cs->effective_mems; 2537 goto out; 2538 } 2539 2540 guarantee_online_mems(cs, &cpuset_attach_nodemask_to); 2541 2542 cgroup_taskset_for_each(task, css, tset) { 2543 if (cs != &top_cpuset) 2544 guarantee_online_cpus(task, cpus_attach); 2545 else 2546 cpumask_copy(cpus_attach, task_cpu_possible_mask(task)); 2547 /* 2548 * can_attach beforehand should guarantee that this doesn't 2549 * fail. TODO: have a better way to handle failure here 2550 */ 2551 WARN_ON_ONCE(set_cpus_allowed_ptr(task, cpus_attach)); 2552 2553 cpuset_change_task_nodemask(task, &cpuset_attach_nodemask_to); 2554 cpuset_update_task_spread_flags(cs, task); 2555 } 2556 2557 /* 2558 * Change mm for all threadgroup leaders. This is expensive and may 2559 * sleep and should be moved outside migration path proper. Skip it 2560 * if there is no change in effective_mems and CS_MEMORY_MIGRATE is 2561 * not set. 2562 */ 2563 cpuset_attach_nodemask_to = cs->effective_mems; 2564 if (!is_memory_migrate(cs) && !mems_updated) 2565 goto out; 2566 2567 cgroup_taskset_for_each_leader(leader, css, tset) { 2568 struct mm_struct *mm = get_task_mm(leader); 2569 2570 if (mm) { 2571 mpol_rebind_mm(mm, &cpuset_attach_nodemask_to); 2572 2573 /* 2574 * old_mems_allowed is the same with mems_allowed 2575 * here, except if this task is being moved 2576 * automatically due to hotplug. In that case 2577 * @mems_allowed has been updated and is empty, so 2578 * @old_mems_allowed is the right nodesets that we 2579 * migrate mm from. 2580 */ 2581 if (is_memory_migrate(cs)) 2582 cpuset_migrate_mm(mm, &oldcs->old_mems_allowed, 2583 &cpuset_attach_nodemask_to); 2584 else 2585 mmput(mm); 2586 } 2587 } 2588 2589 out: 2590 cs->old_mems_allowed = cpuset_attach_nodemask_to; 2591 2592 cs->attach_in_progress--; 2593 if (!cs->attach_in_progress) 2594 wake_up(&cpuset_attach_wq); 2595 2596 percpu_up_write(&cpuset_rwsem); 2597 } 2598 2599 /* The various types of files and directories in a cpuset file system */ 2600 2601 typedef enum { 2602 FILE_MEMORY_MIGRATE, 2603 FILE_CPULIST, 2604 FILE_MEMLIST, 2605 FILE_EFFECTIVE_CPULIST, 2606 FILE_EFFECTIVE_MEMLIST, 2607 FILE_SUBPARTS_CPULIST, 2608 FILE_CPU_EXCLUSIVE, 2609 FILE_MEM_EXCLUSIVE, 2610 FILE_MEM_HARDWALL, 2611 FILE_SCHED_LOAD_BALANCE, 2612 FILE_PARTITION_ROOT, 2613 FILE_SCHED_RELAX_DOMAIN_LEVEL, 2614 FILE_MEMORY_PRESSURE_ENABLED, 2615 FILE_MEMORY_PRESSURE, 2616 FILE_SPREAD_PAGE, 2617 FILE_SPREAD_SLAB, 2618 } cpuset_filetype_t; 2619 2620 static int cpuset_write_u64(struct cgroup_subsys_state *css, struct cftype *cft, 2621 u64 val) 2622 { 2623 struct cpuset *cs = css_cs(css); 2624 cpuset_filetype_t type = cft->private; 2625 int retval = 0; 2626 2627 cpus_read_lock(); 2628 percpu_down_write(&cpuset_rwsem); 2629 if (!is_cpuset_online(cs)) { 2630 retval = -ENODEV; 2631 goto out_unlock; 2632 } 2633 2634 switch (type) { 2635 case FILE_CPU_EXCLUSIVE: 2636 retval = update_flag(CS_CPU_EXCLUSIVE, cs, val); 2637 break; 2638 case FILE_MEM_EXCLUSIVE: 2639 retval = update_flag(CS_MEM_EXCLUSIVE, cs, val); 2640 break; 2641 case FILE_MEM_HARDWALL: 2642 retval = update_flag(CS_MEM_HARDWALL, cs, val); 2643 break; 2644 case FILE_SCHED_LOAD_BALANCE: 2645 retval = update_flag(CS_SCHED_LOAD_BALANCE, cs, val); 2646 break; 2647 case FILE_MEMORY_MIGRATE: 2648 retval = update_flag(CS_MEMORY_MIGRATE, cs, val); 2649 break; 2650 case FILE_MEMORY_PRESSURE_ENABLED: 2651 cpuset_memory_pressure_enabled = !!val; 2652 break; 2653 case FILE_SPREAD_PAGE: 2654 retval = update_flag(CS_SPREAD_PAGE, cs, val); 2655 break; 2656 case FILE_SPREAD_SLAB: 2657 retval = update_flag(CS_SPREAD_SLAB, cs, val); 2658 break; 2659 default: 2660 retval = -EINVAL; 2661 break; 2662 } 2663 out_unlock: 2664 percpu_up_write(&cpuset_rwsem); 2665 cpus_read_unlock(); 2666 return retval; 2667 } 2668 2669 static int cpuset_write_s64(struct cgroup_subsys_state *css, struct cftype *cft, 2670 s64 val) 2671 { 2672 struct cpuset *cs = css_cs(css); 2673 cpuset_filetype_t type = cft->private; 2674 int retval = -ENODEV; 2675 2676 cpus_read_lock(); 2677 percpu_down_write(&cpuset_rwsem); 2678 if (!is_cpuset_online(cs)) 2679 goto out_unlock; 2680 2681 switch (type) { 2682 case FILE_SCHED_RELAX_DOMAIN_LEVEL: 2683 retval = update_relax_domain_level(cs, val); 2684 break; 2685 default: 2686 retval = -EINVAL; 2687 break; 2688 } 2689 out_unlock: 2690 percpu_up_write(&cpuset_rwsem); 2691 cpus_read_unlock(); 2692 return retval; 2693 } 2694 2695 /* 2696 * Common handling for a write to a "cpus" or "mems" file. 2697 */ 2698 static ssize_t cpuset_write_resmask(struct kernfs_open_file *of, 2699 char *buf, size_t nbytes, loff_t off) 2700 { 2701 struct cpuset *cs = css_cs(of_css(of)); 2702 struct cpuset *trialcs; 2703 int retval = -ENODEV; 2704 2705 buf = strstrip(buf); 2706 2707 /* 2708 * CPU or memory hotunplug may leave @cs w/o any execution 2709 * resources, in which case the hotplug code asynchronously updates 2710 * configuration and transfers all tasks to the nearest ancestor 2711 * which can execute. 2712 * 2713 * As writes to "cpus" or "mems" may restore @cs's execution 2714 * resources, wait for the previously scheduled operations before 2715 * proceeding, so that we don't end up keep removing tasks added 2716 * after execution capability is restored. 2717 * 2718 * cpuset_hotplug_work calls back into cgroup core via 2719 * cgroup_transfer_tasks() and waiting for it from a cgroupfs 2720 * operation like this one can lead to a deadlock through kernfs 2721 * active_ref protection. Let's break the protection. Losing the 2722 * protection is okay as we check whether @cs is online after 2723 * grabbing cpuset_rwsem anyway. This only happens on the legacy 2724 * hierarchies. 2725 */ 2726 css_get(&cs->css); 2727 kernfs_break_active_protection(of->kn); 2728 flush_work(&cpuset_hotplug_work); 2729 2730 cpus_read_lock(); 2731 percpu_down_write(&cpuset_rwsem); 2732 if (!is_cpuset_online(cs)) 2733 goto out_unlock; 2734 2735 trialcs = alloc_trial_cpuset(cs); 2736 if (!trialcs) { 2737 retval = -ENOMEM; 2738 goto out_unlock; 2739 } 2740 2741 switch (of_cft(of)->private) { 2742 case FILE_CPULIST: 2743 retval = update_cpumask(cs, trialcs, buf); 2744 break; 2745 case FILE_MEMLIST: 2746 retval = update_nodemask(cs, trialcs, buf); 2747 break; 2748 default: 2749 retval = -EINVAL; 2750 break; 2751 } 2752 2753 free_cpuset(trialcs); 2754 out_unlock: 2755 percpu_up_write(&cpuset_rwsem); 2756 cpus_read_unlock(); 2757 kernfs_unbreak_active_protection(of->kn); 2758 css_put(&cs->css); 2759 flush_workqueue(cpuset_migrate_mm_wq); 2760 return retval ?: nbytes; 2761 } 2762 2763 /* 2764 * These ascii lists should be read in a single call, by using a user 2765 * buffer large enough to hold the entire map. If read in smaller 2766 * chunks, there is no guarantee of atomicity. Since the display format 2767 * used, list of ranges of sequential numbers, is variable length, 2768 * and since these maps can change value dynamically, one could read 2769 * gibberish by doing partial reads while a list was changing. 2770 */ 2771 static int cpuset_common_seq_show(struct seq_file *sf, void *v) 2772 { 2773 struct cpuset *cs = css_cs(seq_css(sf)); 2774 cpuset_filetype_t type = seq_cft(sf)->private; 2775 int ret = 0; 2776 2777 spin_lock_irq(&callback_lock); 2778 2779 switch (type) { 2780 case FILE_CPULIST: 2781 seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->cpus_allowed)); 2782 break; 2783 case FILE_MEMLIST: 2784 seq_printf(sf, "%*pbl\n", nodemask_pr_args(&cs->mems_allowed)); 2785 break; 2786 case FILE_EFFECTIVE_CPULIST: 2787 seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->effective_cpus)); 2788 break; 2789 case FILE_EFFECTIVE_MEMLIST: 2790 seq_printf(sf, "%*pbl\n", nodemask_pr_args(&cs->effective_mems)); 2791 break; 2792 case FILE_SUBPARTS_CPULIST: 2793 seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->subparts_cpus)); 2794 break; 2795 default: 2796 ret = -EINVAL; 2797 } 2798 2799 spin_unlock_irq(&callback_lock); 2800 return ret; 2801 } 2802 2803 static u64 cpuset_read_u64(struct cgroup_subsys_state *css, struct cftype *cft) 2804 { 2805 struct cpuset *cs = css_cs(css); 2806 cpuset_filetype_t type = cft->private; 2807 switch (type) { 2808 case FILE_CPU_EXCLUSIVE: 2809 return is_cpu_exclusive(cs); 2810 case FILE_MEM_EXCLUSIVE: 2811 return is_mem_exclusive(cs); 2812 case FILE_MEM_HARDWALL: 2813 return is_mem_hardwall(cs); 2814 case FILE_SCHED_LOAD_BALANCE: 2815 return is_sched_load_balance(cs); 2816 case FILE_MEMORY_MIGRATE: 2817 return is_memory_migrate(cs); 2818 case FILE_MEMORY_PRESSURE_ENABLED: 2819 return cpuset_memory_pressure_enabled; 2820 case FILE_MEMORY_PRESSURE: 2821 return fmeter_getrate(&cs->fmeter); 2822 case FILE_SPREAD_PAGE: 2823 return is_spread_page(cs); 2824 case FILE_SPREAD_SLAB: 2825 return is_spread_slab(cs); 2826 default: 2827 BUG(); 2828 } 2829 2830 /* Unreachable but makes gcc happy */ 2831 return 0; 2832 } 2833 2834 static s64 cpuset_read_s64(struct cgroup_subsys_state *css, struct cftype *cft) 2835 { 2836 struct cpuset *cs = css_cs(css); 2837 cpuset_filetype_t type = cft->private; 2838 switch (type) { 2839 case FILE_SCHED_RELAX_DOMAIN_LEVEL: 2840 return cs->relax_domain_level; 2841 default: 2842 BUG(); 2843 } 2844 2845 /* Unreachable but makes gcc happy */ 2846 return 0; 2847 } 2848 2849 static int sched_partition_show(struct seq_file *seq, void *v) 2850 { 2851 struct cpuset *cs = css_cs(seq_css(seq)); 2852 const char *err, *type = NULL; 2853 2854 switch (cs->partition_root_state) { 2855 case PRS_ROOT: 2856 seq_puts(seq, "root\n"); 2857 break; 2858 case PRS_ISOLATED: 2859 seq_puts(seq, "isolated\n"); 2860 break; 2861 case PRS_MEMBER: 2862 seq_puts(seq, "member\n"); 2863 break; 2864 case PRS_INVALID_ROOT: 2865 type = "root"; 2866 fallthrough; 2867 case PRS_INVALID_ISOLATED: 2868 if (!type) 2869 type = "isolated"; 2870 err = perr_strings[READ_ONCE(cs->prs_err)]; 2871 if (err) 2872 seq_printf(seq, "%s invalid (%s)\n", type, err); 2873 else 2874 seq_printf(seq, "%s invalid\n", type); 2875 break; 2876 } 2877 return 0; 2878 } 2879 2880 static ssize_t sched_partition_write(struct kernfs_open_file *of, char *buf, 2881 size_t nbytes, loff_t off) 2882 { 2883 struct cpuset *cs = css_cs(of_css(of)); 2884 int val; 2885 int retval = -ENODEV; 2886 2887 buf = strstrip(buf); 2888 2889 /* 2890 * Convert "root" to ENABLED, and convert "member" to DISABLED. 2891 */ 2892 if (!strcmp(buf, "root")) 2893 val = PRS_ROOT; 2894 else if (!strcmp(buf, "member")) 2895 val = PRS_MEMBER; 2896 else if (!strcmp(buf, "isolated")) 2897 val = PRS_ISOLATED; 2898 else 2899 return -EINVAL; 2900 2901 css_get(&cs->css); 2902 cpus_read_lock(); 2903 percpu_down_write(&cpuset_rwsem); 2904 if (!is_cpuset_online(cs)) 2905 goto out_unlock; 2906 2907 retval = update_prstate(cs, val); 2908 out_unlock: 2909 percpu_up_write(&cpuset_rwsem); 2910 cpus_read_unlock(); 2911 css_put(&cs->css); 2912 return retval ?: nbytes; 2913 } 2914 2915 /* 2916 * for the common functions, 'private' gives the type of file 2917 */ 2918 2919 static struct cftype legacy_files[] = { 2920 { 2921 .name = "cpus", 2922 .seq_show = cpuset_common_seq_show, 2923 .write = cpuset_write_resmask, 2924 .max_write_len = (100U + 6 * NR_CPUS), 2925 .private = FILE_CPULIST, 2926 }, 2927 2928 { 2929 .name = "mems", 2930 .seq_show = cpuset_common_seq_show, 2931 .write = cpuset_write_resmask, 2932 .max_write_len = (100U + 6 * MAX_NUMNODES), 2933 .private = FILE_MEMLIST, 2934 }, 2935 2936 { 2937 .name = "effective_cpus", 2938 .seq_show = cpuset_common_seq_show, 2939 .private = FILE_EFFECTIVE_CPULIST, 2940 }, 2941 2942 { 2943 .name = "effective_mems", 2944 .seq_show = cpuset_common_seq_show, 2945 .private = FILE_EFFECTIVE_MEMLIST, 2946 }, 2947 2948 { 2949 .name = "cpu_exclusive", 2950 .read_u64 = cpuset_read_u64, 2951 .write_u64 = cpuset_write_u64, 2952 .private = FILE_CPU_EXCLUSIVE, 2953 }, 2954 2955 { 2956 .name = "mem_exclusive", 2957 .read_u64 = cpuset_read_u64, 2958 .write_u64 = cpuset_write_u64, 2959 .private = FILE_MEM_EXCLUSIVE, 2960 }, 2961 2962 { 2963 .name = "mem_hardwall", 2964 .read_u64 = cpuset_read_u64, 2965 .write_u64 = cpuset_write_u64, 2966 .private = FILE_MEM_HARDWALL, 2967 }, 2968 2969 { 2970 .name = "sched_load_balance", 2971 .read_u64 = cpuset_read_u64, 2972 .write_u64 = cpuset_write_u64, 2973 .private = FILE_SCHED_LOAD_BALANCE, 2974 }, 2975 2976 { 2977 .name = "sched_relax_domain_level", 2978 .read_s64 = cpuset_read_s64, 2979 .write_s64 = cpuset_write_s64, 2980 .private = FILE_SCHED_RELAX_DOMAIN_LEVEL, 2981 }, 2982 2983 { 2984 .name = "memory_migrate", 2985 .read_u64 = cpuset_read_u64, 2986 .write_u64 = cpuset_write_u64, 2987 .private = FILE_MEMORY_MIGRATE, 2988 }, 2989 2990 { 2991 .name = "memory_pressure", 2992 .read_u64 = cpuset_read_u64, 2993 .private = FILE_MEMORY_PRESSURE, 2994 }, 2995 2996 { 2997 .name = "memory_spread_page", 2998 .read_u64 = cpuset_read_u64, 2999 .write_u64 = cpuset_write_u64, 3000 .private = FILE_SPREAD_PAGE, 3001 }, 3002 3003 { 3004 .name = "memory_spread_slab", 3005 .read_u64 = cpuset_read_u64, 3006 .write_u64 = cpuset_write_u64, 3007 .private = FILE_SPREAD_SLAB, 3008 }, 3009 3010 { 3011 .name = "memory_pressure_enabled", 3012 .flags = CFTYPE_ONLY_ON_ROOT, 3013 .read_u64 = cpuset_read_u64, 3014 .write_u64 = cpuset_write_u64, 3015 .private = FILE_MEMORY_PRESSURE_ENABLED, 3016 }, 3017 3018 { } /* terminate */ 3019 }; 3020 3021 /* 3022 * This is currently a minimal set for the default hierarchy. It can be 3023 * expanded later on by migrating more features and control files from v1. 3024 */ 3025 static struct cftype dfl_files[] = { 3026 { 3027 .name = "cpus", 3028 .seq_show = cpuset_common_seq_show, 3029 .write = cpuset_write_resmask, 3030 .max_write_len = (100U + 6 * NR_CPUS), 3031 .private = FILE_CPULIST, 3032 .flags = CFTYPE_NOT_ON_ROOT, 3033 }, 3034 3035 { 3036 .name = "mems", 3037 .seq_show = cpuset_common_seq_show, 3038 .write = cpuset_write_resmask, 3039 .max_write_len = (100U + 6 * MAX_NUMNODES), 3040 .private = FILE_MEMLIST, 3041 .flags = CFTYPE_NOT_ON_ROOT, 3042 }, 3043 3044 { 3045 .name = "cpus.effective", 3046 .seq_show = cpuset_common_seq_show, 3047 .private = FILE_EFFECTIVE_CPULIST, 3048 }, 3049 3050 { 3051 .name = "mems.effective", 3052 .seq_show = cpuset_common_seq_show, 3053 .private = FILE_EFFECTIVE_MEMLIST, 3054 }, 3055 3056 { 3057 .name = "cpus.partition", 3058 .seq_show = sched_partition_show, 3059 .write = sched_partition_write, 3060 .private = FILE_PARTITION_ROOT, 3061 .flags = CFTYPE_NOT_ON_ROOT, 3062 .file_offset = offsetof(struct cpuset, partition_file), 3063 }, 3064 3065 { 3066 .name = "cpus.subpartitions", 3067 .seq_show = cpuset_common_seq_show, 3068 .private = FILE_SUBPARTS_CPULIST, 3069 .flags = CFTYPE_DEBUG, 3070 }, 3071 3072 { } /* terminate */ 3073 }; 3074 3075 3076 /** 3077 * cpuset_css_alloc - Allocate a cpuset css 3078 * @parent_css: Parent css of the control group that the new cpuset will be 3079 * part of 3080 * Return: cpuset css on success, -ENOMEM on failure. 3081 * 3082 * Allocate and initialize a new cpuset css, for non-NULL @parent_css, return 3083 * top cpuset css otherwise. 3084 */ 3085 static struct cgroup_subsys_state * 3086 cpuset_css_alloc(struct cgroup_subsys_state *parent_css) 3087 { 3088 struct cpuset *cs; 3089 3090 if (!parent_css) 3091 return &top_cpuset.css; 3092 3093 cs = kzalloc(sizeof(*cs), GFP_KERNEL); 3094 if (!cs) 3095 return ERR_PTR(-ENOMEM); 3096 3097 if (alloc_cpumasks(cs, NULL)) { 3098 kfree(cs); 3099 return ERR_PTR(-ENOMEM); 3100 } 3101 3102 __set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags); 3103 nodes_clear(cs->mems_allowed); 3104 nodes_clear(cs->effective_mems); 3105 fmeter_init(&cs->fmeter); 3106 cs->relax_domain_level = -1; 3107 3108 /* Set CS_MEMORY_MIGRATE for default hierarchy */ 3109 if (cgroup_subsys_on_dfl(cpuset_cgrp_subsys)) 3110 __set_bit(CS_MEMORY_MIGRATE, &cs->flags); 3111 3112 return &cs->css; 3113 } 3114 3115 static int cpuset_css_online(struct cgroup_subsys_state *css) 3116 { 3117 struct cpuset *cs = css_cs(css); 3118 struct cpuset *parent = parent_cs(cs); 3119 struct cpuset *tmp_cs; 3120 struct cgroup_subsys_state *pos_css; 3121 3122 if (!parent) 3123 return 0; 3124 3125 cpus_read_lock(); 3126 percpu_down_write(&cpuset_rwsem); 3127 3128 set_bit(CS_ONLINE, &cs->flags); 3129 if (is_spread_page(parent)) 3130 set_bit(CS_SPREAD_PAGE, &cs->flags); 3131 if (is_spread_slab(parent)) 3132 set_bit(CS_SPREAD_SLAB, &cs->flags); 3133 3134 cpuset_inc(); 3135 3136 spin_lock_irq(&callback_lock); 3137 if (is_in_v2_mode()) { 3138 cpumask_copy(cs->effective_cpus, parent->effective_cpus); 3139 cs->effective_mems = parent->effective_mems; 3140 cs->use_parent_ecpus = true; 3141 parent->child_ecpus_count++; 3142 } 3143 spin_unlock_irq(&callback_lock); 3144 3145 if (!test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags)) 3146 goto out_unlock; 3147 3148 /* 3149 * Clone @parent's configuration if CGRP_CPUSET_CLONE_CHILDREN is 3150 * set. This flag handling is implemented in cgroup core for 3151 * historical reasons - the flag may be specified during mount. 3152 * 3153 * Currently, if any sibling cpusets have exclusive cpus or mem, we 3154 * refuse to clone the configuration - thereby refusing the task to 3155 * be entered, and as a result refusing the sys_unshare() or 3156 * clone() which initiated it. If this becomes a problem for some 3157 * users who wish to allow that scenario, then this could be 3158 * changed to grant parent->cpus_allowed-sibling_cpus_exclusive 3159 * (and likewise for mems) to the new cgroup. 3160 */ 3161 rcu_read_lock(); 3162 cpuset_for_each_child(tmp_cs, pos_css, parent) { 3163 if (is_mem_exclusive(tmp_cs) || is_cpu_exclusive(tmp_cs)) { 3164 rcu_read_unlock(); 3165 goto out_unlock; 3166 } 3167 } 3168 rcu_read_unlock(); 3169 3170 spin_lock_irq(&callback_lock); 3171 cs->mems_allowed = parent->mems_allowed; 3172 cs->effective_mems = parent->mems_allowed; 3173 cpumask_copy(cs->cpus_allowed, parent->cpus_allowed); 3174 cpumask_copy(cs->effective_cpus, parent->cpus_allowed); 3175 spin_unlock_irq(&callback_lock); 3176 out_unlock: 3177 percpu_up_write(&cpuset_rwsem); 3178 cpus_read_unlock(); 3179 return 0; 3180 } 3181 3182 /* 3183 * If the cpuset being removed has its flag 'sched_load_balance' 3184 * enabled, then simulate turning sched_load_balance off, which 3185 * will call rebuild_sched_domains_locked(). That is not needed 3186 * in the default hierarchy where only changes in partition 3187 * will cause repartitioning. 3188 * 3189 * If the cpuset has the 'sched.partition' flag enabled, simulate 3190 * turning 'sched.partition" off. 3191 */ 3192 3193 static void cpuset_css_offline(struct cgroup_subsys_state *css) 3194 { 3195 struct cpuset *cs = css_cs(css); 3196 3197 cpus_read_lock(); 3198 percpu_down_write(&cpuset_rwsem); 3199 3200 if (is_partition_valid(cs)) 3201 update_prstate(cs, 0); 3202 3203 if (!cgroup_subsys_on_dfl(cpuset_cgrp_subsys) && 3204 is_sched_load_balance(cs)) 3205 update_flag(CS_SCHED_LOAD_BALANCE, cs, 0); 3206 3207 if (cs->use_parent_ecpus) { 3208 struct cpuset *parent = parent_cs(cs); 3209 3210 cs->use_parent_ecpus = false; 3211 parent->child_ecpus_count--; 3212 } 3213 3214 cpuset_dec(); 3215 clear_bit(CS_ONLINE, &cs->flags); 3216 3217 percpu_up_write(&cpuset_rwsem); 3218 cpus_read_unlock(); 3219 } 3220 3221 static void cpuset_css_free(struct cgroup_subsys_state *css) 3222 { 3223 struct cpuset *cs = css_cs(css); 3224 3225 free_cpuset(cs); 3226 } 3227 3228 static void cpuset_bind(struct cgroup_subsys_state *root_css) 3229 { 3230 percpu_down_write(&cpuset_rwsem); 3231 spin_lock_irq(&callback_lock); 3232 3233 if (is_in_v2_mode()) { 3234 cpumask_copy(top_cpuset.cpus_allowed, cpu_possible_mask); 3235 top_cpuset.mems_allowed = node_possible_map; 3236 } else { 3237 cpumask_copy(top_cpuset.cpus_allowed, 3238 top_cpuset.effective_cpus); 3239 top_cpuset.mems_allowed = top_cpuset.effective_mems; 3240 } 3241 3242 spin_unlock_irq(&callback_lock); 3243 percpu_up_write(&cpuset_rwsem); 3244 } 3245 3246 /* 3247 * Make sure the new task conform to the current state of its parent, 3248 * which could have been changed by cpuset just after it inherits the 3249 * state from the parent and before it sits on the cgroup's task list. 3250 */ 3251 static void cpuset_fork(struct task_struct *task) 3252 { 3253 if (task_css_is_root(task, cpuset_cgrp_id)) 3254 return; 3255 3256 set_cpus_allowed_ptr(task, current->cpus_ptr); 3257 task->mems_allowed = current->mems_allowed; 3258 } 3259 3260 struct cgroup_subsys cpuset_cgrp_subsys = { 3261 .css_alloc = cpuset_css_alloc, 3262 .css_online = cpuset_css_online, 3263 .css_offline = cpuset_css_offline, 3264 .css_free = cpuset_css_free, 3265 .can_attach = cpuset_can_attach, 3266 .cancel_attach = cpuset_cancel_attach, 3267 .attach = cpuset_attach, 3268 .post_attach = cpuset_post_attach, 3269 .bind = cpuset_bind, 3270 .fork = cpuset_fork, 3271 .legacy_cftypes = legacy_files, 3272 .dfl_cftypes = dfl_files, 3273 .early_init = true, 3274 .threaded = true, 3275 }; 3276 3277 /** 3278 * cpuset_init - initialize cpusets at system boot 3279 * 3280 * Description: Initialize top_cpuset 3281 **/ 3282 3283 int __init cpuset_init(void) 3284 { 3285 BUG_ON(percpu_init_rwsem(&cpuset_rwsem)); 3286 3287 BUG_ON(!alloc_cpumask_var(&top_cpuset.cpus_allowed, GFP_KERNEL)); 3288 BUG_ON(!alloc_cpumask_var(&top_cpuset.effective_cpus, GFP_KERNEL)); 3289 BUG_ON(!zalloc_cpumask_var(&top_cpuset.subparts_cpus, GFP_KERNEL)); 3290 3291 cpumask_setall(top_cpuset.cpus_allowed); 3292 nodes_setall(top_cpuset.mems_allowed); 3293 cpumask_setall(top_cpuset.effective_cpus); 3294 nodes_setall(top_cpuset.effective_mems); 3295 3296 fmeter_init(&top_cpuset.fmeter); 3297 set_bit(CS_SCHED_LOAD_BALANCE, &top_cpuset.flags); 3298 top_cpuset.relax_domain_level = -1; 3299 3300 BUG_ON(!alloc_cpumask_var(&cpus_attach, GFP_KERNEL)); 3301 3302 return 0; 3303 } 3304 3305 /* 3306 * If CPU and/or memory hotplug handlers, below, unplug any CPUs 3307 * or memory nodes, we need to walk over the cpuset hierarchy, 3308 * removing that CPU or node from all cpusets. If this removes the 3309 * last CPU or node from a cpuset, then move the tasks in the empty 3310 * cpuset to its next-highest non-empty parent. 3311 */ 3312 static void remove_tasks_in_empty_cpuset(struct cpuset *cs) 3313 { 3314 struct cpuset *parent; 3315 3316 /* 3317 * Find its next-highest non-empty parent, (top cpuset 3318 * has online cpus, so can't be empty). 3319 */ 3320 parent = parent_cs(cs); 3321 while (cpumask_empty(parent->cpus_allowed) || 3322 nodes_empty(parent->mems_allowed)) 3323 parent = parent_cs(parent); 3324 3325 if (cgroup_transfer_tasks(parent->css.cgroup, cs->css.cgroup)) { 3326 pr_err("cpuset: failed to transfer tasks out of empty cpuset "); 3327 pr_cont_cgroup_name(cs->css.cgroup); 3328 pr_cont("\n"); 3329 } 3330 } 3331 3332 static void 3333 hotplug_update_tasks_legacy(struct cpuset *cs, 3334 struct cpumask *new_cpus, nodemask_t *new_mems, 3335 bool cpus_updated, bool mems_updated) 3336 { 3337 bool is_empty; 3338 3339 spin_lock_irq(&callback_lock); 3340 cpumask_copy(cs->cpus_allowed, new_cpus); 3341 cpumask_copy(cs->effective_cpus, new_cpus); 3342 cs->mems_allowed = *new_mems; 3343 cs->effective_mems = *new_mems; 3344 spin_unlock_irq(&callback_lock); 3345 3346 /* 3347 * Don't call update_tasks_cpumask() if the cpuset becomes empty, 3348 * as the tasks will be migrated to an ancestor. 3349 */ 3350 if (cpus_updated && !cpumask_empty(cs->cpus_allowed)) 3351 update_tasks_cpumask(cs); 3352 if (mems_updated && !nodes_empty(cs->mems_allowed)) 3353 update_tasks_nodemask(cs); 3354 3355 is_empty = cpumask_empty(cs->cpus_allowed) || 3356 nodes_empty(cs->mems_allowed); 3357 3358 percpu_up_write(&cpuset_rwsem); 3359 3360 /* 3361 * Move tasks to the nearest ancestor with execution resources, 3362 * This is full cgroup operation which will also call back into 3363 * cpuset. Should be done outside any lock. 3364 */ 3365 if (is_empty) 3366 remove_tasks_in_empty_cpuset(cs); 3367 3368 percpu_down_write(&cpuset_rwsem); 3369 } 3370 3371 static void 3372 hotplug_update_tasks(struct cpuset *cs, 3373 struct cpumask *new_cpus, nodemask_t *new_mems, 3374 bool cpus_updated, bool mems_updated) 3375 { 3376 /* A partition root is allowed to have empty effective cpus */ 3377 if (cpumask_empty(new_cpus) && !is_partition_valid(cs)) 3378 cpumask_copy(new_cpus, parent_cs(cs)->effective_cpus); 3379 if (nodes_empty(*new_mems)) 3380 *new_mems = parent_cs(cs)->effective_mems; 3381 3382 spin_lock_irq(&callback_lock); 3383 cpumask_copy(cs->effective_cpus, new_cpus); 3384 cs->effective_mems = *new_mems; 3385 spin_unlock_irq(&callback_lock); 3386 3387 if (cpus_updated) 3388 update_tasks_cpumask(cs); 3389 if (mems_updated) 3390 update_tasks_nodemask(cs); 3391 } 3392 3393 static bool force_rebuild; 3394 3395 void cpuset_force_rebuild(void) 3396 { 3397 force_rebuild = true; 3398 } 3399 3400 /** 3401 * cpuset_hotplug_update_tasks - update tasks in a cpuset for hotunplug 3402 * @cs: cpuset in interest 3403 * @tmp: the tmpmasks structure pointer 3404 * 3405 * Compare @cs's cpu and mem masks against top_cpuset and if some have gone 3406 * offline, update @cs accordingly. If @cs ends up with no CPU or memory, 3407 * all its tasks are moved to the nearest ancestor with both resources. 3408 */ 3409 static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp) 3410 { 3411 static cpumask_t new_cpus; 3412 static nodemask_t new_mems; 3413 bool cpus_updated; 3414 bool mems_updated; 3415 struct cpuset *parent; 3416 retry: 3417 wait_event(cpuset_attach_wq, cs->attach_in_progress == 0); 3418 3419 percpu_down_write(&cpuset_rwsem); 3420 3421 /* 3422 * We have raced with task attaching. We wait until attaching 3423 * is finished, so we won't attach a task to an empty cpuset. 3424 */ 3425 if (cs->attach_in_progress) { 3426 percpu_up_write(&cpuset_rwsem); 3427 goto retry; 3428 } 3429 3430 parent = parent_cs(cs); 3431 compute_effective_cpumask(&new_cpus, cs, parent); 3432 nodes_and(new_mems, cs->mems_allowed, parent->effective_mems); 3433 3434 if (cs->nr_subparts_cpus) 3435 /* 3436 * Make sure that CPUs allocated to child partitions 3437 * do not show up in effective_cpus. 3438 */ 3439 cpumask_andnot(&new_cpus, &new_cpus, cs->subparts_cpus); 3440 3441 if (!tmp || !cs->partition_root_state) 3442 goto update_tasks; 3443 3444 /* 3445 * In the unlikely event that a partition root has empty 3446 * effective_cpus with tasks, we will have to invalidate child 3447 * partitions, if present, by setting nr_subparts_cpus to 0 to 3448 * reclaim their cpus. 3449 */ 3450 if (cs->nr_subparts_cpus && is_partition_valid(cs) && 3451 cpumask_empty(&new_cpus) && partition_is_populated(cs, NULL)) { 3452 spin_lock_irq(&callback_lock); 3453 cs->nr_subparts_cpus = 0; 3454 cpumask_clear(cs->subparts_cpus); 3455 spin_unlock_irq(&callback_lock); 3456 compute_effective_cpumask(&new_cpus, cs, parent); 3457 } 3458 3459 /* 3460 * Force the partition to become invalid if either one of 3461 * the following conditions hold: 3462 * 1) empty effective cpus but not valid empty partition. 3463 * 2) parent is invalid or doesn't grant any cpus to child 3464 * partitions. 3465 */ 3466 if (is_partition_valid(cs) && (!parent->nr_subparts_cpus || 3467 (cpumask_empty(&new_cpus) && partition_is_populated(cs, NULL)))) { 3468 int old_prs, parent_prs; 3469 3470 update_parent_subparts_cpumask(cs, partcmd_disable, NULL, tmp); 3471 if (cs->nr_subparts_cpus) { 3472 spin_lock_irq(&callback_lock); 3473 cs->nr_subparts_cpus = 0; 3474 cpumask_clear(cs->subparts_cpus); 3475 spin_unlock_irq(&callback_lock); 3476 compute_effective_cpumask(&new_cpus, cs, parent); 3477 } 3478 3479 old_prs = cs->partition_root_state; 3480 parent_prs = parent->partition_root_state; 3481 if (is_partition_valid(cs)) { 3482 spin_lock_irq(&callback_lock); 3483 make_partition_invalid(cs); 3484 spin_unlock_irq(&callback_lock); 3485 if (is_prs_invalid(parent_prs)) 3486 WRITE_ONCE(cs->prs_err, PERR_INVPARENT); 3487 else if (!parent_prs) 3488 WRITE_ONCE(cs->prs_err, PERR_NOTPART); 3489 else 3490 WRITE_ONCE(cs->prs_err, PERR_HOTPLUG); 3491 notify_partition_change(cs, old_prs); 3492 } 3493 cpuset_force_rebuild(); 3494 } 3495 3496 /* 3497 * On the other hand, an invalid partition root may be transitioned 3498 * back to a regular one. 3499 */ 3500 else if (is_partition_valid(parent) && is_partition_invalid(cs)) { 3501 update_parent_subparts_cpumask(cs, partcmd_update, NULL, tmp); 3502 if (is_partition_valid(cs)) 3503 cpuset_force_rebuild(); 3504 } 3505 3506 update_tasks: 3507 cpus_updated = !cpumask_equal(&new_cpus, cs->effective_cpus); 3508 mems_updated = !nodes_equal(new_mems, cs->effective_mems); 3509 3510 if (mems_updated) 3511 check_insane_mems_config(&new_mems); 3512 3513 if (is_in_v2_mode()) 3514 hotplug_update_tasks(cs, &new_cpus, &new_mems, 3515 cpus_updated, mems_updated); 3516 else 3517 hotplug_update_tasks_legacy(cs, &new_cpus, &new_mems, 3518 cpus_updated, mems_updated); 3519 3520 percpu_up_write(&cpuset_rwsem); 3521 } 3522 3523 /** 3524 * cpuset_hotplug_workfn - handle CPU/memory hotunplug for a cpuset 3525 * 3526 * This function is called after either CPU or memory configuration has 3527 * changed and updates cpuset accordingly. The top_cpuset is always 3528 * synchronized to cpu_active_mask and N_MEMORY, which is necessary in 3529 * order to make cpusets transparent (of no affect) on systems that are 3530 * actively using CPU hotplug but making no active use of cpusets. 3531 * 3532 * Non-root cpusets are only affected by offlining. If any CPUs or memory 3533 * nodes have been taken down, cpuset_hotplug_update_tasks() is invoked on 3534 * all descendants. 3535 * 3536 * Note that CPU offlining during suspend is ignored. We don't modify 3537 * cpusets across suspend/resume cycles at all. 3538 */ 3539 static void cpuset_hotplug_workfn(struct work_struct *work) 3540 { 3541 static cpumask_t new_cpus; 3542 static nodemask_t new_mems; 3543 bool cpus_updated, mems_updated; 3544 bool on_dfl = is_in_v2_mode(); 3545 struct tmpmasks tmp, *ptmp = NULL; 3546 3547 if (on_dfl && !alloc_cpumasks(NULL, &tmp)) 3548 ptmp = &tmp; 3549 3550 percpu_down_write(&cpuset_rwsem); 3551 3552 /* fetch the available cpus/mems and find out which changed how */ 3553 cpumask_copy(&new_cpus, cpu_active_mask); 3554 new_mems = node_states[N_MEMORY]; 3555 3556 /* 3557 * If subparts_cpus is populated, it is likely that the check below 3558 * will produce a false positive on cpus_updated when the cpu list 3559 * isn't changed. It is extra work, but it is better to be safe. 3560 */ 3561 cpus_updated = !cpumask_equal(top_cpuset.effective_cpus, &new_cpus); 3562 mems_updated = !nodes_equal(top_cpuset.effective_mems, new_mems); 3563 3564 /* 3565 * In the rare case that hotplug removes all the cpus in subparts_cpus, 3566 * we assumed that cpus are updated. 3567 */ 3568 if (!cpus_updated && top_cpuset.nr_subparts_cpus) 3569 cpus_updated = true; 3570 3571 /* synchronize cpus_allowed to cpu_active_mask */ 3572 if (cpus_updated) { 3573 spin_lock_irq(&callback_lock); 3574 if (!on_dfl) 3575 cpumask_copy(top_cpuset.cpus_allowed, &new_cpus); 3576 /* 3577 * Make sure that CPUs allocated to child partitions 3578 * do not show up in effective_cpus. If no CPU is left, 3579 * we clear the subparts_cpus & let the child partitions 3580 * fight for the CPUs again. 3581 */ 3582 if (top_cpuset.nr_subparts_cpus) { 3583 if (cpumask_subset(&new_cpus, 3584 top_cpuset.subparts_cpus)) { 3585 top_cpuset.nr_subparts_cpus = 0; 3586 cpumask_clear(top_cpuset.subparts_cpus); 3587 } else { 3588 cpumask_andnot(&new_cpus, &new_cpus, 3589 top_cpuset.subparts_cpus); 3590 } 3591 } 3592 cpumask_copy(top_cpuset.effective_cpus, &new_cpus); 3593 spin_unlock_irq(&callback_lock); 3594 /* we don't mess with cpumasks of tasks in top_cpuset */ 3595 } 3596 3597 /* synchronize mems_allowed to N_MEMORY */ 3598 if (mems_updated) { 3599 spin_lock_irq(&callback_lock); 3600 if (!on_dfl) 3601 top_cpuset.mems_allowed = new_mems; 3602 top_cpuset.effective_mems = new_mems; 3603 spin_unlock_irq(&callback_lock); 3604 update_tasks_nodemask(&top_cpuset); 3605 } 3606 3607 percpu_up_write(&cpuset_rwsem); 3608 3609 /* if cpus or mems changed, we need to propagate to descendants */ 3610 if (cpus_updated || mems_updated) { 3611 struct cpuset *cs; 3612 struct cgroup_subsys_state *pos_css; 3613 3614 rcu_read_lock(); 3615 cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) { 3616 if (cs == &top_cpuset || !css_tryget_online(&cs->css)) 3617 continue; 3618 rcu_read_unlock(); 3619 3620 cpuset_hotplug_update_tasks(cs, ptmp); 3621 3622 rcu_read_lock(); 3623 css_put(&cs->css); 3624 } 3625 rcu_read_unlock(); 3626 } 3627 3628 /* rebuild sched domains if cpus_allowed has changed */ 3629 if (cpus_updated || force_rebuild) { 3630 force_rebuild = false; 3631 rebuild_sched_domains(); 3632 } 3633 3634 free_cpumasks(NULL, ptmp); 3635 } 3636 3637 void cpuset_update_active_cpus(void) 3638 { 3639 /* 3640 * We're inside cpu hotplug critical region which usually nests 3641 * inside cgroup synchronization. Bounce actual hotplug processing 3642 * to a work item to avoid reverse locking order. 3643 */ 3644 schedule_work(&cpuset_hotplug_work); 3645 } 3646 3647 void cpuset_wait_for_hotplug(void) 3648 { 3649 flush_work(&cpuset_hotplug_work); 3650 } 3651 3652 /* 3653 * Keep top_cpuset.mems_allowed tracking node_states[N_MEMORY]. 3654 * Call this routine anytime after node_states[N_MEMORY] changes. 3655 * See cpuset_update_active_cpus() for CPU hotplug handling. 3656 */ 3657 static int cpuset_track_online_nodes(struct notifier_block *self, 3658 unsigned long action, void *arg) 3659 { 3660 schedule_work(&cpuset_hotplug_work); 3661 return NOTIFY_OK; 3662 } 3663 3664 /** 3665 * cpuset_init_smp - initialize cpus_allowed 3666 * 3667 * Description: Finish top cpuset after cpu, node maps are initialized 3668 */ 3669 void __init cpuset_init_smp(void) 3670 { 3671 /* 3672 * cpus_allowd/mems_allowed set to v2 values in the initial 3673 * cpuset_bind() call will be reset to v1 values in another 3674 * cpuset_bind() call when v1 cpuset is mounted. 3675 */ 3676 top_cpuset.old_mems_allowed = top_cpuset.mems_allowed; 3677 3678 cpumask_copy(top_cpuset.effective_cpus, cpu_active_mask); 3679 top_cpuset.effective_mems = node_states[N_MEMORY]; 3680 3681 hotplug_memory_notifier(cpuset_track_online_nodes, CPUSET_CALLBACK_PRI); 3682 3683 cpuset_migrate_mm_wq = alloc_ordered_workqueue("cpuset_migrate_mm", 0); 3684 BUG_ON(!cpuset_migrate_mm_wq); 3685 } 3686 3687 /** 3688 * cpuset_cpus_allowed - return cpus_allowed mask from a tasks cpuset. 3689 * @tsk: pointer to task_struct from which to obtain cpuset->cpus_allowed. 3690 * @pmask: pointer to struct cpumask variable to receive cpus_allowed set. 3691 * 3692 * Description: Returns the cpumask_var_t cpus_allowed of the cpuset 3693 * attached to the specified @tsk. Guaranteed to return some non-empty 3694 * subset of cpu_online_mask, even if this means going outside the 3695 * tasks cpuset. 3696 **/ 3697 3698 void cpuset_cpus_allowed(struct task_struct *tsk, struct cpumask *pmask) 3699 { 3700 unsigned long flags; 3701 3702 spin_lock_irqsave(&callback_lock, flags); 3703 guarantee_online_cpus(tsk, pmask); 3704 spin_unlock_irqrestore(&callback_lock, flags); 3705 } 3706 3707 /** 3708 * cpuset_cpus_allowed_fallback - final fallback before complete catastrophe. 3709 * @tsk: pointer to task_struct with which the scheduler is struggling 3710 * 3711 * Description: In the case that the scheduler cannot find an allowed cpu in 3712 * tsk->cpus_allowed, we fall back to task_cs(tsk)->cpus_allowed. In legacy 3713 * mode however, this value is the same as task_cs(tsk)->effective_cpus, 3714 * which will not contain a sane cpumask during cases such as cpu hotplugging. 3715 * This is the absolute last resort for the scheduler and it is only used if 3716 * _every_ other avenue has been traveled. 3717 * 3718 * Returns true if the affinity of @tsk was changed, false otherwise. 3719 **/ 3720 3721 bool cpuset_cpus_allowed_fallback(struct task_struct *tsk) 3722 { 3723 const struct cpumask *possible_mask = task_cpu_possible_mask(tsk); 3724 const struct cpumask *cs_mask; 3725 bool changed = false; 3726 3727 rcu_read_lock(); 3728 cs_mask = task_cs(tsk)->cpus_allowed; 3729 if (is_in_v2_mode() && cpumask_subset(cs_mask, possible_mask)) { 3730 do_set_cpus_allowed(tsk, cs_mask); 3731 changed = true; 3732 } 3733 rcu_read_unlock(); 3734 3735 /* 3736 * We own tsk->cpus_allowed, nobody can change it under us. 3737 * 3738 * But we used cs && cs->cpus_allowed lockless and thus can 3739 * race with cgroup_attach_task() or update_cpumask() and get 3740 * the wrong tsk->cpus_allowed. However, both cases imply the 3741 * subsequent cpuset_change_cpumask()->set_cpus_allowed_ptr() 3742 * which takes task_rq_lock(). 3743 * 3744 * If we are called after it dropped the lock we must see all 3745 * changes in tsk_cs()->cpus_allowed. Otherwise we can temporary 3746 * set any mask even if it is not right from task_cs() pov, 3747 * the pending set_cpus_allowed_ptr() will fix things. 3748 * 3749 * select_fallback_rq() will fix things ups and set cpu_possible_mask 3750 * if required. 3751 */ 3752 return changed; 3753 } 3754 3755 void __init cpuset_init_current_mems_allowed(void) 3756 { 3757 nodes_setall(current->mems_allowed); 3758 } 3759 3760 /** 3761 * cpuset_mems_allowed - return mems_allowed mask from a tasks cpuset. 3762 * @tsk: pointer to task_struct from which to obtain cpuset->mems_allowed. 3763 * 3764 * Description: Returns the nodemask_t mems_allowed of the cpuset 3765 * attached to the specified @tsk. Guaranteed to return some non-empty 3766 * subset of node_states[N_MEMORY], even if this means going outside the 3767 * tasks cpuset. 3768 **/ 3769 3770 nodemask_t cpuset_mems_allowed(struct task_struct *tsk) 3771 { 3772 nodemask_t mask; 3773 unsigned long flags; 3774 3775 spin_lock_irqsave(&callback_lock, flags); 3776 rcu_read_lock(); 3777 guarantee_online_mems(task_cs(tsk), &mask); 3778 rcu_read_unlock(); 3779 spin_unlock_irqrestore(&callback_lock, flags); 3780 3781 return mask; 3782 } 3783 3784 /** 3785 * cpuset_nodemask_valid_mems_allowed - check nodemask vs. current mems_allowed 3786 * @nodemask: the nodemask to be checked 3787 * 3788 * Are any of the nodes in the nodemask allowed in current->mems_allowed? 3789 */ 3790 int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask) 3791 { 3792 return nodes_intersects(*nodemask, current->mems_allowed); 3793 } 3794 3795 /* 3796 * nearest_hardwall_ancestor() - Returns the nearest mem_exclusive or 3797 * mem_hardwall ancestor to the specified cpuset. Call holding 3798 * callback_lock. If no ancestor is mem_exclusive or mem_hardwall 3799 * (an unusual configuration), then returns the root cpuset. 3800 */ 3801 static struct cpuset *nearest_hardwall_ancestor(struct cpuset *cs) 3802 { 3803 while (!(is_mem_exclusive(cs) || is_mem_hardwall(cs)) && parent_cs(cs)) 3804 cs = parent_cs(cs); 3805 return cs; 3806 } 3807 3808 /* 3809 * __cpuset_node_allowed - Can we allocate on a memory node? 3810 * @node: is this an allowed node? 3811 * @gfp_mask: memory allocation flags 3812 * 3813 * If we're in interrupt, yes, we can always allocate. If @node is set in 3814 * current's mems_allowed, yes. If it's not a __GFP_HARDWALL request and this 3815 * node is set in the nearest hardwalled cpuset ancestor to current's cpuset, 3816 * yes. If current has access to memory reserves as an oom victim, yes. 3817 * Otherwise, no. 3818 * 3819 * GFP_USER allocations are marked with the __GFP_HARDWALL bit, 3820 * and do not allow allocations outside the current tasks cpuset 3821 * unless the task has been OOM killed. 3822 * GFP_KERNEL allocations are not so marked, so can escape to the 3823 * nearest enclosing hardwalled ancestor cpuset. 3824 * 3825 * Scanning up parent cpusets requires callback_lock. The 3826 * __alloc_pages() routine only calls here with __GFP_HARDWALL bit 3827 * _not_ set if it's a GFP_KERNEL allocation, and all nodes in the 3828 * current tasks mems_allowed came up empty on the first pass over 3829 * the zonelist. So only GFP_KERNEL allocations, if all nodes in the 3830 * cpuset are short of memory, might require taking the callback_lock. 3831 * 3832 * The first call here from mm/page_alloc:get_page_from_freelist() 3833 * has __GFP_HARDWALL set in gfp_mask, enforcing hardwall cpusets, 3834 * so no allocation on a node outside the cpuset is allowed (unless 3835 * in interrupt, of course). 3836 * 3837 * The second pass through get_page_from_freelist() doesn't even call 3838 * here for GFP_ATOMIC calls. For those calls, the __alloc_pages() 3839 * variable 'wait' is not set, and the bit ALLOC_CPUSET is not set 3840 * in alloc_flags. That logic and the checks below have the combined 3841 * affect that: 3842 * in_interrupt - any node ok (current task context irrelevant) 3843 * GFP_ATOMIC - any node ok 3844 * tsk_is_oom_victim - any node ok 3845 * GFP_KERNEL - any node in enclosing hardwalled cpuset ok 3846 * GFP_USER - only nodes in current tasks mems allowed ok. 3847 */ 3848 bool __cpuset_node_allowed(int node, gfp_t gfp_mask) 3849 { 3850 struct cpuset *cs; /* current cpuset ancestors */ 3851 bool allowed; /* is allocation in zone z allowed? */ 3852 unsigned long flags; 3853 3854 if (in_interrupt()) 3855 return true; 3856 if (node_isset(node, current->mems_allowed)) 3857 return true; 3858 /* 3859 * Allow tasks that have access to memory reserves because they have 3860 * been OOM killed to get memory anywhere. 3861 */ 3862 if (unlikely(tsk_is_oom_victim(current))) 3863 return true; 3864 if (gfp_mask & __GFP_HARDWALL) /* If hardwall request, stop here */ 3865 return false; 3866 3867 if (current->flags & PF_EXITING) /* Let dying task have memory */ 3868 return true; 3869 3870 /* Not hardwall and node outside mems_allowed: scan up cpusets */ 3871 spin_lock_irqsave(&callback_lock, flags); 3872 3873 rcu_read_lock(); 3874 cs = nearest_hardwall_ancestor(task_cs(current)); 3875 allowed = node_isset(node, cs->mems_allowed); 3876 rcu_read_unlock(); 3877 3878 spin_unlock_irqrestore(&callback_lock, flags); 3879 return allowed; 3880 } 3881 3882 /** 3883 * cpuset_mem_spread_node() - On which node to begin search for a file page 3884 * cpuset_slab_spread_node() - On which node to begin search for a slab page 3885 * 3886 * If a task is marked PF_SPREAD_PAGE or PF_SPREAD_SLAB (as for 3887 * tasks in a cpuset with is_spread_page or is_spread_slab set), 3888 * and if the memory allocation used cpuset_mem_spread_node() 3889 * to determine on which node to start looking, as it will for 3890 * certain page cache or slab cache pages such as used for file 3891 * system buffers and inode caches, then instead of starting on the 3892 * local node to look for a free page, rather spread the starting 3893 * node around the tasks mems_allowed nodes. 3894 * 3895 * We don't have to worry about the returned node being offline 3896 * because "it can't happen", and even if it did, it would be ok. 3897 * 3898 * The routines calling guarantee_online_mems() are careful to 3899 * only set nodes in task->mems_allowed that are online. So it 3900 * should not be possible for the following code to return an 3901 * offline node. But if it did, that would be ok, as this routine 3902 * is not returning the node where the allocation must be, only 3903 * the node where the search should start. The zonelist passed to 3904 * __alloc_pages() will include all nodes. If the slab allocator 3905 * is passed an offline node, it will fall back to the local node. 3906 * See kmem_cache_alloc_node(). 3907 */ 3908 3909 static int cpuset_spread_node(int *rotor) 3910 { 3911 return *rotor = next_node_in(*rotor, current->mems_allowed); 3912 } 3913 3914 int cpuset_mem_spread_node(void) 3915 { 3916 if (current->cpuset_mem_spread_rotor == NUMA_NO_NODE) 3917 current->cpuset_mem_spread_rotor = 3918 node_random(¤t->mems_allowed); 3919 3920 return cpuset_spread_node(¤t->cpuset_mem_spread_rotor); 3921 } 3922 3923 int cpuset_slab_spread_node(void) 3924 { 3925 if (current->cpuset_slab_spread_rotor == NUMA_NO_NODE) 3926 current->cpuset_slab_spread_rotor = 3927 node_random(¤t->mems_allowed); 3928 3929 return cpuset_spread_node(¤t->cpuset_slab_spread_rotor); 3930 } 3931 3932 EXPORT_SYMBOL_GPL(cpuset_mem_spread_node); 3933 3934 /** 3935 * cpuset_mems_allowed_intersects - Does @tsk1's mems_allowed intersect @tsk2's? 3936 * @tsk1: pointer to task_struct of some task. 3937 * @tsk2: pointer to task_struct of some other task. 3938 * 3939 * Description: Return true if @tsk1's mems_allowed intersects the 3940 * mems_allowed of @tsk2. Used by the OOM killer to determine if 3941 * one of the task's memory usage might impact the memory available 3942 * to the other. 3943 **/ 3944 3945 int cpuset_mems_allowed_intersects(const struct task_struct *tsk1, 3946 const struct task_struct *tsk2) 3947 { 3948 return nodes_intersects(tsk1->mems_allowed, tsk2->mems_allowed); 3949 } 3950 3951 /** 3952 * cpuset_print_current_mems_allowed - prints current's cpuset and mems_allowed 3953 * 3954 * Description: Prints current's name, cpuset name, and cached copy of its 3955 * mems_allowed to the kernel log. 3956 */ 3957 void cpuset_print_current_mems_allowed(void) 3958 { 3959 struct cgroup *cgrp; 3960 3961 rcu_read_lock(); 3962 3963 cgrp = task_cs(current)->css.cgroup; 3964 pr_cont(",cpuset="); 3965 pr_cont_cgroup_name(cgrp); 3966 pr_cont(",mems_allowed=%*pbl", 3967 nodemask_pr_args(¤t->mems_allowed)); 3968 3969 rcu_read_unlock(); 3970 } 3971 3972 /* 3973 * Collection of memory_pressure is suppressed unless 3974 * this flag is enabled by writing "1" to the special 3975 * cpuset file 'memory_pressure_enabled' in the root cpuset. 3976 */ 3977 3978 int cpuset_memory_pressure_enabled __read_mostly; 3979 3980 /* 3981 * __cpuset_memory_pressure_bump - keep stats of per-cpuset reclaims. 3982 * 3983 * Keep a running average of the rate of synchronous (direct) 3984 * page reclaim efforts initiated by tasks in each cpuset. 3985 * 3986 * This represents the rate at which some task in the cpuset 3987 * ran low on memory on all nodes it was allowed to use, and 3988 * had to enter the kernels page reclaim code in an effort to 3989 * create more free memory by tossing clean pages or swapping 3990 * or writing dirty pages. 3991 * 3992 * Display to user space in the per-cpuset read-only file 3993 * "memory_pressure". Value displayed is an integer 3994 * representing the recent rate of entry into the synchronous 3995 * (direct) page reclaim by any task attached to the cpuset. 3996 */ 3997 3998 void __cpuset_memory_pressure_bump(void) 3999 { 4000 rcu_read_lock(); 4001 fmeter_markevent(&task_cs(current)->fmeter); 4002 rcu_read_unlock(); 4003 } 4004 4005 #ifdef CONFIG_PROC_PID_CPUSET 4006 /* 4007 * proc_cpuset_show() 4008 * - Print tasks cpuset path into seq_file. 4009 * - Used for /proc/<pid>/cpuset. 4010 * - No need to task_lock(tsk) on this tsk->cpuset reference, as it 4011 * doesn't really matter if tsk->cpuset changes after we read it, 4012 * and we take cpuset_rwsem, keeping cpuset_attach() from changing it 4013 * anyway. 4014 */ 4015 int proc_cpuset_show(struct seq_file *m, struct pid_namespace *ns, 4016 struct pid *pid, struct task_struct *tsk) 4017 { 4018 char *buf; 4019 struct cgroup_subsys_state *css; 4020 int retval; 4021 4022 retval = -ENOMEM; 4023 buf = kmalloc(PATH_MAX, GFP_KERNEL); 4024 if (!buf) 4025 goto out; 4026 4027 css = task_get_css(tsk, cpuset_cgrp_id); 4028 retval = cgroup_path_ns(css->cgroup, buf, PATH_MAX, 4029 current->nsproxy->cgroup_ns); 4030 css_put(css); 4031 if (retval >= PATH_MAX) 4032 retval = -ENAMETOOLONG; 4033 if (retval < 0) 4034 goto out_free; 4035 seq_puts(m, buf); 4036 seq_putc(m, '\n'); 4037 retval = 0; 4038 out_free: 4039 kfree(buf); 4040 out: 4041 return retval; 4042 } 4043 #endif /* CONFIG_PROC_PID_CPUSET */ 4044 4045 /* Display task mems_allowed in /proc/<pid>/status file. */ 4046 void cpuset_task_status_allowed(struct seq_file *m, struct task_struct *task) 4047 { 4048 seq_printf(m, "Mems_allowed:\t%*pb\n", 4049 nodemask_pr_args(&task->mems_allowed)); 4050 seq_printf(m, "Mems_allowed_list:\t%*pbl\n", 4051 nodemask_pr_args(&task->mems_allowed)); 4052 } 4053