1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Completely Fair Scheduling (CFS) Class (SCHED_NORMAL/SCHED_BATCH) 4 * 5 * Copyright (C) 2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> 6 * 7 * Interactivity improvements by Mike Galbraith 8 * (C) 2007 Mike Galbraith <efault@gmx.de> 9 * 10 * Various enhancements by Dmitry Adamushko. 11 * (C) 2007 Dmitry Adamushko <dmitry.adamushko@gmail.com> 12 * 13 * Group scheduling enhancements by Srivatsa Vaddagiri 14 * Copyright IBM Corporation, 2007 15 * Author: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com> 16 * 17 * Scaled math optimizations by Thomas Gleixner 18 * Copyright (C) 2007, Thomas Gleixner <tglx@linutronix.de> 19 * 20 * Adaptive scheduling granularity, math enhancements by Peter Zijlstra 21 * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra 22 */ 23 #include <linux/energy_model.h> 24 #include <linux/mmap_lock.h> 25 #include <linux/hugetlb_inline.h> 26 #include <linux/jiffies.h> 27 #include <linux/mm_api.h> 28 #include <linux/highmem.h> 29 #include <linux/spinlock_api.h> 30 #include <linux/cpumask_api.h> 31 #include <linux/lockdep_api.h> 32 #include <linux/softirq.h> 33 #include <linux/refcount_api.h> 34 #include <linux/topology.h> 35 #include <linux/sched/clock.h> 36 #include <linux/sched/cond_resched.h> 37 #include <linux/sched/cputime.h> 38 #include <linux/sched/isolation.h> 39 #include <linux/sched/nohz.h> 40 41 #include <linux/cpuidle.h> 42 #include <linux/interrupt.h> 43 #include <linux/memory-tiers.h> 44 #include <linux/mempolicy.h> 45 #include <linux/mutex_api.h> 46 #include <linux/profile.h> 47 #include <linux/psi.h> 48 #include <linux/ratelimit.h> 49 #include <linux/task_work.h> 50 51 #include <asm/switch_to.h> 52 53 #include <linux/sched/cond_resched.h> 54 55 #include "sched.h" 56 #include "stats.h" 57 #include "autogroup.h" 58 59 /* 60 * Targeted preemption latency for CPU-bound tasks: 61 * 62 * NOTE: this latency value is not the same as the concept of 63 * 'timeslice length' - timeslices in CFS are of variable length 64 * and have no persistent notion like in traditional, time-slice 65 * based scheduling concepts. 66 * 67 * (to see the precise effective timeslice length of your workload, 68 * run vmstat and monitor the context-switches (cs) field) 69 * 70 * (default: 6ms * (1 + ilog(ncpus)), units: nanoseconds) 71 */ 72 unsigned int sysctl_sched_latency = 6000000ULL; 73 static unsigned int normalized_sysctl_sched_latency = 6000000ULL; 74 75 /* 76 * The initial- and re-scaling of tunables is configurable 77 * 78 * Options are: 79 * 80 * SCHED_TUNABLESCALING_NONE - unscaled, always *1 81 * SCHED_TUNABLESCALING_LOG - scaled logarithmical, *1+ilog(ncpus) 82 * SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus 83 * 84 * (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus)) 85 */ 86 unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG; 87 88 /* 89 * Minimal preemption granularity for CPU-bound tasks: 90 * 91 * (default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds) 92 */ 93 unsigned int sysctl_sched_min_granularity = 750000ULL; 94 static unsigned int normalized_sysctl_sched_min_granularity = 750000ULL; 95 96 /* 97 * Minimal preemption granularity for CPU-bound SCHED_IDLE tasks. 98 * Applies only when SCHED_IDLE tasks compete with normal tasks. 99 * 100 * (default: 0.75 msec) 101 */ 102 unsigned int sysctl_sched_idle_min_granularity = 750000ULL; 103 104 /* 105 * This value is kept at sysctl_sched_latency/sysctl_sched_min_granularity 106 */ 107 static unsigned int sched_nr_latency = 8; 108 109 /* 110 * After fork, child runs first. If set to 0 (default) then 111 * parent will (try to) run first. 112 */ 113 unsigned int sysctl_sched_child_runs_first __read_mostly; 114 115 /* 116 * SCHED_OTHER wake-up granularity. 117 * 118 * This option delays the preemption effects of decoupled workloads 119 * and reduces their over-scheduling. Synchronous workloads will still 120 * have immediate wakeup/sleep latencies. 121 * 122 * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds) 123 */ 124 unsigned int sysctl_sched_wakeup_granularity = 1000000UL; 125 static unsigned int normalized_sysctl_sched_wakeup_granularity = 1000000UL; 126 127 const_debug unsigned int sysctl_sched_migration_cost = 500000UL; 128 129 int sched_thermal_decay_shift; 130 static int __init setup_sched_thermal_decay_shift(char *str) 131 { 132 int _shift = 0; 133 134 if (kstrtoint(str, 0, &_shift)) 135 pr_warn("Unable to set scheduler thermal pressure decay shift parameter\n"); 136 137 sched_thermal_decay_shift = clamp(_shift, 0, 10); 138 return 1; 139 } 140 __setup("sched_thermal_decay_shift=", setup_sched_thermal_decay_shift); 141 142 #ifdef CONFIG_SMP 143 /* 144 * For asym packing, by default the lower numbered CPU has higher priority. 145 */ 146 int __weak arch_asym_cpu_priority(int cpu) 147 { 148 return -cpu; 149 } 150 151 /* 152 * The margin used when comparing utilization with CPU capacity. 153 * 154 * (default: ~20%) 155 */ 156 #define fits_capacity(cap, max) ((cap) * 1280 < (max) * 1024) 157 158 /* 159 * The margin used when comparing CPU capacities. 160 * is 'cap1' noticeably greater than 'cap2' 161 * 162 * (default: ~5%) 163 */ 164 #define capacity_greater(cap1, cap2) ((cap1) * 1024 > (cap2) * 1078) 165 #endif 166 167 #ifdef CONFIG_CFS_BANDWIDTH 168 /* 169 * Amount of runtime to allocate from global (tg) to local (per-cfs_rq) pool 170 * each time a cfs_rq requests quota. 171 * 172 * Note: in the case that the slice exceeds the runtime remaining (either due 173 * to consumption or the quota being specified to be smaller than the slice) 174 * we will always only issue the remaining available time. 175 * 176 * (default: 5 msec, units: microseconds) 177 */ 178 static unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; 179 #endif 180 181 #ifdef CONFIG_NUMA_BALANCING 182 /* Restrict the NUMA promotion throughput (MB/s) for each target node. */ 183 static unsigned int sysctl_numa_balancing_promote_rate_limit = 65536; 184 #endif 185 186 #ifdef CONFIG_SYSCTL 187 static struct ctl_table sched_fair_sysctls[] = { 188 { 189 .procname = "sched_child_runs_first", 190 .data = &sysctl_sched_child_runs_first, 191 .maxlen = sizeof(unsigned int), 192 .mode = 0644, 193 .proc_handler = proc_dointvec, 194 }, 195 #ifdef CONFIG_CFS_BANDWIDTH 196 { 197 .procname = "sched_cfs_bandwidth_slice_us", 198 .data = &sysctl_sched_cfs_bandwidth_slice, 199 .maxlen = sizeof(unsigned int), 200 .mode = 0644, 201 .proc_handler = proc_dointvec_minmax, 202 .extra1 = SYSCTL_ONE, 203 }, 204 #endif 205 #ifdef CONFIG_NUMA_BALANCING 206 { 207 .procname = "numa_balancing_promote_rate_limit_MBps", 208 .data = &sysctl_numa_balancing_promote_rate_limit, 209 .maxlen = sizeof(unsigned int), 210 .mode = 0644, 211 .proc_handler = proc_dointvec_minmax, 212 .extra1 = SYSCTL_ZERO, 213 }, 214 #endif /* CONFIG_NUMA_BALANCING */ 215 {} 216 }; 217 218 static int __init sched_fair_sysctl_init(void) 219 { 220 register_sysctl_init("kernel", sched_fair_sysctls); 221 return 0; 222 } 223 late_initcall(sched_fair_sysctl_init); 224 #endif 225 226 static inline void update_load_add(struct load_weight *lw, unsigned long inc) 227 { 228 lw->weight += inc; 229 lw->inv_weight = 0; 230 } 231 232 static inline void update_load_sub(struct load_weight *lw, unsigned long dec) 233 { 234 lw->weight -= dec; 235 lw->inv_weight = 0; 236 } 237 238 static inline void update_load_set(struct load_weight *lw, unsigned long w) 239 { 240 lw->weight = w; 241 lw->inv_weight = 0; 242 } 243 244 /* 245 * Increase the granularity value when there are more CPUs, 246 * because with more CPUs the 'effective latency' as visible 247 * to users decreases. But the relationship is not linear, 248 * so pick a second-best guess by going with the log2 of the 249 * number of CPUs. 250 * 251 * This idea comes from the SD scheduler of Con Kolivas: 252 */ 253 static unsigned int get_update_sysctl_factor(void) 254 { 255 unsigned int cpus = min_t(unsigned int, num_online_cpus(), 8); 256 unsigned int factor; 257 258 switch (sysctl_sched_tunable_scaling) { 259 case SCHED_TUNABLESCALING_NONE: 260 factor = 1; 261 break; 262 case SCHED_TUNABLESCALING_LINEAR: 263 factor = cpus; 264 break; 265 case SCHED_TUNABLESCALING_LOG: 266 default: 267 factor = 1 + ilog2(cpus); 268 break; 269 } 270 271 return factor; 272 } 273 274 static void update_sysctl(void) 275 { 276 unsigned int factor = get_update_sysctl_factor(); 277 278 #define SET_SYSCTL(name) \ 279 (sysctl_##name = (factor) * normalized_sysctl_##name) 280 SET_SYSCTL(sched_min_granularity); 281 SET_SYSCTL(sched_latency); 282 SET_SYSCTL(sched_wakeup_granularity); 283 #undef SET_SYSCTL 284 } 285 286 void __init sched_init_granularity(void) 287 { 288 update_sysctl(); 289 } 290 291 #define WMULT_CONST (~0U) 292 #define WMULT_SHIFT 32 293 294 static void __update_inv_weight(struct load_weight *lw) 295 { 296 unsigned long w; 297 298 if (likely(lw->inv_weight)) 299 return; 300 301 w = scale_load_down(lw->weight); 302 303 if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST)) 304 lw->inv_weight = 1; 305 else if (unlikely(!w)) 306 lw->inv_weight = WMULT_CONST; 307 else 308 lw->inv_weight = WMULT_CONST / w; 309 } 310 311 /* 312 * delta_exec * weight / lw.weight 313 * OR 314 * (delta_exec * (weight * lw->inv_weight)) >> WMULT_SHIFT 315 * 316 * Either weight := NICE_0_LOAD and lw \e sched_prio_to_wmult[], in which case 317 * we're guaranteed shift stays positive because inv_weight is guaranteed to 318 * fit 32 bits, and NICE_0_LOAD gives another 10 bits; therefore shift >= 22. 319 * 320 * Or, weight =< lw.weight (because lw.weight is the runqueue weight), thus 321 * weight/lw.weight <= 1, and therefore our shift will also be positive. 322 */ 323 static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw) 324 { 325 u64 fact = scale_load_down(weight); 326 u32 fact_hi = (u32)(fact >> 32); 327 int shift = WMULT_SHIFT; 328 int fs; 329 330 __update_inv_weight(lw); 331 332 if (unlikely(fact_hi)) { 333 fs = fls(fact_hi); 334 shift -= fs; 335 fact >>= fs; 336 } 337 338 fact = mul_u32_u32(fact, lw->inv_weight); 339 340 fact_hi = (u32)(fact >> 32); 341 if (fact_hi) { 342 fs = fls(fact_hi); 343 shift -= fs; 344 fact >>= fs; 345 } 346 347 return mul_u64_u32_shr(delta_exec, fact, shift); 348 } 349 350 351 const struct sched_class fair_sched_class; 352 353 /************************************************************** 354 * CFS operations on generic schedulable entities: 355 */ 356 357 #ifdef CONFIG_FAIR_GROUP_SCHED 358 359 /* Walk up scheduling entities hierarchy */ 360 #define for_each_sched_entity(se) \ 361 for (; se; se = se->parent) 362 363 static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) 364 { 365 struct rq *rq = rq_of(cfs_rq); 366 int cpu = cpu_of(rq); 367 368 if (cfs_rq->on_list) 369 return rq->tmp_alone_branch == &rq->leaf_cfs_rq_list; 370 371 cfs_rq->on_list = 1; 372 373 /* 374 * Ensure we either appear before our parent (if already 375 * enqueued) or force our parent to appear after us when it is 376 * enqueued. The fact that we always enqueue bottom-up 377 * reduces this to two cases and a special case for the root 378 * cfs_rq. Furthermore, it also means that we will always reset 379 * tmp_alone_branch either when the branch is connected 380 * to a tree or when we reach the top of the tree 381 */ 382 if (cfs_rq->tg->parent && 383 cfs_rq->tg->parent->cfs_rq[cpu]->on_list) { 384 /* 385 * If parent is already on the list, we add the child 386 * just before. Thanks to circular linked property of 387 * the list, this means to put the child at the tail 388 * of the list that starts by parent. 389 */ 390 list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, 391 &(cfs_rq->tg->parent->cfs_rq[cpu]->leaf_cfs_rq_list)); 392 /* 393 * The branch is now connected to its tree so we can 394 * reset tmp_alone_branch to the beginning of the 395 * list. 396 */ 397 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 398 return true; 399 } 400 401 if (!cfs_rq->tg->parent) { 402 /* 403 * cfs rq without parent should be put 404 * at the tail of the list. 405 */ 406 list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, 407 &rq->leaf_cfs_rq_list); 408 /* 409 * We have reach the top of a tree so we can reset 410 * tmp_alone_branch to the beginning of the list. 411 */ 412 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 413 return true; 414 } 415 416 /* 417 * The parent has not already been added so we want to 418 * make sure that it will be put after us. 419 * tmp_alone_branch points to the begin of the branch 420 * where we will add parent. 421 */ 422 list_add_rcu(&cfs_rq->leaf_cfs_rq_list, rq->tmp_alone_branch); 423 /* 424 * update tmp_alone_branch to points to the new begin 425 * of the branch 426 */ 427 rq->tmp_alone_branch = &cfs_rq->leaf_cfs_rq_list; 428 return false; 429 } 430 431 static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) 432 { 433 if (cfs_rq->on_list) { 434 struct rq *rq = rq_of(cfs_rq); 435 436 /* 437 * With cfs_rq being unthrottled/throttled during an enqueue, 438 * it can happen the tmp_alone_branch points the a leaf that 439 * we finally want to del. In this case, tmp_alone_branch moves 440 * to the prev element but it will point to rq->leaf_cfs_rq_list 441 * at the end of the enqueue. 442 */ 443 if (rq->tmp_alone_branch == &cfs_rq->leaf_cfs_rq_list) 444 rq->tmp_alone_branch = cfs_rq->leaf_cfs_rq_list.prev; 445 446 list_del_rcu(&cfs_rq->leaf_cfs_rq_list); 447 cfs_rq->on_list = 0; 448 } 449 } 450 451 static inline void assert_list_leaf_cfs_rq(struct rq *rq) 452 { 453 SCHED_WARN_ON(rq->tmp_alone_branch != &rq->leaf_cfs_rq_list); 454 } 455 456 /* Iterate thr' all leaf cfs_rq's on a runqueue */ 457 #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \ 458 list_for_each_entry_safe(cfs_rq, pos, &rq->leaf_cfs_rq_list, \ 459 leaf_cfs_rq_list) 460 461 /* Do the two (enqueued) entities belong to the same group ? */ 462 static inline struct cfs_rq * 463 is_same_group(struct sched_entity *se, struct sched_entity *pse) 464 { 465 if (se->cfs_rq == pse->cfs_rq) 466 return se->cfs_rq; 467 468 return NULL; 469 } 470 471 static inline struct sched_entity *parent_entity(const struct sched_entity *se) 472 { 473 return se->parent; 474 } 475 476 static void 477 find_matching_se(struct sched_entity **se, struct sched_entity **pse) 478 { 479 int se_depth, pse_depth; 480 481 /* 482 * preemption test can be made between sibling entities who are in the 483 * same cfs_rq i.e who have a common parent. Walk up the hierarchy of 484 * both tasks until we find their ancestors who are siblings of common 485 * parent. 486 */ 487 488 /* First walk up until both entities are at same depth */ 489 se_depth = (*se)->depth; 490 pse_depth = (*pse)->depth; 491 492 while (se_depth > pse_depth) { 493 se_depth--; 494 *se = parent_entity(*se); 495 } 496 497 while (pse_depth > se_depth) { 498 pse_depth--; 499 *pse = parent_entity(*pse); 500 } 501 502 while (!is_same_group(*se, *pse)) { 503 *se = parent_entity(*se); 504 *pse = parent_entity(*pse); 505 } 506 } 507 508 static int tg_is_idle(struct task_group *tg) 509 { 510 return tg->idle > 0; 511 } 512 513 static int cfs_rq_is_idle(struct cfs_rq *cfs_rq) 514 { 515 return cfs_rq->idle > 0; 516 } 517 518 static int se_is_idle(struct sched_entity *se) 519 { 520 if (entity_is_task(se)) 521 return task_has_idle_policy(task_of(se)); 522 return cfs_rq_is_idle(group_cfs_rq(se)); 523 } 524 525 #else /* !CONFIG_FAIR_GROUP_SCHED */ 526 527 #define for_each_sched_entity(se) \ 528 for (; se; se = NULL) 529 530 static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) 531 { 532 return true; 533 } 534 535 static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) 536 { 537 } 538 539 static inline void assert_list_leaf_cfs_rq(struct rq *rq) 540 { 541 } 542 543 #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \ 544 for (cfs_rq = &rq->cfs, pos = NULL; cfs_rq; cfs_rq = pos) 545 546 static inline struct sched_entity *parent_entity(struct sched_entity *se) 547 { 548 return NULL; 549 } 550 551 static inline void 552 find_matching_se(struct sched_entity **se, struct sched_entity **pse) 553 { 554 } 555 556 static inline int tg_is_idle(struct task_group *tg) 557 { 558 return 0; 559 } 560 561 static int cfs_rq_is_idle(struct cfs_rq *cfs_rq) 562 { 563 return 0; 564 } 565 566 static int se_is_idle(struct sched_entity *se) 567 { 568 return 0; 569 } 570 571 #endif /* CONFIG_FAIR_GROUP_SCHED */ 572 573 static __always_inline 574 void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec); 575 576 /************************************************************** 577 * Scheduling class tree data structure manipulation methods: 578 */ 579 580 static inline u64 max_vruntime(u64 max_vruntime, u64 vruntime) 581 { 582 s64 delta = (s64)(vruntime - max_vruntime); 583 if (delta > 0) 584 max_vruntime = vruntime; 585 586 return max_vruntime; 587 } 588 589 static inline u64 min_vruntime(u64 min_vruntime, u64 vruntime) 590 { 591 s64 delta = (s64)(vruntime - min_vruntime); 592 if (delta < 0) 593 min_vruntime = vruntime; 594 595 return min_vruntime; 596 } 597 598 static inline bool entity_before(const struct sched_entity *a, 599 const struct sched_entity *b) 600 { 601 return (s64)(a->vruntime - b->vruntime) < 0; 602 } 603 604 #define __node_2_se(node) \ 605 rb_entry((node), struct sched_entity, run_node) 606 607 static void update_min_vruntime(struct cfs_rq *cfs_rq) 608 { 609 struct sched_entity *curr = cfs_rq->curr; 610 struct rb_node *leftmost = rb_first_cached(&cfs_rq->tasks_timeline); 611 612 u64 vruntime = cfs_rq->min_vruntime; 613 614 if (curr) { 615 if (curr->on_rq) 616 vruntime = curr->vruntime; 617 else 618 curr = NULL; 619 } 620 621 if (leftmost) { /* non-empty tree */ 622 struct sched_entity *se = __node_2_se(leftmost); 623 624 if (!curr) 625 vruntime = se->vruntime; 626 else 627 vruntime = min_vruntime(vruntime, se->vruntime); 628 } 629 630 /* ensure we never gain time by being placed backwards. */ 631 u64_u32_store(cfs_rq->min_vruntime, 632 max_vruntime(cfs_rq->min_vruntime, vruntime)); 633 } 634 635 static inline bool __entity_less(struct rb_node *a, const struct rb_node *b) 636 { 637 return entity_before(__node_2_se(a), __node_2_se(b)); 638 } 639 640 /* 641 * Enqueue an entity into the rb-tree: 642 */ 643 static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 644 { 645 rb_add_cached(&se->run_node, &cfs_rq->tasks_timeline, __entity_less); 646 } 647 648 static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 649 { 650 rb_erase_cached(&se->run_node, &cfs_rq->tasks_timeline); 651 } 652 653 struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq) 654 { 655 struct rb_node *left = rb_first_cached(&cfs_rq->tasks_timeline); 656 657 if (!left) 658 return NULL; 659 660 return __node_2_se(left); 661 } 662 663 static struct sched_entity *__pick_next_entity(struct sched_entity *se) 664 { 665 struct rb_node *next = rb_next(&se->run_node); 666 667 if (!next) 668 return NULL; 669 670 return __node_2_se(next); 671 } 672 673 #ifdef CONFIG_SCHED_DEBUG 674 struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq) 675 { 676 struct rb_node *last = rb_last(&cfs_rq->tasks_timeline.rb_root); 677 678 if (!last) 679 return NULL; 680 681 return __node_2_se(last); 682 } 683 684 /************************************************************** 685 * Scheduling class statistics methods: 686 */ 687 688 int sched_update_scaling(void) 689 { 690 unsigned int factor = get_update_sysctl_factor(); 691 692 sched_nr_latency = DIV_ROUND_UP(sysctl_sched_latency, 693 sysctl_sched_min_granularity); 694 695 #define WRT_SYSCTL(name) \ 696 (normalized_sysctl_##name = sysctl_##name / (factor)) 697 WRT_SYSCTL(sched_min_granularity); 698 WRT_SYSCTL(sched_latency); 699 WRT_SYSCTL(sched_wakeup_granularity); 700 #undef WRT_SYSCTL 701 702 return 0; 703 } 704 #endif 705 706 /* 707 * delta /= w 708 */ 709 static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se) 710 { 711 if (unlikely(se->load.weight != NICE_0_LOAD)) 712 delta = __calc_delta(delta, NICE_0_LOAD, &se->load); 713 714 return delta; 715 } 716 717 /* 718 * The idea is to set a period in which each task runs once. 719 * 720 * When there are too many tasks (sched_nr_latency) we have to stretch 721 * this period because otherwise the slices get too small. 722 * 723 * p = (nr <= nl) ? l : l*nr/nl 724 */ 725 static u64 __sched_period(unsigned long nr_running) 726 { 727 if (unlikely(nr_running > sched_nr_latency)) 728 return nr_running * sysctl_sched_min_granularity; 729 else 730 return sysctl_sched_latency; 731 } 732 733 static bool sched_idle_cfs_rq(struct cfs_rq *cfs_rq); 734 735 /* 736 * We calculate the wall-time slice from the period by taking a part 737 * proportional to the weight. 738 * 739 * s = p*P[w/rw] 740 */ 741 static u64 sched_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) 742 { 743 unsigned int nr_running = cfs_rq->nr_running; 744 struct sched_entity *init_se = se; 745 unsigned int min_gran; 746 u64 slice; 747 748 if (sched_feat(ALT_PERIOD)) 749 nr_running = rq_of(cfs_rq)->cfs.h_nr_running; 750 751 slice = __sched_period(nr_running + !se->on_rq); 752 753 for_each_sched_entity(se) { 754 struct load_weight *load; 755 struct load_weight lw; 756 struct cfs_rq *qcfs_rq; 757 758 qcfs_rq = cfs_rq_of(se); 759 load = &qcfs_rq->load; 760 761 if (unlikely(!se->on_rq)) { 762 lw = qcfs_rq->load; 763 764 update_load_add(&lw, se->load.weight); 765 load = &lw; 766 } 767 slice = __calc_delta(slice, se->load.weight, load); 768 } 769 770 if (sched_feat(BASE_SLICE)) { 771 if (se_is_idle(init_se) && !sched_idle_cfs_rq(cfs_rq)) 772 min_gran = sysctl_sched_idle_min_granularity; 773 else 774 min_gran = sysctl_sched_min_granularity; 775 776 slice = max_t(u64, slice, min_gran); 777 } 778 779 return slice; 780 } 781 782 /* 783 * We calculate the vruntime slice of a to-be-inserted task. 784 * 785 * vs = s/w 786 */ 787 static u64 sched_vslice(struct cfs_rq *cfs_rq, struct sched_entity *se) 788 { 789 return calc_delta_fair(sched_slice(cfs_rq, se), se); 790 } 791 792 #include "pelt.h" 793 #ifdef CONFIG_SMP 794 795 static int select_idle_sibling(struct task_struct *p, int prev_cpu, int cpu); 796 static unsigned long task_h_load(struct task_struct *p); 797 static unsigned long capacity_of(int cpu); 798 799 /* Give new sched_entity start runnable values to heavy its load in infant time */ 800 void init_entity_runnable_average(struct sched_entity *se) 801 { 802 struct sched_avg *sa = &se->avg; 803 804 memset(sa, 0, sizeof(*sa)); 805 806 /* 807 * Tasks are initialized with full load to be seen as heavy tasks until 808 * they get a chance to stabilize to their real load level. 809 * Group entities are initialized with zero load to reflect the fact that 810 * nothing has been attached to the task group yet. 811 */ 812 if (entity_is_task(se)) 813 sa->load_avg = scale_load_down(se->load.weight); 814 815 /* when this task enqueue'ed, it will contribute to its cfs_rq's load_avg */ 816 } 817 818 /* 819 * With new tasks being created, their initial util_avgs are extrapolated 820 * based on the cfs_rq's current util_avg: 821 * 822 * util_avg = cfs_rq->util_avg / (cfs_rq->load_avg + 1) * se.load.weight 823 * 824 * However, in many cases, the above util_avg does not give a desired 825 * value. Moreover, the sum of the util_avgs may be divergent, such 826 * as when the series is a harmonic series. 827 * 828 * To solve this problem, we also cap the util_avg of successive tasks to 829 * only 1/2 of the left utilization budget: 830 * 831 * util_avg_cap = (cpu_scale - cfs_rq->avg.util_avg) / 2^n 832 * 833 * where n denotes the nth task and cpu_scale the CPU capacity. 834 * 835 * For example, for a CPU with 1024 of capacity, a simplest series from 836 * the beginning would be like: 837 * 838 * task util_avg: 512, 256, 128, 64, 32, 16, 8, ... 839 * cfs_rq util_avg: 512, 768, 896, 960, 992, 1008, 1016, ... 840 * 841 * Finally, that extrapolated util_avg is clamped to the cap (util_avg_cap) 842 * if util_avg > util_avg_cap. 843 */ 844 void post_init_entity_util_avg(struct task_struct *p) 845 { 846 struct sched_entity *se = &p->se; 847 struct cfs_rq *cfs_rq = cfs_rq_of(se); 848 struct sched_avg *sa = &se->avg; 849 long cpu_scale = arch_scale_cpu_capacity(cpu_of(rq_of(cfs_rq))); 850 long cap = (long)(cpu_scale - cfs_rq->avg.util_avg) / 2; 851 852 if (p->sched_class != &fair_sched_class) { 853 /* 854 * For !fair tasks do: 855 * 856 update_cfs_rq_load_avg(now, cfs_rq); 857 attach_entity_load_avg(cfs_rq, se); 858 switched_from_fair(rq, p); 859 * 860 * such that the next switched_to_fair() has the 861 * expected state. 862 */ 863 se->avg.last_update_time = cfs_rq_clock_pelt(cfs_rq); 864 return; 865 } 866 867 if (cap > 0) { 868 if (cfs_rq->avg.util_avg != 0) { 869 sa->util_avg = cfs_rq->avg.util_avg * se->load.weight; 870 sa->util_avg /= (cfs_rq->avg.load_avg + 1); 871 872 if (sa->util_avg > cap) 873 sa->util_avg = cap; 874 } else { 875 sa->util_avg = cap; 876 } 877 } 878 879 sa->runnable_avg = sa->util_avg; 880 } 881 882 #else /* !CONFIG_SMP */ 883 void init_entity_runnable_average(struct sched_entity *se) 884 { 885 } 886 void post_init_entity_util_avg(struct task_struct *p) 887 { 888 } 889 static void update_tg_load_avg(struct cfs_rq *cfs_rq) 890 { 891 } 892 #endif /* CONFIG_SMP */ 893 894 /* 895 * Update the current task's runtime statistics. 896 */ 897 static void update_curr(struct cfs_rq *cfs_rq) 898 { 899 struct sched_entity *curr = cfs_rq->curr; 900 u64 now = rq_clock_task(rq_of(cfs_rq)); 901 u64 delta_exec; 902 903 if (unlikely(!curr)) 904 return; 905 906 delta_exec = now - curr->exec_start; 907 if (unlikely((s64)delta_exec <= 0)) 908 return; 909 910 curr->exec_start = now; 911 912 if (schedstat_enabled()) { 913 struct sched_statistics *stats; 914 915 stats = __schedstats_from_se(curr); 916 __schedstat_set(stats->exec_max, 917 max(delta_exec, stats->exec_max)); 918 } 919 920 curr->sum_exec_runtime += delta_exec; 921 schedstat_add(cfs_rq->exec_clock, delta_exec); 922 923 curr->vruntime += calc_delta_fair(delta_exec, curr); 924 update_min_vruntime(cfs_rq); 925 926 if (entity_is_task(curr)) { 927 struct task_struct *curtask = task_of(curr); 928 929 trace_sched_stat_runtime(curtask, delta_exec, curr->vruntime); 930 cgroup_account_cputime(curtask, delta_exec); 931 account_group_exec_runtime(curtask, delta_exec); 932 } 933 934 account_cfs_rq_runtime(cfs_rq, delta_exec); 935 } 936 937 static void update_curr_fair(struct rq *rq) 938 { 939 update_curr(cfs_rq_of(&rq->curr->se)); 940 } 941 942 static inline void 943 update_stats_wait_start_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) 944 { 945 struct sched_statistics *stats; 946 struct task_struct *p = NULL; 947 948 if (!schedstat_enabled()) 949 return; 950 951 stats = __schedstats_from_se(se); 952 953 if (entity_is_task(se)) 954 p = task_of(se); 955 956 __update_stats_wait_start(rq_of(cfs_rq), p, stats); 957 } 958 959 static inline void 960 update_stats_wait_end_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) 961 { 962 struct sched_statistics *stats; 963 struct task_struct *p = NULL; 964 965 if (!schedstat_enabled()) 966 return; 967 968 stats = __schedstats_from_se(se); 969 970 /* 971 * When the sched_schedstat changes from 0 to 1, some sched se 972 * maybe already in the runqueue, the se->statistics.wait_start 973 * will be 0.So it will let the delta wrong. We need to avoid this 974 * scenario. 975 */ 976 if (unlikely(!schedstat_val(stats->wait_start))) 977 return; 978 979 if (entity_is_task(se)) 980 p = task_of(se); 981 982 __update_stats_wait_end(rq_of(cfs_rq), p, stats); 983 } 984 985 static inline void 986 update_stats_enqueue_sleeper_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) 987 { 988 struct sched_statistics *stats; 989 struct task_struct *tsk = NULL; 990 991 if (!schedstat_enabled()) 992 return; 993 994 stats = __schedstats_from_se(se); 995 996 if (entity_is_task(se)) 997 tsk = task_of(se); 998 999 __update_stats_enqueue_sleeper(rq_of(cfs_rq), tsk, stats); 1000 } 1001 1002 /* 1003 * Task is being enqueued - update stats: 1004 */ 1005 static inline void 1006 update_stats_enqueue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 1007 { 1008 if (!schedstat_enabled()) 1009 return; 1010 1011 /* 1012 * Are we enqueueing a waiting task? (for current tasks 1013 * a dequeue/enqueue event is a NOP) 1014 */ 1015 if (se != cfs_rq->curr) 1016 update_stats_wait_start_fair(cfs_rq, se); 1017 1018 if (flags & ENQUEUE_WAKEUP) 1019 update_stats_enqueue_sleeper_fair(cfs_rq, se); 1020 } 1021 1022 static inline void 1023 update_stats_dequeue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 1024 { 1025 1026 if (!schedstat_enabled()) 1027 return; 1028 1029 /* 1030 * Mark the end of the wait period if dequeueing a 1031 * waiting task: 1032 */ 1033 if (se != cfs_rq->curr) 1034 update_stats_wait_end_fair(cfs_rq, se); 1035 1036 if ((flags & DEQUEUE_SLEEP) && entity_is_task(se)) { 1037 struct task_struct *tsk = task_of(se); 1038 unsigned int state; 1039 1040 /* XXX racy against TTWU */ 1041 state = READ_ONCE(tsk->__state); 1042 if (state & TASK_INTERRUPTIBLE) 1043 __schedstat_set(tsk->stats.sleep_start, 1044 rq_clock(rq_of(cfs_rq))); 1045 if (state & TASK_UNINTERRUPTIBLE) 1046 __schedstat_set(tsk->stats.block_start, 1047 rq_clock(rq_of(cfs_rq))); 1048 } 1049 } 1050 1051 /* 1052 * We are picking a new current task - update its stats: 1053 */ 1054 static inline void 1055 update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se) 1056 { 1057 /* 1058 * We are starting a new run period: 1059 */ 1060 se->exec_start = rq_clock_task(rq_of(cfs_rq)); 1061 } 1062 1063 /************************************************** 1064 * Scheduling class queueing methods: 1065 */ 1066 1067 static inline bool is_core_idle(int cpu) 1068 { 1069 #ifdef CONFIG_SCHED_SMT 1070 int sibling; 1071 1072 for_each_cpu(sibling, cpu_smt_mask(cpu)) { 1073 if (cpu == sibling) 1074 continue; 1075 1076 if (!idle_cpu(sibling)) 1077 return false; 1078 } 1079 #endif 1080 1081 return true; 1082 } 1083 1084 #ifdef CONFIG_NUMA 1085 #define NUMA_IMBALANCE_MIN 2 1086 1087 static inline long 1088 adjust_numa_imbalance(int imbalance, int dst_running, int imb_numa_nr) 1089 { 1090 /* 1091 * Allow a NUMA imbalance if busy CPUs is less than the maximum 1092 * threshold. Above this threshold, individual tasks may be contending 1093 * for both memory bandwidth and any shared HT resources. This is an 1094 * approximation as the number of running tasks may not be related to 1095 * the number of busy CPUs due to sched_setaffinity. 1096 */ 1097 if (dst_running > imb_numa_nr) 1098 return imbalance; 1099 1100 /* 1101 * Allow a small imbalance based on a simple pair of communicating 1102 * tasks that remain local when the destination is lightly loaded. 1103 */ 1104 if (imbalance <= NUMA_IMBALANCE_MIN) 1105 return 0; 1106 1107 return imbalance; 1108 } 1109 #endif /* CONFIG_NUMA */ 1110 1111 #ifdef CONFIG_NUMA_BALANCING 1112 /* 1113 * Approximate time to scan a full NUMA task in ms. The task scan period is 1114 * calculated based on the tasks virtual memory size and 1115 * numa_balancing_scan_size. 1116 */ 1117 unsigned int sysctl_numa_balancing_scan_period_min = 1000; 1118 unsigned int sysctl_numa_balancing_scan_period_max = 60000; 1119 1120 /* Portion of address space to scan in MB */ 1121 unsigned int sysctl_numa_balancing_scan_size = 256; 1122 1123 /* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */ 1124 unsigned int sysctl_numa_balancing_scan_delay = 1000; 1125 1126 /* The page with hint page fault latency < threshold in ms is considered hot */ 1127 unsigned int sysctl_numa_balancing_hot_threshold = MSEC_PER_SEC; 1128 1129 struct numa_group { 1130 refcount_t refcount; 1131 1132 spinlock_t lock; /* nr_tasks, tasks */ 1133 int nr_tasks; 1134 pid_t gid; 1135 int active_nodes; 1136 1137 struct rcu_head rcu; 1138 unsigned long total_faults; 1139 unsigned long max_faults_cpu; 1140 /* 1141 * faults[] array is split into two regions: faults_mem and faults_cpu. 1142 * 1143 * Faults_cpu is used to decide whether memory should move 1144 * towards the CPU. As a consequence, these stats are weighted 1145 * more by CPU use than by memory faults. 1146 */ 1147 unsigned long faults[]; 1148 }; 1149 1150 /* 1151 * For functions that can be called in multiple contexts that permit reading 1152 * ->numa_group (see struct task_struct for locking rules). 1153 */ 1154 static struct numa_group *deref_task_numa_group(struct task_struct *p) 1155 { 1156 return rcu_dereference_check(p->numa_group, p == current || 1157 (lockdep_is_held(__rq_lockp(task_rq(p))) && !READ_ONCE(p->on_cpu))); 1158 } 1159 1160 static struct numa_group *deref_curr_numa_group(struct task_struct *p) 1161 { 1162 return rcu_dereference_protected(p->numa_group, p == current); 1163 } 1164 1165 static inline unsigned long group_faults_priv(struct numa_group *ng); 1166 static inline unsigned long group_faults_shared(struct numa_group *ng); 1167 1168 static unsigned int task_nr_scan_windows(struct task_struct *p) 1169 { 1170 unsigned long rss = 0; 1171 unsigned long nr_scan_pages; 1172 1173 /* 1174 * Calculations based on RSS as non-present and empty pages are skipped 1175 * by the PTE scanner and NUMA hinting faults should be trapped based 1176 * on resident pages 1177 */ 1178 nr_scan_pages = sysctl_numa_balancing_scan_size << (20 - PAGE_SHIFT); 1179 rss = get_mm_rss(p->mm); 1180 if (!rss) 1181 rss = nr_scan_pages; 1182 1183 rss = round_up(rss, nr_scan_pages); 1184 return rss / nr_scan_pages; 1185 } 1186 1187 /* For sanity's sake, never scan more PTEs than MAX_SCAN_WINDOW MB/sec. */ 1188 #define MAX_SCAN_WINDOW 2560 1189 1190 static unsigned int task_scan_min(struct task_struct *p) 1191 { 1192 unsigned int scan_size = READ_ONCE(sysctl_numa_balancing_scan_size); 1193 unsigned int scan, floor; 1194 unsigned int windows = 1; 1195 1196 if (scan_size < MAX_SCAN_WINDOW) 1197 windows = MAX_SCAN_WINDOW / scan_size; 1198 floor = 1000 / windows; 1199 1200 scan = sysctl_numa_balancing_scan_period_min / task_nr_scan_windows(p); 1201 return max_t(unsigned int, floor, scan); 1202 } 1203 1204 static unsigned int task_scan_start(struct task_struct *p) 1205 { 1206 unsigned long smin = task_scan_min(p); 1207 unsigned long period = smin; 1208 struct numa_group *ng; 1209 1210 /* Scale the maximum scan period with the amount of shared memory. */ 1211 rcu_read_lock(); 1212 ng = rcu_dereference(p->numa_group); 1213 if (ng) { 1214 unsigned long shared = group_faults_shared(ng); 1215 unsigned long private = group_faults_priv(ng); 1216 1217 period *= refcount_read(&ng->refcount); 1218 period *= shared + 1; 1219 period /= private + shared + 1; 1220 } 1221 rcu_read_unlock(); 1222 1223 return max(smin, period); 1224 } 1225 1226 static unsigned int task_scan_max(struct task_struct *p) 1227 { 1228 unsigned long smin = task_scan_min(p); 1229 unsigned long smax; 1230 struct numa_group *ng; 1231 1232 /* Watch for min being lower than max due to floor calculations */ 1233 smax = sysctl_numa_balancing_scan_period_max / task_nr_scan_windows(p); 1234 1235 /* Scale the maximum scan period with the amount of shared memory. */ 1236 ng = deref_curr_numa_group(p); 1237 if (ng) { 1238 unsigned long shared = group_faults_shared(ng); 1239 unsigned long private = group_faults_priv(ng); 1240 unsigned long period = smax; 1241 1242 period *= refcount_read(&ng->refcount); 1243 period *= shared + 1; 1244 period /= private + shared + 1; 1245 1246 smax = max(smax, period); 1247 } 1248 1249 return max(smin, smax); 1250 } 1251 1252 static void account_numa_enqueue(struct rq *rq, struct task_struct *p) 1253 { 1254 rq->nr_numa_running += (p->numa_preferred_nid != NUMA_NO_NODE); 1255 rq->nr_preferred_running += (p->numa_preferred_nid == task_node(p)); 1256 } 1257 1258 static void account_numa_dequeue(struct rq *rq, struct task_struct *p) 1259 { 1260 rq->nr_numa_running -= (p->numa_preferred_nid != NUMA_NO_NODE); 1261 rq->nr_preferred_running -= (p->numa_preferred_nid == task_node(p)); 1262 } 1263 1264 /* Shared or private faults. */ 1265 #define NR_NUMA_HINT_FAULT_TYPES 2 1266 1267 /* Memory and CPU locality */ 1268 #define NR_NUMA_HINT_FAULT_STATS (NR_NUMA_HINT_FAULT_TYPES * 2) 1269 1270 /* Averaged statistics, and temporary buffers. */ 1271 #define NR_NUMA_HINT_FAULT_BUCKETS (NR_NUMA_HINT_FAULT_STATS * 2) 1272 1273 pid_t task_numa_group_id(struct task_struct *p) 1274 { 1275 struct numa_group *ng; 1276 pid_t gid = 0; 1277 1278 rcu_read_lock(); 1279 ng = rcu_dereference(p->numa_group); 1280 if (ng) 1281 gid = ng->gid; 1282 rcu_read_unlock(); 1283 1284 return gid; 1285 } 1286 1287 /* 1288 * The averaged statistics, shared & private, memory & CPU, 1289 * occupy the first half of the array. The second half of the 1290 * array is for current counters, which are averaged into the 1291 * first set by task_numa_placement. 1292 */ 1293 static inline int task_faults_idx(enum numa_faults_stats s, int nid, int priv) 1294 { 1295 return NR_NUMA_HINT_FAULT_TYPES * (s * nr_node_ids + nid) + priv; 1296 } 1297 1298 static inline unsigned long task_faults(struct task_struct *p, int nid) 1299 { 1300 if (!p->numa_faults) 1301 return 0; 1302 1303 return p->numa_faults[task_faults_idx(NUMA_MEM, nid, 0)] + 1304 p->numa_faults[task_faults_idx(NUMA_MEM, nid, 1)]; 1305 } 1306 1307 static inline unsigned long group_faults(struct task_struct *p, int nid) 1308 { 1309 struct numa_group *ng = deref_task_numa_group(p); 1310 1311 if (!ng) 1312 return 0; 1313 1314 return ng->faults[task_faults_idx(NUMA_MEM, nid, 0)] + 1315 ng->faults[task_faults_idx(NUMA_MEM, nid, 1)]; 1316 } 1317 1318 static inline unsigned long group_faults_cpu(struct numa_group *group, int nid) 1319 { 1320 return group->faults[task_faults_idx(NUMA_CPU, nid, 0)] + 1321 group->faults[task_faults_idx(NUMA_CPU, nid, 1)]; 1322 } 1323 1324 static inline unsigned long group_faults_priv(struct numa_group *ng) 1325 { 1326 unsigned long faults = 0; 1327 int node; 1328 1329 for_each_online_node(node) { 1330 faults += ng->faults[task_faults_idx(NUMA_MEM, node, 1)]; 1331 } 1332 1333 return faults; 1334 } 1335 1336 static inline unsigned long group_faults_shared(struct numa_group *ng) 1337 { 1338 unsigned long faults = 0; 1339 int node; 1340 1341 for_each_online_node(node) { 1342 faults += ng->faults[task_faults_idx(NUMA_MEM, node, 0)]; 1343 } 1344 1345 return faults; 1346 } 1347 1348 /* 1349 * A node triggering more than 1/3 as many NUMA faults as the maximum is 1350 * considered part of a numa group's pseudo-interleaving set. Migrations 1351 * between these nodes are slowed down, to allow things to settle down. 1352 */ 1353 #define ACTIVE_NODE_FRACTION 3 1354 1355 static bool numa_is_active_node(int nid, struct numa_group *ng) 1356 { 1357 return group_faults_cpu(ng, nid) * ACTIVE_NODE_FRACTION > ng->max_faults_cpu; 1358 } 1359 1360 /* Handle placement on systems where not all nodes are directly connected. */ 1361 static unsigned long score_nearby_nodes(struct task_struct *p, int nid, 1362 int lim_dist, bool task) 1363 { 1364 unsigned long score = 0; 1365 int node, max_dist; 1366 1367 /* 1368 * All nodes are directly connected, and the same distance 1369 * from each other. No need for fancy placement algorithms. 1370 */ 1371 if (sched_numa_topology_type == NUMA_DIRECT) 1372 return 0; 1373 1374 /* sched_max_numa_distance may be changed in parallel. */ 1375 max_dist = READ_ONCE(sched_max_numa_distance); 1376 /* 1377 * This code is called for each node, introducing N^2 complexity, 1378 * which should be ok given the number of nodes rarely exceeds 8. 1379 */ 1380 for_each_online_node(node) { 1381 unsigned long faults; 1382 int dist = node_distance(nid, node); 1383 1384 /* 1385 * The furthest away nodes in the system are not interesting 1386 * for placement; nid was already counted. 1387 */ 1388 if (dist >= max_dist || node == nid) 1389 continue; 1390 1391 /* 1392 * On systems with a backplane NUMA topology, compare groups 1393 * of nodes, and move tasks towards the group with the most 1394 * memory accesses. When comparing two nodes at distance 1395 * "hoplimit", only nodes closer by than "hoplimit" are part 1396 * of each group. Skip other nodes. 1397 */ 1398 if (sched_numa_topology_type == NUMA_BACKPLANE && dist >= lim_dist) 1399 continue; 1400 1401 /* Add up the faults from nearby nodes. */ 1402 if (task) 1403 faults = task_faults(p, node); 1404 else 1405 faults = group_faults(p, node); 1406 1407 /* 1408 * On systems with a glueless mesh NUMA topology, there are 1409 * no fixed "groups of nodes". Instead, nodes that are not 1410 * directly connected bounce traffic through intermediate 1411 * nodes; a numa_group can occupy any set of nodes. 1412 * The further away a node is, the less the faults count. 1413 * This seems to result in good task placement. 1414 */ 1415 if (sched_numa_topology_type == NUMA_GLUELESS_MESH) { 1416 faults *= (max_dist - dist); 1417 faults /= (max_dist - LOCAL_DISTANCE); 1418 } 1419 1420 score += faults; 1421 } 1422 1423 return score; 1424 } 1425 1426 /* 1427 * These return the fraction of accesses done by a particular task, or 1428 * task group, on a particular numa node. The group weight is given a 1429 * larger multiplier, in order to group tasks together that are almost 1430 * evenly spread out between numa nodes. 1431 */ 1432 static inline unsigned long task_weight(struct task_struct *p, int nid, 1433 int dist) 1434 { 1435 unsigned long faults, total_faults; 1436 1437 if (!p->numa_faults) 1438 return 0; 1439 1440 total_faults = p->total_numa_faults; 1441 1442 if (!total_faults) 1443 return 0; 1444 1445 faults = task_faults(p, nid); 1446 faults += score_nearby_nodes(p, nid, dist, true); 1447 1448 return 1000 * faults / total_faults; 1449 } 1450 1451 static inline unsigned long group_weight(struct task_struct *p, int nid, 1452 int dist) 1453 { 1454 struct numa_group *ng = deref_task_numa_group(p); 1455 unsigned long faults, total_faults; 1456 1457 if (!ng) 1458 return 0; 1459 1460 total_faults = ng->total_faults; 1461 1462 if (!total_faults) 1463 return 0; 1464 1465 faults = group_faults(p, nid); 1466 faults += score_nearby_nodes(p, nid, dist, false); 1467 1468 return 1000 * faults / total_faults; 1469 } 1470 1471 /* 1472 * If memory tiering mode is enabled, cpupid of slow memory page is 1473 * used to record scan time instead of CPU and PID. When tiering mode 1474 * is disabled at run time, the scan time (in cpupid) will be 1475 * interpreted as CPU and PID. So CPU needs to be checked to avoid to 1476 * access out of array bound. 1477 */ 1478 static inline bool cpupid_valid(int cpupid) 1479 { 1480 return cpupid_to_cpu(cpupid) < nr_cpu_ids; 1481 } 1482 1483 /* 1484 * For memory tiering mode, if there are enough free pages (more than 1485 * enough watermark defined here) in fast memory node, to take full 1486 * advantage of fast memory capacity, all recently accessed slow 1487 * memory pages will be migrated to fast memory node without 1488 * considering hot threshold. 1489 */ 1490 static bool pgdat_free_space_enough(struct pglist_data *pgdat) 1491 { 1492 int z; 1493 unsigned long enough_wmark; 1494 1495 enough_wmark = max(1UL * 1024 * 1024 * 1024 >> PAGE_SHIFT, 1496 pgdat->node_present_pages >> 4); 1497 for (z = pgdat->nr_zones - 1; z >= 0; z--) { 1498 struct zone *zone = pgdat->node_zones + z; 1499 1500 if (!populated_zone(zone)) 1501 continue; 1502 1503 if (zone_watermark_ok(zone, 0, 1504 wmark_pages(zone, WMARK_PROMO) + enough_wmark, 1505 ZONE_MOVABLE, 0)) 1506 return true; 1507 } 1508 return false; 1509 } 1510 1511 /* 1512 * For memory tiering mode, when page tables are scanned, the scan 1513 * time will be recorded in struct page in addition to make page 1514 * PROT_NONE for slow memory page. So when the page is accessed, in 1515 * hint page fault handler, the hint page fault latency is calculated 1516 * via, 1517 * 1518 * hint page fault latency = hint page fault time - scan time 1519 * 1520 * The smaller the hint page fault latency, the higher the possibility 1521 * for the page to be hot. 1522 */ 1523 static int numa_hint_fault_latency(struct page *page) 1524 { 1525 int last_time, time; 1526 1527 time = jiffies_to_msecs(jiffies); 1528 last_time = xchg_page_access_time(page, time); 1529 1530 return (time - last_time) & PAGE_ACCESS_TIME_MASK; 1531 } 1532 1533 /* 1534 * For memory tiering mode, too high promotion/demotion throughput may 1535 * hurt application latency. So we provide a mechanism to rate limit 1536 * the number of pages that are tried to be promoted. 1537 */ 1538 static bool numa_promotion_rate_limit(struct pglist_data *pgdat, 1539 unsigned long rate_limit, int nr) 1540 { 1541 unsigned long nr_cand; 1542 unsigned int now, start; 1543 1544 now = jiffies_to_msecs(jiffies); 1545 mod_node_page_state(pgdat, PGPROMOTE_CANDIDATE, nr); 1546 nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE); 1547 start = pgdat->nbp_rl_start; 1548 if (now - start > MSEC_PER_SEC && 1549 cmpxchg(&pgdat->nbp_rl_start, start, now) == start) 1550 pgdat->nbp_rl_nr_cand = nr_cand; 1551 if (nr_cand - pgdat->nbp_rl_nr_cand >= rate_limit) 1552 return true; 1553 return false; 1554 } 1555 1556 #define NUMA_MIGRATION_ADJUST_STEPS 16 1557 1558 static void numa_promotion_adjust_threshold(struct pglist_data *pgdat, 1559 unsigned long rate_limit, 1560 unsigned int ref_th) 1561 { 1562 unsigned int now, start, th_period, unit_th, th; 1563 unsigned long nr_cand, ref_cand, diff_cand; 1564 1565 now = jiffies_to_msecs(jiffies); 1566 th_period = sysctl_numa_balancing_scan_period_max; 1567 start = pgdat->nbp_th_start; 1568 if (now - start > th_period && 1569 cmpxchg(&pgdat->nbp_th_start, start, now) == start) { 1570 ref_cand = rate_limit * 1571 sysctl_numa_balancing_scan_period_max / MSEC_PER_SEC; 1572 nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE); 1573 diff_cand = nr_cand - pgdat->nbp_th_nr_cand; 1574 unit_th = ref_th * 2 / NUMA_MIGRATION_ADJUST_STEPS; 1575 th = pgdat->nbp_threshold ? : ref_th; 1576 if (diff_cand > ref_cand * 11 / 10) 1577 th = max(th - unit_th, unit_th); 1578 else if (diff_cand < ref_cand * 9 / 10) 1579 th = min(th + unit_th, ref_th * 2); 1580 pgdat->nbp_th_nr_cand = nr_cand; 1581 pgdat->nbp_threshold = th; 1582 } 1583 } 1584 1585 bool should_numa_migrate_memory(struct task_struct *p, struct page * page, 1586 int src_nid, int dst_cpu) 1587 { 1588 struct numa_group *ng = deref_curr_numa_group(p); 1589 int dst_nid = cpu_to_node(dst_cpu); 1590 int last_cpupid, this_cpupid; 1591 1592 /* 1593 * The pages in slow memory node should be migrated according 1594 * to hot/cold instead of private/shared. 1595 */ 1596 if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING && 1597 !node_is_toptier(src_nid)) { 1598 struct pglist_data *pgdat; 1599 unsigned long rate_limit; 1600 unsigned int latency, th, def_th; 1601 1602 pgdat = NODE_DATA(dst_nid); 1603 if (pgdat_free_space_enough(pgdat)) { 1604 /* workload changed, reset hot threshold */ 1605 pgdat->nbp_threshold = 0; 1606 return true; 1607 } 1608 1609 def_th = sysctl_numa_balancing_hot_threshold; 1610 rate_limit = sysctl_numa_balancing_promote_rate_limit << \ 1611 (20 - PAGE_SHIFT); 1612 numa_promotion_adjust_threshold(pgdat, rate_limit, def_th); 1613 1614 th = pgdat->nbp_threshold ? : def_th; 1615 latency = numa_hint_fault_latency(page); 1616 if (latency >= th) 1617 return false; 1618 1619 return !numa_promotion_rate_limit(pgdat, rate_limit, 1620 thp_nr_pages(page)); 1621 } 1622 1623 this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid); 1624 last_cpupid = page_cpupid_xchg_last(page, this_cpupid); 1625 1626 if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) && 1627 !node_is_toptier(src_nid) && !cpupid_valid(last_cpupid)) 1628 return false; 1629 1630 /* 1631 * Allow first faults or private faults to migrate immediately early in 1632 * the lifetime of a task. The magic number 4 is based on waiting for 1633 * two full passes of the "multi-stage node selection" test that is 1634 * executed below. 1635 */ 1636 if ((p->numa_preferred_nid == NUMA_NO_NODE || p->numa_scan_seq <= 4) && 1637 (cpupid_pid_unset(last_cpupid) || cpupid_match_pid(p, last_cpupid))) 1638 return true; 1639 1640 /* 1641 * Multi-stage node selection is used in conjunction with a periodic 1642 * migration fault to build a temporal task<->page relation. By using 1643 * a two-stage filter we remove short/unlikely relations. 1644 * 1645 * Using P(p) ~ n_p / n_t as per frequentist probability, we can equate 1646 * a task's usage of a particular page (n_p) per total usage of this 1647 * page (n_t) (in a given time-span) to a probability. 1648 * 1649 * Our periodic faults will sample this probability and getting the 1650 * same result twice in a row, given these samples are fully 1651 * independent, is then given by P(n)^2, provided our sample period 1652 * is sufficiently short compared to the usage pattern. 1653 * 1654 * This quadric squishes small probabilities, making it less likely we 1655 * act on an unlikely task<->page relation. 1656 */ 1657 if (!cpupid_pid_unset(last_cpupid) && 1658 cpupid_to_nid(last_cpupid) != dst_nid) 1659 return false; 1660 1661 /* Always allow migrate on private faults */ 1662 if (cpupid_match_pid(p, last_cpupid)) 1663 return true; 1664 1665 /* A shared fault, but p->numa_group has not been set up yet. */ 1666 if (!ng) 1667 return true; 1668 1669 /* 1670 * Destination node is much more heavily used than the source 1671 * node? Allow migration. 1672 */ 1673 if (group_faults_cpu(ng, dst_nid) > group_faults_cpu(ng, src_nid) * 1674 ACTIVE_NODE_FRACTION) 1675 return true; 1676 1677 /* 1678 * Distribute memory according to CPU & memory use on each node, 1679 * with 3/4 hysteresis to avoid unnecessary memory migrations: 1680 * 1681 * faults_cpu(dst) 3 faults_cpu(src) 1682 * --------------- * - > --------------- 1683 * faults_mem(dst) 4 faults_mem(src) 1684 */ 1685 return group_faults_cpu(ng, dst_nid) * group_faults(p, src_nid) * 3 > 1686 group_faults_cpu(ng, src_nid) * group_faults(p, dst_nid) * 4; 1687 } 1688 1689 /* 1690 * 'numa_type' describes the node at the moment of load balancing. 1691 */ 1692 enum numa_type { 1693 /* The node has spare capacity that can be used to run more tasks. */ 1694 node_has_spare = 0, 1695 /* 1696 * The node is fully used and the tasks don't compete for more CPU 1697 * cycles. Nevertheless, some tasks might wait before running. 1698 */ 1699 node_fully_busy, 1700 /* 1701 * The node is overloaded and can't provide expected CPU cycles to all 1702 * tasks. 1703 */ 1704 node_overloaded 1705 }; 1706 1707 /* Cached statistics for all CPUs within a node */ 1708 struct numa_stats { 1709 unsigned long load; 1710 unsigned long runnable; 1711 unsigned long util; 1712 /* Total compute capacity of CPUs on a node */ 1713 unsigned long compute_capacity; 1714 unsigned int nr_running; 1715 unsigned int weight; 1716 enum numa_type node_type; 1717 int idle_cpu; 1718 }; 1719 1720 struct task_numa_env { 1721 struct task_struct *p; 1722 1723 int src_cpu, src_nid; 1724 int dst_cpu, dst_nid; 1725 int imb_numa_nr; 1726 1727 struct numa_stats src_stats, dst_stats; 1728 1729 int imbalance_pct; 1730 int dist; 1731 1732 struct task_struct *best_task; 1733 long best_imp; 1734 int best_cpu; 1735 }; 1736 1737 static unsigned long cpu_load(struct rq *rq); 1738 static unsigned long cpu_runnable(struct rq *rq); 1739 1740 static inline enum 1741 numa_type numa_classify(unsigned int imbalance_pct, 1742 struct numa_stats *ns) 1743 { 1744 if ((ns->nr_running > ns->weight) && 1745 (((ns->compute_capacity * 100) < (ns->util * imbalance_pct)) || 1746 ((ns->compute_capacity * imbalance_pct) < (ns->runnable * 100)))) 1747 return node_overloaded; 1748 1749 if ((ns->nr_running < ns->weight) || 1750 (((ns->compute_capacity * 100) > (ns->util * imbalance_pct)) && 1751 ((ns->compute_capacity * imbalance_pct) > (ns->runnable * 100)))) 1752 return node_has_spare; 1753 1754 return node_fully_busy; 1755 } 1756 1757 #ifdef CONFIG_SCHED_SMT 1758 /* Forward declarations of select_idle_sibling helpers */ 1759 static inline bool test_idle_cores(int cpu); 1760 static inline int numa_idle_core(int idle_core, int cpu) 1761 { 1762 if (!static_branch_likely(&sched_smt_present) || 1763 idle_core >= 0 || !test_idle_cores(cpu)) 1764 return idle_core; 1765 1766 /* 1767 * Prefer cores instead of packing HT siblings 1768 * and triggering future load balancing. 1769 */ 1770 if (is_core_idle(cpu)) 1771 idle_core = cpu; 1772 1773 return idle_core; 1774 } 1775 #else 1776 static inline int numa_idle_core(int idle_core, int cpu) 1777 { 1778 return idle_core; 1779 } 1780 #endif 1781 1782 /* 1783 * Gather all necessary information to make NUMA balancing placement 1784 * decisions that are compatible with standard load balancer. This 1785 * borrows code and logic from update_sg_lb_stats but sharing a 1786 * common implementation is impractical. 1787 */ 1788 static void update_numa_stats(struct task_numa_env *env, 1789 struct numa_stats *ns, int nid, 1790 bool find_idle) 1791 { 1792 int cpu, idle_core = -1; 1793 1794 memset(ns, 0, sizeof(*ns)); 1795 ns->idle_cpu = -1; 1796 1797 rcu_read_lock(); 1798 for_each_cpu(cpu, cpumask_of_node(nid)) { 1799 struct rq *rq = cpu_rq(cpu); 1800 1801 ns->load += cpu_load(rq); 1802 ns->runnable += cpu_runnable(rq); 1803 ns->util += cpu_util_cfs(cpu); 1804 ns->nr_running += rq->cfs.h_nr_running; 1805 ns->compute_capacity += capacity_of(cpu); 1806 1807 if (find_idle && idle_core < 0 && !rq->nr_running && idle_cpu(cpu)) { 1808 if (READ_ONCE(rq->numa_migrate_on) || 1809 !cpumask_test_cpu(cpu, env->p->cpus_ptr)) 1810 continue; 1811 1812 if (ns->idle_cpu == -1) 1813 ns->idle_cpu = cpu; 1814 1815 idle_core = numa_idle_core(idle_core, cpu); 1816 } 1817 } 1818 rcu_read_unlock(); 1819 1820 ns->weight = cpumask_weight(cpumask_of_node(nid)); 1821 1822 ns->node_type = numa_classify(env->imbalance_pct, ns); 1823 1824 if (idle_core >= 0) 1825 ns->idle_cpu = idle_core; 1826 } 1827 1828 static void task_numa_assign(struct task_numa_env *env, 1829 struct task_struct *p, long imp) 1830 { 1831 struct rq *rq = cpu_rq(env->dst_cpu); 1832 1833 /* Check if run-queue part of active NUMA balance. */ 1834 if (env->best_cpu != env->dst_cpu && xchg(&rq->numa_migrate_on, 1)) { 1835 int cpu; 1836 int start = env->dst_cpu; 1837 1838 /* Find alternative idle CPU. */ 1839 for_each_cpu_wrap(cpu, cpumask_of_node(env->dst_nid), start + 1) { 1840 if (cpu == env->best_cpu || !idle_cpu(cpu) || 1841 !cpumask_test_cpu(cpu, env->p->cpus_ptr)) { 1842 continue; 1843 } 1844 1845 env->dst_cpu = cpu; 1846 rq = cpu_rq(env->dst_cpu); 1847 if (!xchg(&rq->numa_migrate_on, 1)) 1848 goto assign; 1849 } 1850 1851 /* Failed to find an alternative idle CPU */ 1852 return; 1853 } 1854 1855 assign: 1856 /* 1857 * Clear previous best_cpu/rq numa-migrate flag, since task now 1858 * found a better CPU to move/swap. 1859 */ 1860 if (env->best_cpu != -1 && env->best_cpu != env->dst_cpu) { 1861 rq = cpu_rq(env->best_cpu); 1862 WRITE_ONCE(rq->numa_migrate_on, 0); 1863 } 1864 1865 if (env->best_task) 1866 put_task_struct(env->best_task); 1867 if (p) 1868 get_task_struct(p); 1869 1870 env->best_task = p; 1871 env->best_imp = imp; 1872 env->best_cpu = env->dst_cpu; 1873 } 1874 1875 static bool load_too_imbalanced(long src_load, long dst_load, 1876 struct task_numa_env *env) 1877 { 1878 long imb, old_imb; 1879 long orig_src_load, orig_dst_load; 1880 long src_capacity, dst_capacity; 1881 1882 /* 1883 * The load is corrected for the CPU capacity available on each node. 1884 * 1885 * src_load dst_load 1886 * ------------ vs --------- 1887 * src_capacity dst_capacity 1888 */ 1889 src_capacity = env->src_stats.compute_capacity; 1890 dst_capacity = env->dst_stats.compute_capacity; 1891 1892 imb = abs(dst_load * src_capacity - src_load * dst_capacity); 1893 1894 orig_src_load = env->src_stats.load; 1895 orig_dst_load = env->dst_stats.load; 1896 1897 old_imb = abs(orig_dst_load * src_capacity - orig_src_load * dst_capacity); 1898 1899 /* Would this change make things worse? */ 1900 return (imb > old_imb); 1901 } 1902 1903 /* 1904 * Maximum NUMA importance can be 1998 (2*999); 1905 * SMALLIMP @ 30 would be close to 1998/64. 1906 * Used to deter task migration. 1907 */ 1908 #define SMALLIMP 30 1909 1910 /* 1911 * This checks if the overall compute and NUMA accesses of the system would 1912 * be improved if the source tasks was migrated to the target dst_cpu taking 1913 * into account that it might be best if task running on the dst_cpu should 1914 * be exchanged with the source task 1915 */ 1916 static bool task_numa_compare(struct task_numa_env *env, 1917 long taskimp, long groupimp, bool maymove) 1918 { 1919 struct numa_group *cur_ng, *p_ng = deref_curr_numa_group(env->p); 1920 struct rq *dst_rq = cpu_rq(env->dst_cpu); 1921 long imp = p_ng ? groupimp : taskimp; 1922 struct task_struct *cur; 1923 long src_load, dst_load; 1924 int dist = env->dist; 1925 long moveimp = imp; 1926 long load; 1927 bool stopsearch = false; 1928 1929 if (READ_ONCE(dst_rq->numa_migrate_on)) 1930 return false; 1931 1932 rcu_read_lock(); 1933 cur = rcu_dereference(dst_rq->curr); 1934 if (cur && ((cur->flags & PF_EXITING) || is_idle_task(cur))) 1935 cur = NULL; 1936 1937 /* 1938 * Because we have preemption enabled we can get migrated around and 1939 * end try selecting ourselves (current == env->p) as a swap candidate. 1940 */ 1941 if (cur == env->p) { 1942 stopsearch = true; 1943 goto unlock; 1944 } 1945 1946 if (!cur) { 1947 if (maymove && moveimp >= env->best_imp) 1948 goto assign; 1949 else 1950 goto unlock; 1951 } 1952 1953 /* Skip this swap candidate if cannot move to the source cpu. */ 1954 if (!cpumask_test_cpu(env->src_cpu, cur->cpus_ptr)) 1955 goto unlock; 1956 1957 /* 1958 * Skip this swap candidate if it is not moving to its preferred 1959 * node and the best task is. 1960 */ 1961 if (env->best_task && 1962 env->best_task->numa_preferred_nid == env->src_nid && 1963 cur->numa_preferred_nid != env->src_nid) { 1964 goto unlock; 1965 } 1966 1967 /* 1968 * "imp" is the fault differential for the source task between the 1969 * source and destination node. Calculate the total differential for 1970 * the source task and potential destination task. The more negative 1971 * the value is, the more remote accesses that would be expected to 1972 * be incurred if the tasks were swapped. 1973 * 1974 * If dst and source tasks are in the same NUMA group, or not 1975 * in any group then look only at task weights. 1976 */ 1977 cur_ng = rcu_dereference(cur->numa_group); 1978 if (cur_ng == p_ng) { 1979 /* 1980 * Do not swap within a group or between tasks that have 1981 * no group if there is spare capacity. Swapping does 1982 * not address the load imbalance and helps one task at 1983 * the cost of punishing another. 1984 */ 1985 if (env->dst_stats.node_type == node_has_spare) 1986 goto unlock; 1987 1988 imp = taskimp + task_weight(cur, env->src_nid, dist) - 1989 task_weight(cur, env->dst_nid, dist); 1990 /* 1991 * Add some hysteresis to prevent swapping the 1992 * tasks within a group over tiny differences. 1993 */ 1994 if (cur_ng) 1995 imp -= imp / 16; 1996 } else { 1997 /* 1998 * Compare the group weights. If a task is all by itself 1999 * (not part of a group), use the task weight instead. 2000 */ 2001 if (cur_ng && p_ng) 2002 imp += group_weight(cur, env->src_nid, dist) - 2003 group_weight(cur, env->dst_nid, dist); 2004 else 2005 imp += task_weight(cur, env->src_nid, dist) - 2006 task_weight(cur, env->dst_nid, dist); 2007 } 2008 2009 /* Discourage picking a task already on its preferred node */ 2010 if (cur->numa_preferred_nid == env->dst_nid) 2011 imp -= imp / 16; 2012 2013 /* 2014 * Encourage picking a task that moves to its preferred node. 2015 * This potentially makes imp larger than it's maximum of 2016 * 1998 (see SMALLIMP and task_weight for why) but in this 2017 * case, it does not matter. 2018 */ 2019 if (cur->numa_preferred_nid == env->src_nid) 2020 imp += imp / 8; 2021 2022 if (maymove && moveimp > imp && moveimp > env->best_imp) { 2023 imp = moveimp; 2024 cur = NULL; 2025 goto assign; 2026 } 2027 2028 /* 2029 * Prefer swapping with a task moving to its preferred node over a 2030 * task that is not. 2031 */ 2032 if (env->best_task && cur->numa_preferred_nid == env->src_nid && 2033 env->best_task->numa_preferred_nid != env->src_nid) { 2034 goto assign; 2035 } 2036 2037 /* 2038 * If the NUMA importance is less than SMALLIMP, 2039 * task migration might only result in ping pong 2040 * of tasks and also hurt performance due to cache 2041 * misses. 2042 */ 2043 if (imp < SMALLIMP || imp <= env->best_imp + SMALLIMP / 2) 2044 goto unlock; 2045 2046 /* 2047 * In the overloaded case, try and keep the load balanced. 2048 */ 2049 load = task_h_load(env->p) - task_h_load(cur); 2050 if (!load) 2051 goto assign; 2052 2053 dst_load = env->dst_stats.load + load; 2054 src_load = env->src_stats.load - load; 2055 2056 if (load_too_imbalanced(src_load, dst_load, env)) 2057 goto unlock; 2058 2059 assign: 2060 /* Evaluate an idle CPU for a task numa move. */ 2061 if (!cur) { 2062 int cpu = env->dst_stats.idle_cpu; 2063 2064 /* Nothing cached so current CPU went idle since the search. */ 2065 if (cpu < 0) 2066 cpu = env->dst_cpu; 2067 2068 /* 2069 * If the CPU is no longer truly idle and the previous best CPU 2070 * is, keep using it. 2071 */ 2072 if (!idle_cpu(cpu) && env->best_cpu >= 0 && 2073 idle_cpu(env->best_cpu)) { 2074 cpu = env->best_cpu; 2075 } 2076 2077 env->dst_cpu = cpu; 2078 } 2079 2080 task_numa_assign(env, cur, imp); 2081 2082 /* 2083 * If a move to idle is allowed because there is capacity or load 2084 * balance improves then stop the search. While a better swap 2085 * candidate may exist, a search is not free. 2086 */ 2087 if (maymove && !cur && env->best_cpu >= 0 && idle_cpu(env->best_cpu)) 2088 stopsearch = true; 2089 2090 /* 2091 * If a swap candidate must be identified and the current best task 2092 * moves its preferred node then stop the search. 2093 */ 2094 if (!maymove && env->best_task && 2095 env->best_task->numa_preferred_nid == env->src_nid) { 2096 stopsearch = true; 2097 } 2098 unlock: 2099 rcu_read_unlock(); 2100 2101 return stopsearch; 2102 } 2103 2104 static void task_numa_find_cpu(struct task_numa_env *env, 2105 long taskimp, long groupimp) 2106 { 2107 bool maymove = false; 2108 int cpu; 2109 2110 /* 2111 * If dst node has spare capacity, then check if there is an 2112 * imbalance that would be overruled by the load balancer. 2113 */ 2114 if (env->dst_stats.node_type == node_has_spare) { 2115 unsigned int imbalance; 2116 int src_running, dst_running; 2117 2118 /* 2119 * Would movement cause an imbalance? Note that if src has 2120 * more running tasks that the imbalance is ignored as the 2121 * move improves the imbalance from the perspective of the 2122 * CPU load balancer. 2123 * */ 2124 src_running = env->src_stats.nr_running - 1; 2125 dst_running = env->dst_stats.nr_running + 1; 2126 imbalance = max(0, dst_running - src_running); 2127 imbalance = adjust_numa_imbalance(imbalance, dst_running, 2128 env->imb_numa_nr); 2129 2130 /* Use idle CPU if there is no imbalance */ 2131 if (!imbalance) { 2132 maymove = true; 2133 if (env->dst_stats.idle_cpu >= 0) { 2134 env->dst_cpu = env->dst_stats.idle_cpu; 2135 task_numa_assign(env, NULL, 0); 2136 return; 2137 } 2138 } 2139 } else { 2140 long src_load, dst_load, load; 2141 /* 2142 * If the improvement from just moving env->p direction is better 2143 * than swapping tasks around, check if a move is possible. 2144 */ 2145 load = task_h_load(env->p); 2146 dst_load = env->dst_stats.load + load; 2147 src_load = env->src_stats.load - load; 2148 maymove = !load_too_imbalanced(src_load, dst_load, env); 2149 } 2150 2151 for_each_cpu(cpu, cpumask_of_node(env->dst_nid)) { 2152 /* Skip this CPU if the source task cannot migrate */ 2153 if (!cpumask_test_cpu(cpu, env->p->cpus_ptr)) 2154 continue; 2155 2156 env->dst_cpu = cpu; 2157 if (task_numa_compare(env, taskimp, groupimp, maymove)) 2158 break; 2159 } 2160 } 2161 2162 static int task_numa_migrate(struct task_struct *p) 2163 { 2164 struct task_numa_env env = { 2165 .p = p, 2166 2167 .src_cpu = task_cpu(p), 2168 .src_nid = task_node(p), 2169 2170 .imbalance_pct = 112, 2171 2172 .best_task = NULL, 2173 .best_imp = 0, 2174 .best_cpu = -1, 2175 }; 2176 unsigned long taskweight, groupweight; 2177 struct sched_domain *sd; 2178 long taskimp, groupimp; 2179 struct numa_group *ng; 2180 struct rq *best_rq; 2181 int nid, ret, dist; 2182 2183 /* 2184 * Pick the lowest SD_NUMA domain, as that would have the smallest 2185 * imbalance and would be the first to start moving tasks about. 2186 * 2187 * And we want to avoid any moving of tasks about, as that would create 2188 * random movement of tasks -- counter the numa conditions we're trying 2189 * to satisfy here. 2190 */ 2191 rcu_read_lock(); 2192 sd = rcu_dereference(per_cpu(sd_numa, env.src_cpu)); 2193 if (sd) { 2194 env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2; 2195 env.imb_numa_nr = sd->imb_numa_nr; 2196 } 2197 rcu_read_unlock(); 2198 2199 /* 2200 * Cpusets can break the scheduler domain tree into smaller 2201 * balance domains, some of which do not cross NUMA boundaries. 2202 * Tasks that are "trapped" in such domains cannot be migrated 2203 * elsewhere, so there is no point in (re)trying. 2204 */ 2205 if (unlikely(!sd)) { 2206 sched_setnuma(p, task_node(p)); 2207 return -EINVAL; 2208 } 2209 2210 env.dst_nid = p->numa_preferred_nid; 2211 dist = env.dist = node_distance(env.src_nid, env.dst_nid); 2212 taskweight = task_weight(p, env.src_nid, dist); 2213 groupweight = group_weight(p, env.src_nid, dist); 2214 update_numa_stats(&env, &env.src_stats, env.src_nid, false); 2215 taskimp = task_weight(p, env.dst_nid, dist) - taskweight; 2216 groupimp = group_weight(p, env.dst_nid, dist) - groupweight; 2217 update_numa_stats(&env, &env.dst_stats, env.dst_nid, true); 2218 2219 /* Try to find a spot on the preferred nid. */ 2220 task_numa_find_cpu(&env, taskimp, groupimp); 2221 2222 /* 2223 * Look at other nodes in these cases: 2224 * - there is no space available on the preferred_nid 2225 * - the task is part of a numa_group that is interleaved across 2226 * multiple NUMA nodes; in order to better consolidate the group, 2227 * we need to check other locations. 2228 */ 2229 ng = deref_curr_numa_group(p); 2230 if (env.best_cpu == -1 || (ng && ng->active_nodes > 1)) { 2231 for_each_node_state(nid, N_CPU) { 2232 if (nid == env.src_nid || nid == p->numa_preferred_nid) 2233 continue; 2234 2235 dist = node_distance(env.src_nid, env.dst_nid); 2236 if (sched_numa_topology_type == NUMA_BACKPLANE && 2237 dist != env.dist) { 2238 taskweight = task_weight(p, env.src_nid, dist); 2239 groupweight = group_weight(p, env.src_nid, dist); 2240 } 2241 2242 /* Only consider nodes where both task and groups benefit */ 2243 taskimp = task_weight(p, nid, dist) - taskweight; 2244 groupimp = group_weight(p, nid, dist) - groupweight; 2245 if (taskimp < 0 && groupimp < 0) 2246 continue; 2247 2248 env.dist = dist; 2249 env.dst_nid = nid; 2250 update_numa_stats(&env, &env.dst_stats, env.dst_nid, true); 2251 task_numa_find_cpu(&env, taskimp, groupimp); 2252 } 2253 } 2254 2255 /* 2256 * If the task is part of a workload that spans multiple NUMA nodes, 2257 * and is migrating into one of the workload's active nodes, remember 2258 * this node as the task's preferred numa node, so the workload can 2259 * settle down. 2260 * A task that migrated to a second choice node will be better off 2261 * trying for a better one later. Do not set the preferred node here. 2262 */ 2263 if (ng) { 2264 if (env.best_cpu == -1) 2265 nid = env.src_nid; 2266 else 2267 nid = cpu_to_node(env.best_cpu); 2268 2269 if (nid != p->numa_preferred_nid) 2270 sched_setnuma(p, nid); 2271 } 2272 2273 /* No better CPU than the current one was found. */ 2274 if (env.best_cpu == -1) { 2275 trace_sched_stick_numa(p, env.src_cpu, NULL, -1); 2276 return -EAGAIN; 2277 } 2278 2279 best_rq = cpu_rq(env.best_cpu); 2280 if (env.best_task == NULL) { 2281 ret = migrate_task_to(p, env.best_cpu); 2282 WRITE_ONCE(best_rq->numa_migrate_on, 0); 2283 if (ret != 0) 2284 trace_sched_stick_numa(p, env.src_cpu, NULL, env.best_cpu); 2285 return ret; 2286 } 2287 2288 ret = migrate_swap(p, env.best_task, env.best_cpu, env.src_cpu); 2289 WRITE_ONCE(best_rq->numa_migrate_on, 0); 2290 2291 if (ret != 0) 2292 trace_sched_stick_numa(p, env.src_cpu, env.best_task, env.best_cpu); 2293 put_task_struct(env.best_task); 2294 return ret; 2295 } 2296 2297 /* Attempt to migrate a task to a CPU on the preferred node. */ 2298 static void numa_migrate_preferred(struct task_struct *p) 2299 { 2300 unsigned long interval = HZ; 2301 2302 /* This task has no NUMA fault statistics yet */ 2303 if (unlikely(p->numa_preferred_nid == NUMA_NO_NODE || !p->numa_faults)) 2304 return; 2305 2306 /* Periodically retry migrating the task to the preferred node */ 2307 interval = min(interval, msecs_to_jiffies(p->numa_scan_period) / 16); 2308 p->numa_migrate_retry = jiffies + interval; 2309 2310 /* Success if task is already running on preferred CPU */ 2311 if (task_node(p) == p->numa_preferred_nid) 2312 return; 2313 2314 /* Otherwise, try migrate to a CPU on the preferred node */ 2315 task_numa_migrate(p); 2316 } 2317 2318 /* 2319 * Find out how many nodes the workload is actively running on. Do this by 2320 * tracking the nodes from which NUMA hinting faults are triggered. This can 2321 * be different from the set of nodes where the workload's memory is currently 2322 * located. 2323 */ 2324 static void numa_group_count_active_nodes(struct numa_group *numa_group) 2325 { 2326 unsigned long faults, max_faults = 0; 2327 int nid, active_nodes = 0; 2328 2329 for_each_node_state(nid, N_CPU) { 2330 faults = group_faults_cpu(numa_group, nid); 2331 if (faults > max_faults) 2332 max_faults = faults; 2333 } 2334 2335 for_each_node_state(nid, N_CPU) { 2336 faults = group_faults_cpu(numa_group, nid); 2337 if (faults * ACTIVE_NODE_FRACTION > max_faults) 2338 active_nodes++; 2339 } 2340 2341 numa_group->max_faults_cpu = max_faults; 2342 numa_group->active_nodes = active_nodes; 2343 } 2344 2345 /* 2346 * When adapting the scan rate, the period is divided into NUMA_PERIOD_SLOTS 2347 * increments. The more local the fault statistics are, the higher the scan 2348 * period will be for the next scan window. If local/(local+remote) ratio is 2349 * below NUMA_PERIOD_THRESHOLD (where range of ratio is 1..NUMA_PERIOD_SLOTS) 2350 * the scan period will decrease. Aim for 70% local accesses. 2351 */ 2352 #define NUMA_PERIOD_SLOTS 10 2353 #define NUMA_PERIOD_THRESHOLD 7 2354 2355 /* 2356 * Increase the scan period (slow down scanning) if the majority of 2357 * our memory is already on our local node, or if the majority of 2358 * the page accesses are shared with other processes. 2359 * Otherwise, decrease the scan period. 2360 */ 2361 static void update_task_scan_period(struct task_struct *p, 2362 unsigned long shared, unsigned long private) 2363 { 2364 unsigned int period_slot; 2365 int lr_ratio, ps_ratio; 2366 int diff; 2367 2368 unsigned long remote = p->numa_faults_locality[0]; 2369 unsigned long local = p->numa_faults_locality[1]; 2370 2371 /* 2372 * If there were no record hinting faults then either the task is 2373 * completely idle or all activity is in areas that are not of interest 2374 * to automatic numa balancing. Related to that, if there were failed 2375 * migration then it implies we are migrating too quickly or the local 2376 * node is overloaded. In either case, scan slower 2377 */ 2378 if (local + shared == 0 || p->numa_faults_locality[2]) { 2379 p->numa_scan_period = min(p->numa_scan_period_max, 2380 p->numa_scan_period << 1); 2381 2382 p->mm->numa_next_scan = jiffies + 2383 msecs_to_jiffies(p->numa_scan_period); 2384 2385 return; 2386 } 2387 2388 /* 2389 * Prepare to scale scan period relative to the current period. 2390 * == NUMA_PERIOD_THRESHOLD scan period stays the same 2391 * < NUMA_PERIOD_THRESHOLD scan period decreases (scan faster) 2392 * >= NUMA_PERIOD_THRESHOLD scan period increases (scan slower) 2393 */ 2394 period_slot = DIV_ROUND_UP(p->numa_scan_period, NUMA_PERIOD_SLOTS); 2395 lr_ratio = (local * NUMA_PERIOD_SLOTS) / (local + remote); 2396 ps_ratio = (private * NUMA_PERIOD_SLOTS) / (private + shared); 2397 2398 if (ps_ratio >= NUMA_PERIOD_THRESHOLD) { 2399 /* 2400 * Most memory accesses are local. There is no need to 2401 * do fast NUMA scanning, since memory is already local. 2402 */ 2403 int slot = ps_ratio - NUMA_PERIOD_THRESHOLD; 2404 if (!slot) 2405 slot = 1; 2406 diff = slot * period_slot; 2407 } else if (lr_ratio >= NUMA_PERIOD_THRESHOLD) { 2408 /* 2409 * Most memory accesses are shared with other tasks. 2410 * There is no point in continuing fast NUMA scanning, 2411 * since other tasks may just move the memory elsewhere. 2412 */ 2413 int slot = lr_ratio - NUMA_PERIOD_THRESHOLD; 2414 if (!slot) 2415 slot = 1; 2416 diff = slot * period_slot; 2417 } else { 2418 /* 2419 * Private memory faults exceed (SLOTS-THRESHOLD)/SLOTS, 2420 * yet they are not on the local NUMA node. Speed up 2421 * NUMA scanning to get the memory moved over. 2422 */ 2423 int ratio = max(lr_ratio, ps_ratio); 2424 diff = -(NUMA_PERIOD_THRESHOLD - ratio) * period_slot; 2425 } 2426 2427 p->numa_scan_period = clamp(p->numa_scan_period + diff, 2428 task_scan_min(p), task_scan_max(p)); 2429 memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); 2430 } 2431 2432 /* 2433 * Get the fraction of time the task has been running since the last 2434 * NUMA placement cycle. The scheduler keeps similar statistics, but 2435 * decays those on a 32ms period, which is orders of magnitude off 2436 * from the dozens-of-seconds NUMA balancing period. Use the scheduler 2437 * stats only if the task is so new there are no NUMA statistics yet. 2438 */ 2439 static u64 numa_get_avg_runtime(struct task_struct *p, u64 *period) 2440 { 2441 u64 runtime, delta, now; 2442 /* Use the start of this time slice to avoid calculations. */ 2443 now = p->se.exec_start; 2444 runtime = p->se.sum_exec_runtime; 2445 2446 if (p->last_task_numa_placement) { 2447 delta = runtime - p->last_sum_exec_runtime; 2448 *period = now - p->last_task_numa_placement; 2449 2450 /* Avoid time going backwards, prevent potential divide error: */ 2451 if (unlikely((s64)*period < 0)) 2452 *period = 0; 2453 } else { 2454 delta = p->se.avg.load_sum; 2455 *period = LOAD_AVG_MAX; 2456 } 2457 2458 p->last_sum_exec_runtime = runtime; 2459 p->last_task_numa_placement = now; 2460 2461 return delta; 2462 } 2463 2464 /* 2465 * Determine the preferred nid for a task in a numa_group. This needs to 2466 * be done in a way that produces consistent results with group_weight, 2467 * otherwise workloads might not converge. 2468 */ 2469 static int preferred_group_nid(struct task_struct *p, int nid) 2470 { 2471 nodemask_t nodes; 2472 int dist; 2473 2474 /* Direct connections between all NUMA nodes. */ 2475 if (sched_numa_topology_type == NUMA_DIRECT) 2476 return nid; 2477 2478 /* 2479 * On a system with glueless mesh NUMA topology, group_weight 2480 * scores nodes according to the number of NUMA hinting faults on 2481 * both the node itself, and on nearby nodes. 2482 */ 2483 if (sched_numa_topology_type == NUMA_GLUELESS_MESH) { 2484 unsigned long score, max_score = 0; 2485 int node, max_node = nid; 2486 2487 dist = sched_max_numa_distance; 2488 2489 for_each_node_state(node, N_CPU) { 2490 score = group_weight(p, node, dist); 2491 if (score > max_score) { 2492 max_score = score; 2493 max_node = node; 2494 } 2495 } 2496 return max_node; 2497 } 2498 2499 /* 2500 * Finding the preferred nid in a system with NUMA backplane 2501 * interconnect topology is more involved. The goal is to locate 2502 * tasks from numa_groups near each other in the system, and 2503 * untangle workloads from different sides of the system. This requires 2504 * searching down the hierarchy of node groups, recursively searching 2505 * inside the highest scoring group of nodes. The nodemask tricks 2506 * keep the complexity of the search down. 2507 */ 2508 nodes = node_states[N_CPU]; 2509 for (dist = sched_max_numa_distance; dist > LOCAL_DISTANCE; dist--) { 2510 unsigned long max_faults = 0; 2511 nodemask_t max_group = NODE_MASK_NONE; 2512 int a, b; 2513 2514 /* Are there nodes at this distance from each other? */ 2515 if (!find_numa_distance(dist)) 2516 continue; 2517 2518 for_each_node_mask(a, nodes) { 2519 unsigned long faults = 0; 2520 nodemask_t this_group; 2521 nodes_clear(this_group); 2522 2523 /* Sum group's NUMA faults; includes a==b case. */ 2524 for_each_node_mask(b, nodes) { 2525 if (node_distance(a, b) < dist) { 2526 faults += group_faults(p, b); 2527 node_set(b, this_group); 2528 node_clear(b, nodes); 2529 } 2530 } 2531 2532 /* Remember the top group. */ 2533 if (faults > max_faults) { 2534 max_faults = faults; 2535 max_group = this_group; 2536 /* 2537 * subtle: at the smallest distance there is 2538 * just one node left in each "group", the 2539 * winner is the preferred nid. 2540 */ 2541 nid = a; 2542 } 2543 } 2544 /* Next round, evaluate the nodes within max_group. */ 2545 if (!max_faults) 2546 break; 2547 nodes = max_group; 2548 } 2549 return nid; 2550 } 2551 2552 static void task_numa_placement(struct task_struct *p) 2553 { 2554 int seq, nid, max_nid = NUMA_NO_NODE; 2555 unsigned long max_faults = 0; 2556 unsigned long fault_types[2] = { 0, 0 }; 2557 unsigned long total_faults; 2558 u64 runtime, period; 2559 spinlock_t *group_lock = NULL; 2560 struct numa_group *ng; 2561 2562 /* 2563 * The p->mm->numa_scan_seq field gets updated without 2564 * exclusive access. Use READ_ONCE() here to ensure 2565 * that the field is read in a single access: 2566 */ 2567 seq = READ_ONCE(p->mm->numa_scan_seq); 2568 if (p->numa_scan_seq == seq) 2569 return; 2570 p->numa_scan_seq = seq; 2571 p->numa_scan_period_max = task_scan_max(p); 2572 2573 total_faults = p->numa_faults_locality[0] + 2574 p->numa_faults_locality[1]; 2575 runtime = numa_get_avg_runtime(p, &period); 2576 2577 /* If the task is part of a group prevent parallel updates to group stats */ 2578 ng = deref_curr_numa_group(p); 2579 if (ng) { 2580 group_lock = &ng->lock; 2581 spin_lock_irq(group_lock); 2582 } 2583 2584 /* Find the node with the highest number of faults */ 2585 for_each_online_node(nid) { 2586 /* Keep track of the offsets in numa_faults array */ 2587 int mem_idx, membuf_idx, cpu_idx, cpubuf_idx; 2588 unsigned long faults = 0, group_faults = 0; 2589 int priv; 2590 2591 for (priv = 0; priv < NR_NUMA_HINT_FAULT_TYPES; priv++) { 2592 long diff, f_diff, f_weight; 2593 2594 mem_idx = task_faults_idx(NUMA_MEM, nid, priv); 2595 membuf_idx = task_faults_idx(NUMA_MEMBUF, nid, priv); 2596 cpu_idx = task_faults_idx(NUMA_CPU, nid, priv); 2597 cpubuf_idx = task_faults_idx(NUMA_CPUBUF, nid, priv); 2598 2599 /* Decay existing window, copy faults since last scan */ 2600 diff = p->numa_faults[membuf_idx] - p->numa_faults[mem_idx] / 2; 2601 fault_types[priv] += p->numa_faults[membuf_idx]; 2602 p->numa_faults[membuf_idx] = 0; 2603 2604 /* 2605 * Normalize the faults_from, so all tasks in a group 2606 * count according to CPU use, instead of by the raw 2607 * number of faults. Tasks with little runtime have 2608 * little over-all impact on throughput, and thus their 2609 * faults are less important. 2610 */ 2611 f_weight = div64_u64(runtime << 16, period + 1); 2612 f_weight = (f_weight * p->numa_faults[cpubuf_idx]) / 2613 (total_faults + 1); 2614 f_diff = f_weight - p->numa_faults[cpu_idx] / 2; 2615 p->numa_faults[cpubuf_idx] = 0; 2616 2617 p->numa_faults[mem_idx] += diff; 2618 p->numa_faults[cpu_idx] += f_diff; 2619 faults += p->numa_faults[mem_idx]; 2620 p->total_numa_faults += diff; 2621 if (ng) { 2622 /* 2623 * safe because we can only change our own group 2624 * 2625 * mem_idx represents the offset for a given 2626 * nid and priv in a specific region because it 2627 * is at the beginning of the numa_faults array. 2628 */ 2629 ng->faults[mem_idx] += diff; 2630 ng->faults[cpu_idx] += f_diff; 2631 ng->total_faults += diff; 2632 group_faults += ng->faults[mem_idx]; 2633 } 2634 } 2635 2636 if (!ng) { 2637 if (faults > max_faults) { 2638 max_faults = faults; 2639 max_nid = nid; 2640 } 2641 } else if (group_faults > max_faults) { 2642 max_faults = group_faults; 2643 max_nid = nid; 2644 } 2645 } 2646 2647 /* Cannot migrate task to CPU-less node */ 2648 if (max_nid != NUMA_NO_NODE && !node_state(max_nid, N_CPU)) { 2649 int near_nid = max_nid; 2650 int distance, near_distance = INT_MAX; 2651 2652 for_each_node_state(nid, N_CPU) { 2653 distance = node_distance(max_nid, nid); 2654 if (distance < near_distance) { 2655 near_nid = nid; 2656 near_distance = distance; 2657 } 2658 } 2659 max_nid = near_nid; 2660 } 2661 2662 if (ng) { 2663 numa_group_count_active_nodes(ng); 2664 spin_unlock_irq(group_lock); 2665 max_nid = preferred_group_nid(p, max_nid); 2666 } 2667 2668 if (max_faults) { 2669 /* Set the new preferred node */ 2670 if (max_nid != p->numa_preferred_nid) 2671 sched_setnuma(p, max_nid); 2672 } 2673 2674 update_task_scan_period(p, fault_types[0], fault_types[1]); 2675 } 2676 2677 static inline int get_numa_group(struct numa_group *grp) 2678 { 2679 return refcount_inc_not_zero(&grp->refcount); 2680 } 2681 2682 static inline void put_numa_group(struct numa_group *grp) 2683 { 2684 if (refcount_dec_and_test(&grp->refcount)) 2685 kfree_rcu(grp, rcu); 2686 } 2687 2688 static void task_numa_group(struct task_struct *p, int cpupid, int flags, 2689 int *priv) 2690 { 2691 struct numa_group *grp, *my_grp; 2692 struct task_struct *tsk; 2693 bool join = false; 2694 int cpu = cpupid_to_cpu(cpupid); 2695 int i; 2696 2697 if (unlikely(!deref_curr_numa_group(p))) { 2698 unsigned int size = sizeof(struct numa_group) + 2699 NR_NUMA_HINT_FAULT_STATS * 2700 nr_node_ids * sizeof(unsigned long); 2701 2702 grp = kzalloc(size, GFP_KERNEL | __GFP_NOWARN); 2703 if (!grp) 2704 return; 2705 2706 refcount_set(&grp->refcount, 1); 2707 grp->active_nodes = 1; 2708 grp->max_faults_cpu = 0; 2709 spin_lock_init(&grp->lock); 2710 grp->gid = p->pid; 2711 2712 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 2713 grp->faults[i] = p->numa_faults[i]; 2714 2715 grp->total_faults = p->total_numa_faults; 2716 2717 grp->nr_tasks++; 2718 rcu_assign_pointer(p->numa_group, grp); 2719 } 2720 2721 rcu_read_lock(); 2722 tsk = READ_ONCE(cpu_rq(cpu)->curr); 2723 2724 if (!cpupid_match_pid(tsk, cpupid)) 2725 goto no_join; 2726 2727 grp = rcu_dereference(tsk->numa_group); 2728 if (!grp) 2729 goto no_join; 2730 2731 my_grp = deref_curr_numa_group(p); 2732 if (grp == my_grp) 2733 goto no_join; 2734 2735 /* 2736 * Only join the other group if its bigger; if we're the bigger group, 2737 * the other task will join us. 2738 */ 2739 if (my_grp->nr_tasks > grp->nr_tasks) 2740 goto no_join; 2741 2742 /* 2743 * Tie-break on the grp address. 2744 */ 2745 if (my_grp->nr_tasks == grp->nr_tasks && my_grp > grp) 2746 goto no_join; 2747 2748 /* Always join threads in the same process. */ 2749 if (tsk->mm == current->mm) 2750 join = true; 2751 2752 /* Simple filter to avoid false positives due to PID collisions */ 2753 if (flags & TNF_SHARED) 2754 join = true; 2755 2756 /* Update priv based on whether false sharing was detected */ 2757 *priv = !join; 2758 2759 if (join && !get_numa_group(grp)) 2760 goto no_join; 2761 2762 rcu_read_unlock(); 2763 2764 if (!join) 2765 return; 2766 2767 WARN_ON_ONCE(irqs_disabled()); 2768 double_lock_irq(&my_grp->lock, &grp->lock); 2769 2770 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) { 2771 my_grp->faults[i] -= p->numa_faults[i]; 2772 grp->faults[i] += p->numa_faults[i]; 2773 } 2774 my_grp->total_faults -= p->total_numa_faults; 2775 grp->total_faults += p->total_numa_faults; 2776 2777 my_grp->nr_tasks--; 2778 grp->nr_tasks++; 2779 2780 spin_unlock(&my_grp->lock); 2781 spin_unlock_irq(&grp->lock); 2782 2783 rcu_assign_pointer(p->numa_group, grp); 2784 2785 put_numa_group(my_grp); 2786 return; 2787 2788 no_join: 2789 rcu_read_unlock(); 2790 return; 2791 } 2792 2793 /* 2794 * Get rid of NUMA statistics associated with a task (either current or dead). 2795 * If @final is set, the task is dead and has reached refcount zero, so we can 2796 * safely free all relevant data structures. Otherwise, there might be 2797 * concurrent reads from places like load balancing and procfs, and we should 2798 * reset the data back to default state without freeing ->numa_faults. 2799 */ 2800 void task_numa_free(struct task_struct *p, bool final) 2801 { 2802 /* safe: p either is current or is being freed by current */ 2803 struct numa_group *grp = rcu_dereference_raw(p->numa_group); 2804 unsigned long *numa_faults = p->numa_faults; 2805 unsigned long flags; 2806 int i; 2807 2808 if (!numa_faults) 2809 return; 2810 2811 if (grp) { 2812 spin_lock_irqsave(&grp->lock, flags); 2813 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 2814 grp->faults[i] -= p->numa_faults[i]; 2815 grp->total_faults -= p->total_numa_faults; 2816 2817 grp->nr_tasks--; 2818 spin_unlock_irqrestore(&grp->lock, flags); 2819 RCU_INIT_POINTER(p->numa_group, NULL); 2820 put_numa_group(grp); 2821 } 2822 2823 if (final) { 2824 p->numa_faults = NULL; 2825 kfree(numa_faults); 2826 } else { 2827 p->total_numa_faults = 0; 2828 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 2829 numa_faults[i] = 0; 2830 } 2831 } 2832 2833 /* 2834 * Got a PROT_NONE fault for a page on @node. 2835 */ 2836 void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags) 2837 { 2838 struct task_struct *p = current; 2839 bool migrated = flags & TNF_MIGRATED; 2840 int cpu_node = task_node(current); 2841 int local = !!(flags & TNF_FAULT_LOCAL); 2842 struct numa_group *ng; 2843 int priv; 2844 2845 if (!static_branch_likely(&sched_numa_balancing)) 2846 return; 2847 2848 /* for example, ksmd faulting in a user's mm */ 2849 if (!p->mm) 2850 return; 2851 2852 /* 2853 * NUMA faults statistics are unnecessary for the slow memory 2854 * node for memory tiering mode. 2855 */ 2856 if (!node_is_toptier(mem_node) && 2857 (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING || 2858 !cpupid_valid(last_cpupid))) 2859 return; 2860 2861 /* Allocate buffer to track faults on a per-node basis */ 2862 if (unlikely(!p->numa_faults)) { 2863 int size = sizeof(*p->numa_faults) * 2864 NR_NUMA_HINT_FAULT_BUCKETS * nr_node_ids; 2865 2866 p->numa_faults = kzalloc(size, GFP_KERNEL|__GFP_NOWARN); 2867 if (!p->numa_faults) 2868 return; 2869 2870 p->total_numa_faults = 0; 2871 memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); 2872 } 2873 2874 /* 2875 * First accesses are treated as private, otherwise consider accesses 2876 * to be private if the accessing pid has not changed 2877 */ 2878 if (unlikely(last_cpupid == (-1 & LAST_CPUPID_MASK))) { 2879 priv = 1; 2880 } else { 2881 priv = cpupid_match_pid(p, last_cpupid); 2882 if (!priv && !(flags & TNF_NO_GROUP)) 2883 task_numa_group(p, last_cpupid, flags, &priv); 2884 } 2885 2886 /* 2887 * If a workload spans multiple NUMA nodes, a shared fault that 2888 * occurs wholly within the set of nodes that the workload is 2889 * actively using should be counted as local. This allows the 2890 * scan rate to slow down when a workload has settled down. 2891 */ 2892 ng = deref_curr_numa_group(p); 2893 if (!priv && !local && ng && ng->active_nodes > 1 && 2894 numa_is_active_node(cpu_node, ng) && 2895 numa_is_active_node(mem_node, ng)) 2896 local = 1; 2897 2898 /* 2899 * Retry to migrate task to preferred node periodically, in case it 2900 * previously failed, or the scheduler moved us. 2901 */ 2902 if (time_after(jiffies, p->numa_migrate_retry)) { 2903 task_numa_placement(p); 2904 numa_migrate_preferred(p); 2905 } 2906 2907 if (migrated) 2908 p->numa_pages_migrated += pages; 2909 if (flags & TNF_MIGRATE_FAIL) 2910 p->numa_faults_locality[2] += pages; 2911 2912 p->numa_faults[task_faults_idx(NUMA_MEMBUF, mem_node, priv)] += pages; 2913 p->numa_faults[task_faults_idx(NUMA_CPUBUF, cpu_node, priv)] += pages; 2914 p->numa_faults_locality[local] += pages; 2915 } 2916 2917 static void reset_ptenuma_scan(struct task_struct *p) 2918 { 2919 /* 2920 * We only did a read acquisition of the mmap sem, so 2921 * p->mm->numa_scan_seq is written to without exclusive access 2922 * and the update is not guaranteed to be atomic. That's not 2923 * much of an issue though, since this is just used for 2924 * statistical sampling. Use READ_ONCE/WRITE_ONCE, which are not 2925 * expensive, to avoid any form of compiler optimizations: 2926 */ 2927 WRITE_ONCE(p->mm->numa_scan_seq, READ_ONCE(p->mm->numa_scan_seq) + 1); 2928 p->mm->numa_scan_offset = 0; 2929 } 2930 2931 static bool vma_is_accessed(struct vm_area_struct *vma) 2932 { 2933 unsigned long pids; 2934 /* 2935 * Allow unconditional access first two times, so that all the (pages) 2936 * of VMAs get prot_none fault introduced irrespective of accesses. 2937 * This is also done to avoid any side effect of task scanning 2938 * amplifying the unfairness of disjoint set of VMAs' access. 2939 */ 2940 if (READ_ONCE(current->mm->numa_scan_seq) < 2) 2941 return true; 2942 2943 pids = vma->numab_state->access_pids[0] | vma->numab_state->access_pids[1]; 2944 return test_bit(hash_32(current->pid, ilog2(BITS_PER_LONG)), &pids); 2945 } 2946 2947 #define VMA_PID_RESET_PERIOD (4 * sysctl_numa_balancing_scan_delay) 2948 2949 /* 2950 * The expensive part of numa migration is done from task_work context. 2951 * Triggered from task_tick_numa(). 2952 */ 2953 static void task_numa_work(struct callback_head *work) 2954 { 2955 unsigned long migrate, next_scan, now = jiffies; 2956 struct task_struct *p = current; 2957 struct mm_struct *mm = p->mm; 2958 u64 runtime = p->se.sum_exec_runtime; 2959 struct vm_area_struct *vma; 2960 unsigned long start, end; 2961 unsigned long nr_pte_updates = 0; 2962 long pages, virtpages; 2963 struct vma_iterator vmi; 2964 2965 SCHED_WARN_ON(p != container_of(work, struct task_struct, numa_work)); 2966 2967 work->next = work; 2968 /* 2969 * Who cares about NUMA placement when they're dying. 2970 * 2971 * NOTE: make sure not to dereference p->mm before this check, 2972 * exit_task_work() happens _after_ exit_mm() so we could be called 2973 * without p->mm even though we still had it when we enqueued this 2974 * work. 2975 */ 2976 if (p->flags & PF_EXITING) 2977 return; 2978 2979 if (!mm->numa_next_scan) { 2980 mm->numa_next_scan = now + 2981 msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 2982 } 2983 2984 /* 2985 * Enforce maximal scan/migration frequency.. 2986 */ 2987 migrate = mm->numa_next_scan; 2988 if (time_before(now, migrate)) 2989 return; 2990 2991 if (p->numa_scan_period == 0) { 2992 p->numa_scan_period_max = task_scan_max(p); 2993 p->numa_scan_period = task_scan_start(p); 2994 } 2995 2996 next_scan = now + msecs_to_jiffies(p->numa_scan_period); 2997 if (!try_cmpxchg(&mm->numa_next_scan, &migrate, next_scan)) 2998 return; 2999 3000 /* 3001 * Delay this task enough that another task of this mm will likely win 3002 * the next time around. 3003 */ 3004 p->node_stamp += 2 * TICK_NSEC; 3005 3006 start = mm->numa_scan_offset; 3007 pages = sysctl_numa_balancing_scan_size; 3008 pages <<= 20 - PAGE_SHIFT; /* MB in pages */ 3009 virtpages = pages * 8; /* Scan up to this much virtual space */ 3010 if (!pages) 3011 return; 3012 3013 3014 if (!mmap_read_trylock(mm)) 3015 return; 3016 vma_iter_init(&vmi, mm, start); 3017 vma = vma_next(&vmi); 3018 if (!vma) { 3019 reset_ptenuma_scan(p); 3020 start = 0; 3021 vma_iter_set(&vmi, start); 3022 vma = vma_next(&vmi); 3023 } 3024 3025 do { 3026 if (!vma_migratable(vma) || !vma_policy_mof(vma) || 3027 is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) { 3028 continue; 3029 } 3030 3031 /* 3032 * Shared library pages mapped by multiple processes are not 3033 * migrated as it is expected they are cache replicated. Avoid 3034 * hinting faults in read-only file-backed mappings or the vdso 3035 * as migrating the pages will be of marginal benefit. 3036 */ 3037 if (!vma->vm_mm || 3038 (vma->vm_file && (vma->vm_flags & (VM_READ|VM_WRITE)) == (VM_READ))) 3039 continue; 3040 3041 /* 3042 * Skip inaccessible VMAs to avoid any confusion between 3043 * PROT_NONE and NUMA hinting ptes 3044 */ 3045 if (!vma_is_accessible(vma)) 3046 continue; 3047 3048 /* Initialise new per-VMA NUMAB state. */ 3049 if (!vma->numab_state) { 3050 vma->numab_state = kzalloc(sizeof(struct vma_numab_state), 3051 GFP_KERNEL); 3052 if (!vma->numab_state) 3053 continue; 3054 3055 vma->numab_state->next_scan = now + 3056 msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 3057 3058 /* Reset happens after 4 times scan delay of scan start */ 3059 vma->numab_state->next_pid_reset = vma->numab_state->next_scan + 3060 msecs_to_jiffies(VMA_PID_RESET_PERIOD); 3061 } 3062 3063 /* 3064 * Scanning the VMA's of short lived tasks add more overhead. So 3065 * delay the scan for new VMAs. 3066 */ 3067 if (mm->numa_scan_seq && time_before(jiffies, 3068 vma->numab_state->next_scan)) 3069 continue; 3070 3071 /* Do not scan the VMA if task has not accessed */ 3072 if (!vma_is_accessed(vma)) 3073 continue; 3074 3075 /* 3076 * RESET access PIDs regularly for old VMAs. Resetting after checking 3077 * vma for recent access to avoid clearing PID info before access.. 3078 */ 3079 if (mm->numa_scan_seq && 3080 time_after(jiffies, vma->numab_state->next_pid_reset)) { 3081 vma->numab_state->next_pid_reset = vma->numab_state->next_pid_reset + 3082 msecs_to_jiffies(VMA_PID_RESET_PERIOD); 3083 vma->numab_state->access_pids[0] = READ_ONCE(vma->numab_state->access_pids[1]); 3084 vma->numab_state->access_pids[1] = 0; 3085 } 3086 3087 do { 3088 start = max(start, vma->vm_start); 3089 end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE); 3090 end = min(end, vma->vm_end); 3091 nr_pte_updates = change_prot_numa(vma, start, end); 3092 3093 /* 3094 * Try to scan sysctl_numa_balancing_size worth of 3095 * hpages that have at least one present PTE that 3096 * is not already pte-numa. If the VMA contains 3097 * areas that are unused or already full of prot_numa 3098 * PTEs, scan up to virtpages, to skip through those 3099 * areas faster. 3100 */ 3101 if (nr_pte_updates) 3102 pages -= (end - start) >> PAGE_SHIFT; 3103 virtpages -= (end - start) >> PAGE_SHIFT; 3104 3105 start = end; 3106 if (pages <= 0 || virtpages <= 0) 3107 goto out; 3108 3109 cond_resched(); 3110 } while (end != vma->vm_end); 3111 } for_each_vma(vmi, vma); 3112 3113 out: 3114 /* 3115 * It is possible to reach the end of the VMA list but the last few 3116 * VMAs are not guaranteed to the vma_migratable. If they are not, we 3117 * would find the !migratable VMA on the next scan but not reset the 3118 * scanner to the start so check it now. 3119 */ 3120 if (vma) 3121 mm->numa_scan_offset = start; 3122 else 3123 reset_ptenuma_scan(p); 3124 mmap_read_unlock(mm); 3125 3126 /* 3127 * Make sure tasks use at least 32x as much time to run other code 3128 * than they used here, to limit NUMA PTE scanning overhead to 3% max. 3129 * Usually update_task_scan_period slows down scanning enough; on an 3130 * overloaded system we need to limit overhead on a per task basis. 3131 */ 3132 if (unlikely(p->se.sum_exec_runtime != runtime)) { 3133 u64 diff = p->se.sum_exec_runtime - runtime; 3134 p->node_stamp += 32 * diff; 3135 } 3136 } 3137 3138 void init_numa_balancing(unsigned long clone_flags, struct task_struct *p) 3139 { 3140 int mm_users = 0; 3141 struct mm_struct *mm = p->mm; 3142 3143 if (mm) { 3144 mm_users = atomic_read(&mm->mm_users); 3145 if (mm_users == 1) { 3146 mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 3147 mm->numa_scan_seq = 0; 3148 } 3149 } 3150 p->node_stamp = 0; 3151 p->numa_scan_seq = mm ? mm->numa_scan_seq : 0; 3152 p->numa_scan_period = sysctl_numa_balancing_scan_delay; 3153 p->numa_migrate_retry = 0; 3154 /* Protect against double add, see task_tick_numa and task_numa_work */ 3155 p->numa_work.next = &p->numa_work; 3156 p->numa_faults = NULL; 3157 p->numa_pages_migrated = 0; 3158 p->total_numa_faults = 0; 3159 RCU_INIT_POINTER(p->numa_group, NULL); 3160 p->last_task_numa_placement = 0; 3161 p->last_sum_exec_runtime = 0; 3162 3163 init_task_work(&p->numa_work, task_numa_work); 3164 3165 /* New address space, reset the preferred nid */ 3166 if (!(clone_flags & CLONE_VM)) { 3167 p->numa_preferred_nid = NUMA_NO_NODE; 3168 return; 3169 } 3170 3171 /* 3172 * New thread, keep existing numa_preferred_nid which should be copied 3173 * already by arch_dup_task_struct but stagger when scans start. 3174 */ 3175 if (mm) { 3176 unsigned int delay; 3177 3178 delay = min_t(unsigned int, task_scan_max(current), 3179 current->numa_scan_period * mm_users * NSEC_PER_MSEC); 3180 delay += 2 * TICK_NSEC; 3181 p->node_stamp = delay; 3182 } 3183 } 3184 3185 /* 3186 * Drive the periodic memory faults.. 3187 */ 3188 static void task_tick_numa(struct rq *rq, struct task_struct *curr) 3189 { 3190 struct callback_head *work = &curr->numa_work; 3191 u64 period, now; 3192 3193 /* 3194 * We don't care about NUMA placement if we don't have memory. 3195 */ 3196 if (!curr->mm || (curr->flags & (PF_EXITING | PF_KTHREAD)) || work->next != work) 3197 return; 3198 3199 /* 3200 * Using runtime rather than walltime has the dual advantage that 3201 * we (mostly) drive the selection from busy threads and that the 3202 * task needs to have done some actual work before we bother with 3203 * NUMA placement. 3204 */ 3205 now = curr->se.sum_exec_runtime; 3206 period = (u64)curr->numa_scan_period * NSEC_PER_MSEC; 3207 3208 if (now > curr->node_stamp + period) { 3209 if (!curr->node_stamp) 3210 curr->numa_scan_period = task_scan_start(curr); 3211 curr->node_stamp += period; 3212 3213 if (!time_before(jiffies, curr->mm->numa_next_scan)) 3214 task_work_add(curr, work, TWA_RESUME); 3215 } 3216 } 3217 3218 static void update_scan_period(struct task_struct *p, int new_cpu) 3219 { 3220 int src_nid = cpu_to_node(task_cpu(p)); 3221 int dst_nid = cpu_to_node(new_cpu); 3222 3223 if (!static_branch_likely(&sched_numa_balancing)) 3224 return; 3225 3226 if (!p->mm || !p->numa_faults || (p->flags & PF_EXITING)) 3227 return; 3228 3229 if (src_nid == dst_nid) 3230 return; 3231 3232 /* 3233 * Allow resets if faults have been trapped before one scan 3234 * has completed. This is most likely due to a new task that 3235 * is pulled cross-node due to wakeups or load balancing. 3236 */ 3237 if (p->numa_scan_seq) { 3238 /* 3239 * Avoid scan adjustments if moving to the preferred 3240 * node or if the task was not previously running on 3241 * the preferred node. 3242 */ 3243 if (dst_nid == p->numa_preferred_nid || 3244 (p->numa_preferred_nid != NUMA_NO_NODE && 3245 src_nid != p->numa_preferred_nid)) 3246 return; 3247 } 3248 3249 p->numa_scan_period = task_scan_start(p); 3250 } 3251 3252 #else 3253 static void task_tick_numa(struct rq *rq, struct task_struct *curr) 3254 { 3255 } 3256 3257 static inline void account_numa_enqueue(struct rq *rq, struct task_struct *p) 3258 { 3259 } 3260 3261 static inline void account_numa_dequeue(struct rq *rq, struct task_struct *p) 3262 { 3263 } 3264 3265 static inline void update_scan_period(struct task_struct *p, int new_cpu) 3266 { 3267 } 3268 3269 #endif /* CONFIG_NUMA_BALANCING */ 3270 3271 static void 3272 account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) 3273 { 3274 update_load_add(&cfs_rq->load, se->load.weight); 3275 #ifdef CONFIG_SMP 3276 if (entity_is_task(se)) { 3277 struct rq *rq = rq_of(cfs_rq); 3278 3279 account_numa_enqueue(rq, task_of(se)); 3280 list_add(&se->group_node, &rq->cfs_tasks); 3281 } 3282 #endif 3283 cfs_rq->nr_running++; 3284 if (se_is_idle(se)) 3285 cfs_rq->idle_nr_running++; 3286 } 3287 3288 static void 3289 account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) 3290 { 3291 update_load_sub(&cfs_rq->load, se->load.weight); 3292 #ifdef CONFIG_SMP 3293 if (entity_is_task(se)) { 3294 account_numa_dequeue(rq_of(cfs_rq), task_of(se)); 3295 list_del_init(&se->group_node); 3296 } 3297 #endif 3298 cfs_rq->nr_running--; 3299 if (se_is_idle(se)) 3300 cfs_rq->idle_nr_running--; 3301 } 3302 3303 /* 3304 * Signed add and clamp on underflow. 3305 * 3306 * Explicitly do a load-store to ensure the intermediate value never hits 3307 * memory. This allows lockless observations without ever seeing the negative 3308 * values. 3309 */ 3310 #define add_positive(_ptr, _val) do { \ 3311 typeof(_ptr) ptr = (_ptr); \ 3312 typeof(_val) val = (_val); \ 3313 typeof(*ptr) res, var = READ_ONCE(*ptr); \ 3314 \ 3315 res = var + val; \ 3316 \ 3317 if (val < 0 && res > var) \ 3318 res = 0; \ 3319 \ 3320 WRITE_ONCE(*ptr, res); \ 3321 } while (0) 3322 3323 /* 3324 * Unsigned subtract and clamp on underflow. 3325 * 3326 * Explicitly do a load-store to ensure the intermediate value never hits 3327 * memory. This allows lockless observations without ever seeing the negative 3328 * values. 3329 */ 3330 #define sub_positive(_ptr, _val) do { \ 3331 typeof(_ptr) ptr = (_ptr); \ 3332 typeof(*ptr) val = (_val); \ 3333 typeof(*ptr) res, var = READ_ONCE(*ptr); \ 3334 res = var - val; \ 3335 if (res > var) \ 3336 res = 0; \ 3337 WRITE_ONCE(*ptr, res); \ 3338 } while (0) 3339 3340 /* 3341 * Remove and clamp on negative, from a local variable. 3342 * 3343 * A variant of sub_positive(), which does not use explicit load-store 3344 * and is thus optimized for local variable updates. 3345 */ 3346 #define lsub_positive(_ptr, _val) do { \ 3347 typeof(_ptr) ptr = (_ptr); \ 3348 *ptr -= min_t(typeof(*ptr), *ptr, _val); \ 3349 } while (0) 3350 3351 #ifdef CONFIG_SMP 3352 static inline void 3353 enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 3354 { 3355 cfs_rq->avg.load_avg += se->avg.load_avg; 3356 cfs_rq->avg.load_sum += se_weight(se) * se->avg.load_sum; 3357 } 3358 3359 static inline void 3360 dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 3361 { 3362 sub_positive(&cfs_rq->avg.load_avg, se->avg.load_avg); 3363 sub_positive(&cfs_rq->avg.load_sum, se_weight(se) * se->avg.load_sum); 3364 /* See update_cfs_rq_load_avg() */ 3365 cfs_rq->avg.load_sum = max_t(u32, cfs_rq->avg.load_sum, 3366 cfs_rq->avg.load_avg * PELT_MIN_DIVIDER); 3367 } 3368 #else 3369 static inline void 3370 enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { } 3371 static inline void 3372 dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { } 3373 #endif 3374 3375 static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, 3376 unsigned long weight) 3377 { 3378 if (se->on_rq) { 3379 /* commit outstanding execution time */ 3380 if (cfs_rq->curr == se) 3381 update_curr(cfs_rq); 3382 update_load_sub(&cfs_rq->load, se->load.weight); 3383 } 3384 dequeue_load_avg(cfs_rq, se); 3385 3386 update_load_set(&se->load, weight); 3387 3388 #ifdef CONFIG_SMP 3389 do { 3390 u32 divider = get_pelt_divider(&se->avg); 3391 3392 se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider); 3393 } while (0); 3394 #endif 3395 3396 enqueue_load_avg(cfs_rq, se); 3397 if (se->on_rq) 3398 update_load_add(&cfs_rq->load, se->load.weight); 3399 3400 } 3401 3402 void reweight_task(struct task_struct *p, int prio) 3403 { 3404 struct sched_entity *se = &p->se; 3405 struct cfs_rq *cfs_rq = cfs_rq_of(se); 3406 struct load_weight *load = &se->load; 3407 unsigned long weight = scale_load(sched_prio_to_weight[prio]); 3408 3409 reweight_entity(cfs_rq, se, weight); 3410 load->inv_weight = sched_prio_to_wmult[prio]; 3411 } 3412 3413 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); 3414 3415 #ifdef CONFIG_FAIR_GROUP_SCHED 3416 #ifdef CONFIG_SMP 3417 /* 3418 * All this does is approximate the hierarchical proportion which includes that 3419 * global sum we all love to hate. 3420 * 3421 * That is, the weight of a group entity, is the proportional share of the 3422 * group weight based on the group runqueue weights. That is: 3423 * 3424 * tg->weight * grq->load.weight 3425 * ge->load.weight = ----------------------------- (1) 3426 * \Sum grq->load.weight 3427 * 3428 * Now, because computing that sum is prohibitively expensive to compute (been 3429 * there, done that) we approximate it with this average stuff. The average 3430 * moves slower and therefore the approximation is cheaper and more stable. 3431 * 3432 * So instead of the above, we substitute: 3433 * 3434 * grq->load.weight -> grq->avg.load_avg (2) 3435 * 3436 * which yields the following: 3437 * 3438 * tg->weight * grq->avg.load_avg 3439 * ge->load.weight = ------------------------------ (3) 3440 * tg->load_avg 3441 * 3442 * Where: tg->load_avg ~= \Sum grq->avg.load_avg 3443 * 3444 * That is shares_avg, and it is right (given the approximation (2)). 3445 * 3446 * The problem with it is that because the average is slow -- it was designed 3447 * to be exactly that of course -- this leads to transients in boundary 3448 * conditions. In specific, the case where the group was idle and we start the 3449 * one task. It takes time for our CPU's grq->avg.load_avg to build up, 3450 * yielding bad latency etc.. 3451 * 3452 * Now, in that special case (1) reduces to: 3453 * 3454 * tg->weight * grq->load.weight 3455 * ge->load.weight = ----------------------------- = tg->weight (4) 3456 * grp->load.weight 3457 * 3458 * That is, the sum collapses because all other CPUs are idle; the UP scenario. 3459 * 3460 * So what we do is modify our approximation (3) to approach (4) in the (near) 3461 * UP case, like: 3462 * 3463 * ge->load.weight = 3464 * 3465 * tg->weight * grq->load.weight 3466 * --------------------------------------------------- (5) 3467 * tg->load_avg - grq->avg.load_avg + grq->load.weight 3468 * 3469 * But because grq->load.weight can drop to 0, resulting in a divide by zero, 3470 * we need to use grq->avg.load_avg as its lower bound, which then gives: 3471 * 3472 * 3473 * tg->weight * grq->load.weight 3474 * ge->load.weight = ----------------------------- (6) 3475 * tg_load_avg' 3476 * 3477 * Where: 3478 * 3479 * tg_load_avg' = tg->load_avg - grq->avg.load_avg + 3480 * max(grq->load.weight, grq->avg.load_avg) 3481 * 3482 * And that is shares_weight and is icky. In the (near) UP case it approaches 3483 * (4) while in the normal case it approaches (3). It consistently 3484 * overestimates the ge->load.weight and therefore: 3485 * 3486 * \Sum ge->load.weight >= tg->weight 3487 * 3488 * hence icky! 3489 */ 3490 static long calc_group_shares(struct cfs_rq *cfs_rq) 3491 { 3492 long tg_weight, tg_shares, load, shares; 3493 struct task_group *tg = cfs_rq->tg; 3494 3495 tg_shares = READ_ONCE(tg->shares); 3496 3497 load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg); 3498 3499 tg_weight = atomic_long_read(&tg->load_avg); 3500 3501 /* Ensure tg_weight >= load */ 3502 tg_weight -= cfs_rq->tg_load_avg_contrib; 3503 tg_weight += load; 3504 3505 shares = (tg_shares * load); 3506 if (tg_weight) 3507 shares /= tg_weight; 3508 3509 /* 3510 * MIN_SHARES has to be unscaled here to support per-CPU partitioning 3511 * of a group with small tg->shares value. It is a floor value which is 3512 * assigned as a minimum load.weight to the sched_entity representing 3513 * the group on a CPU. 3514 * 3515 * E.g. on 64-bit for a group with tg->shares of scale_load(15)=15*1024 3516 * on an 8-core system with 8 tasks each runnable on one CPU shares has 3517 * to be 15*1024*1/8=1920 instead of scale_load(MIN_SHARES)=2*1024. In 3518 * case no task is runnable on a CPU MIN_SHARES=2 should be returned 3519 * instead of 0. 3520 */ 3521 return clamp_t(long, shares, MIN_SHARES, tg_shares); 3522 } 3523 #endif /* CONFIG_SMP */ 3524 3525 /* 3526 * Recomputes the group entity based on the current state of its group 3527 * runqueue. 3528 */ 3529 static void update_cfs_group(struct sched_entity *se) 3530 { 3531 struct cfs_rq *gcfs_rq = group_cfs_rq(se); 3532 long shares; 3533 3534 if (!gcfs_rq) 3535 return; 3536 3537 if (throttled_hierarchy(gcfs_rq)) 3538 return; 3539 3540 #ifndef CONFIG_SMP 3541 shares = READ_ONCE(gcfs_rq->tg->shares); 3542 3543 if (likely(se->load.weight == shares)) 3544 return; 3545 #else 3546 shares = calc_group_shares(gcfs_rq); 3547 #endif 3548 3549 reweight_entity(cfs_rq_of(se), se, shares); 3550 } 3551 3552 #else /* CONFIG_FAIR_GROUP_SCHED */ 3553 static inline void update_cfs_group(struct sched_entity *se) 3554 { 3555 } 3556 #endif /* CONFIG_FAIR_GROUP_SCHED */ 3557 3558 static inline void cfs_rq_util_change(struct cfs_rq *cfs_rq, int flags) 3559 { 3560 struct rq *rq = rq_of(cfs_rq); 3561 3562 if (&rq->cfs == cfs_rq) { 3563 /* 3564 * There are a few boundary cases this might miss but it should 3565 * get called often enough that that should (hopefully) not be 3566 * a real problem. 3567 * 3568 * It will not get called when we go idle, because the idle 3569 * thread is a different class (!fair), nor will the utilization 3570 * number include things like RT tasks. 3571 * 3572 * As is, the util number is not freq-invariant (we'd have to 3573 * implement arch_scale_freq_capacity() for that). 3574 * 3575 * See cpu_util_cfs(). 3576 */ 3577 cpufreq_update_util(rq, flags); 3578 } 3579 } 3580 3581 #ifdef CONFIG_SMP 3582 static inline bool load_avg_is_decayed(struct sched_avg *sa) 3583 { 3584 if (sa->load_sum) 3585 return false; 3586 3587 if (sa->util_sum) 3588 return false; 3589 3590 if (sa->runnable_sum) 3591 return false; 3592 3593 /* 3594 * _avg must be null when _sum are null because _avg = _sum / divider 3595 * Make sure that rounding and/or propagation of PELT values never 3596 * break this. 3597 */ 3598 SCHED_WARN_ON(sa->load_avg || 3599 sa->util_avg || 3600 sa->runnable_avg); 3601 3602 return true; 3603 } 3604 3605 static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq) 3606 { 3607 return u64_u32_load_copy(cfs_rq->avg.last_update_time, 3608 cfs_rq->last_update_time_copy); 3609 } 3610 #ifdef CONFIG_FAIR_GROUP_SCHED 3611 /* 3612 * Because list_add_leaf_cfs_rq always places a child cfs_rq on the list 3613 * immediately before a parent cfs_rq, and cfs_rqs are removed from the list 3614 * bottom-up, we only have to test whether the cfs_rq before us on the list 3615 * is our child. 3616 * If cfs_rq is not on the list, test whether a child needs its to be added to 3617 * connect a branch to the tree * (see list_add_leaf_cfs_rq() for details). 3618 */ 3619 static inline bool child_cfs_rq_on_list(struct cfs_rq *cfs_rq) 3620 { 3621 struct cfs_rq *prev_cfs_rq; 3622 struct list_head *prev; 3623 3624 if (cfs_rq->on_list) { 3625 prev = cfs_rq->leaf_cfs_rq_list.prev; 3626 } else { 3627 struct rq *rq = rq_of(cfs_rq); 3628 3629 prev = rq->tmp_alone_branch; 3630 } 3631 3632 prev_cfs_rq = container_of(prev, struct cfs_rq, leaf_cfs_rq_list); 3633 3634 return (prev_cfs_rq->tg->parent == cfs_rq->tg); 3635 } 3636 3637 static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq) 3638 { 3639 if (cfs_rq->load.weight) 3640 return false; 3641 3642 if (!load_avg_is_decayed(&cfs_rq->avg)) 3643 return false; 3644 3645 if (child_cfs_rq_on_list(cfs_rq)) 3646 return false; 3647 3648 return true; 3649 } 3650 3651 /** 3652 * update_tg_load_avg - update the tg's load avg 3653 * @cfs_rq: the cfs_rq whose avg changed 3654 * 3655 * This function 'ensures': tg->load_avg := \Sum tg->cfs_rq[]->avg.load. 3656 * However, because tg->load_avg is a global value there are performance 3657 * considerations. 3658 * 3659 * In order to avoid having to look at the other cfs_rq's, we use a 3660 * differential update where we store the last value we propagated. This in 3661 * turn allows skipping updates if the differential is 'small'. 3662 * 3663 * Updating tg's load_avg is necessary before update_cfs_share(). 3664 */ 3665 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) 3666 { 3667 long delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; 3668 3669 /* 3670 * No need to update load_avg for root_task_group as it is not used. 3671 */ 3672 if (cfs_rq->tg == &root_task_group) 3673 return; 3674 3675 if (abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { 3676 atomic_long_add(delta, &cfs_rq->tg->load_avg); 3677 cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; 3678 } 3679 } 3680 3681 /* 3682 * Called within set_task_rq() right before setting a task's CPU. The 3683 * caller only guarantees p->pi_lock is held; no other assumptions, 3684 * including the state of rq->lock, should be made. 3685 */ 3686 void set_task_rq_fair(struct sched_entity *se, 3687 struct cfs_rq *prev, struct cfs_rq *next) 3688 { 3689 u64 p_last_update_time; 3690 u64 n_last_update_time; 3691 3692 if (!sched_feat(ATTACH_AGE_LOAD)) 3693 return; 3694 3695 /* 3696 * We are supposed to update the task to "current" time, then its up to 3697 * date and ready to go to new CPU/cfs_rq. But we have difficulty in 3698 * getting what current time is, so simply throw away the out-of-date 3699 * time. This will result in the wakee task is less decayed, but giving 3700 * the wakee more load sounds not bad. 3701 */ 3702 if (!(se->avg.last_update_time && prev)) 3703 return; 3704 3705 p_last_update_time = cfs_rq_last_update_time(prev); 3706 n_last_update_time = cfs_rq_last_update_time(next); 3707 3708 __update_load_avg_blocked_se(p_last_update_time, se); 3709 se->avg.last_update_time = n_last_update_time; 3710 } 3711 3712 /* 3713 * When on migration a sched_entity joins/leaves the PELT hierarchy, we need to 3714 * propagate its contribution. The key to this propagation is the invariant 3715 * that for each group: 3716 * 3717 * ge->avg == grq->avg (1) 3718 * 3719 * _IFF_ we look at the pure running and runnable sums. Because they 3720 * represent the very same entity, just at different points in the hierarchy. 3721 * 3722 * Per the above update_tg_cfs_util() and update_tg_cfs_runnable() are trivial 3723 * and simply copies the running/runnable sum over (but still wrong, because 3724 * the group entity and group rq do not have their PELT windows aligned). 3725 * 3726 * However, update_tg_cfs_load() is more complex. So we have: 3727 * 3728 * ge->avg.load_avg = ge->load.weight * ge->avg.runnable_avg (2) 3729 * 3730 * And since, like util, the runnable part should be directly transferable, 3731 * the following would _appear_ to be the straight forward approach: 3732 * 3733 * grq->avg.load_avg = grq->load.weight * grq->avg.runnable_avg (3) 3734 * 3735 * And per (1) we have: 3736 * 3737 * ge->avg.runnable_avg == grq->avg.runnable_avg 3738 * 3739 * Which gives: 3740 * 3741 * ge->load.weight * grq->avg.load_avg 3742 * ge->avg.load_avg = ----------------------------------- (4) 3743 * grq->load.weight 3744 * 3745 * Except that is wrong! 3746 * 3747 * Because while for entities historical weight is not important and we 3748 * really only care about our future and therefore can consider a pure 3749 * runnable sum, runqueues can NOT do this. 3750 * 3751 * We specifically want runqueues to have a load_avg that includes 3752 * historical weights. Those represent the blocked load, the load we expect 3753 * to (shortly) return to us. This only works by keeping the weights as 3754 * integral part of the sum. We therefore cannot decompose as per (3). 3755 * 3756 * Another reason this doesn't work is that runnable isn't a 0-sum entity. 3757 * Imagine a rq with 2 tasks that each are runnable 2/3 of the time. Then the 3758 * rq itself is runnable anywhere between 2/3 and 1 depending on how the 3759 * runnable section of these tasks overlap (or not). If they were to perfectly 3760 * align the rq as a whole would be runnable 2/3 of the time. If however we 3761 * always have at least 1 runnable task, the rq as a whole is always runnable. 3762 * 3763 * So we'll have to approximate.. :/ 3764 * 3765 * Given the constraint: 3766 * 3767 * ge->avg.running_sum <= ge->avg.runnable_sum <= LOAD_AVG_MAX 3768 * 3769 * We can construct a rule that adds runnable to a rq by assuming minimal 3770 * overlap. 3771 * 3772 * On removal, we'll assume each task is equally runnable; which yields: 3773 * 3774 * grq->avg.runnable_sum = grq->avg.load_sum / grq->load.weight 3775 * 3776 * XXX: only do this for the part of runnable > running ? 3777 * 3778 */ 3779 static inline void 3780 update_tg_cfs_util(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 3781 { 3782 long delta_sum, delta_avg = gcfs_rq->avg.util_avg - se->avg.util_avg; 3783 u32 new_sum, divider; 3784 3785 /* Nothing to update */ 3786 if (!delta_avg) 3787 return; 3788 3789 /* 3790 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 3791 * See ___update_load_avg() for details. 3792 */ 3793 divider = get_pelt_divider(&cfs_rq->avg); 3794 3795 3796 /* Set new sched_entity's utilization */ 3797 se->avg.util_avg = gcfs_rq->avg.util_avg; 3798 new_sum = se->avg.util_avg * divider; 3799 delta_sum = (long)new_sum - (long)se->avg.util_sum; 3800 se->avg.util_sum = new_sum; 3801 3802 /* Update parent cfs_rq utilization */ 3803 add_positive(&cfs_rq->avg.util_avg, delta_avg); 3804 add_positive(&cfs_rq->avg.util_sum, delta_sum); 3805 3806 /* See update_cfs_rq_load_avg() */ 3807 cfs_rq->avg.util_sum = max_t(u32, cfs_rq->avg.util_sum, 3808 cfs_rq->avg.util_avg * PELT_MIN_DIVIDER); 3809 } 3810 3811 static inline void 3812 update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 3813 { 3814 long delta_sum, delta_avg = gcfs_rq->avg.runnable_avg - se->avg.runnable_avg; 3815 u32 new_sum, divider; 3816 3817 /* Nothing to update */ 3818 if (!delta_avg) 3819 return; 3820 3821 /* 3822 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 3823 * See ___update_load_avg() for details. 3824 */ 3825 divider = get_pelt_divider(&cfs_rq->avg); 3826 3827 /* Set new sched_entity's runnable */ 3828 se->avg.runnable_avg = gcfs_rq->avg.runnable_avg; 3829 new_sum = se->avg.runnable_avg * divider; 3830 delta_sum = (long)new_sum - (long)se->avg.runnable_sum; 3831 se->avg.runnable_sum = new_sum; 3832 3833 /* Update parent cfs_rq runnable */ 3834 add_positive(&cfs_rq->avg.runnable_avg, delta_avg); 3835 add_positive(&cfs_rq->avg.runnable_sum, delta_sum); 3836 /* See update_cfs_rq_load_avg() */ 3837 cfs_rq->avg.runnable_sum = max_t(u32, cfs_rq->avg.runnable_sum, 3838 cfs_rq->avg.runnable_avg * PELT_MIN_DIVIDER); 3839 } 3840 3841 static inline void 3842 update_tg_cfs_load(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 3843 { 3844 long delta_avg, running_sum, runnable_sum = gcfs_rq->prop_runnable_sum; 3845 unsigned long load_avg; 3846 u64 load_sum = 0; 3847 s64 delta_sum; 3848 u32 divider; 3849 3850 if (!runnable_sum) 3851 return; 3852 3853 gcfs_rq->prop_runnable_sum = 0; 3854 3855 /* 3856 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 3857 * See ___update_load_avg() for details. 3858 */ 3859 divider = get_pelt_divider(&cfs_rq->avg); 3860 3861 if (runnable_sum >= 0) { 3862 /* 3863 * Add runnable; clip at LOAD_AVG_MAX. Reflects that until 3864 * the CPU is saturated running == runnable. 3865 */ 3866 runnable_sum += se->avg.load_sum; 3867 runnable_sum = min_t(long, runnable_sum, divider); 3868 } else { 3869 /* 3870 * Estimate the new unweighted runnable_sum of the gcfs_rq by 3871 * assuming all tasks are equally runnable. 3872 */ 3873 if (scale_load_down(gcfs_rq->load.weight)) { 3874 load_sum = div_u64(gcfs_rq->avg.load_sum, 3875 scale_load_down(gcfs_rq->load.weight)); 3876 } 3877 3878 /* But make sure to not inflate se's runnable */ 3879 runnable_sum = min(se->avg.load_sum, load_sum); 3880 } 3881 3882 /* 3883 * runnable_sum can't be lower than running_sum 3884 * Rescale running sum to be in the same range as runnable sum 3885 * running_sum is in [0 : LOAD_AVG_MAX << SCHED_CAPACITY_SHIFT] 3886 * runnable_sum is in [0 : LOAD_AVG_MAX] 3887 */ 3888 running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT; 3889 runnable_sum = max(runnable_sum, running_sum); 3890 3891 load_sum = se_weight(se) * runnable_sum; 3892 load_avg = div_u64(load_sum, divider); 3893 3894 delta_avg = load_avg - se->avg.load_avg; 3895 if (!delta_avg) 3896 return; 3897 3898 delta_sum = load_sum - (s64)se_weight(se) * se->avg.load_sum; 3899 3900 se->avg.load_sum = runnable_sum; 3901 se->avg.load_avg = load_avg; 3902 add_positive(&cfs_rq->avg.load_avg, delta_avg); 3903 add_positive(&cfs_rq->avg.load_sum, delta_sum); 3904 /* See update_cfs_rq_load_avg() */ 3905 cfs_rq->avg.load_sum = max_t(u32, cfs_rq->avg.load_sum, 3906 cfs_rq->avg.load_avg * PELT_MIN_DIVIDER); 3907 } 3908 3909 static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) 3910 { 3911 cfs_rq->propagate = 1; 3912 cfs_rq->prop_runnable_sum += runnable_sum; 3913 } 3914 3915 /* Update task and its cfs_rq load average */ 3916 static inline int propagate_entity_load_avg(struct sched_entity *se) 3917 { 3918 struct cfs_rq *cfs_rq, *gcfs_rq; 3919 3920 if (entity_is_task(se)) 3921 return 0; 3922 3923 gcfs_rq = group_cfs_rq(se); 3924 if (!gcfs_rq->propagate) 3925 return 0; 3926 3927 gcfs_rq->propagate = 0; 3928 3929 cfs_rq = cfs_rq_of(se); 3930 3931 add_tg_cfs_propagate(cfs_rq, gcfs_rq->prop_runnable_sum); 3932 3933 update_tg_cfs_util(cfs_rq, se, gcfs_rq); 3934 update_tg_cfs_runnable(cfs_rq, se, gcfs_rq); 3935 update_tg_cfs_load(cfs_rq, se, gcfs_rq); 3936 3937 trace_pelt_cfs_tp(cfs_rq); 3938 trace_pelt_se_tp(se); 3939 3940 return 1; 3941 } 3942 3943 /* 3944 * Check if we need to update the load and the utilization of a blocked 3945 * group_entity: 3946 */ 3947 static inline bool skip_blocked_update(struct sched_entity *se) 3948 { 3949 struct cfs_rq *gcfs_rq = group_cfs_rq(se); 3950 3951 /* 3952 * If sched_entity still have not zero load or utilization, we have to 3953 * decay it: 3954 */ 3955 if (se->avg.load_avg || se->avg.util_avg) 3956 return false; 3957 3958 /* 3959 * If there is a pending propagation, we have to update the load and 3960 * the utilization of the sched_entity: 3961 */ 3962 if (gcfs_rq->propagate) 3963 return false; 3964 3965 /* 3966 * Otherwise, the load and the utilization of the sched_entity is 3967 * already zero and there is no pending propagation, so it will be a 3968 * waste of time to try to decay it: 3969 */ 3970 return true; 3971 } 3972 3973 #else /* CONFIG_FAIR_GROUP_SCHED */ 3974 3975 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) {} 3976 3977 static inline int propagate_entity_load_avg(struct sched_entity *se) 3978 { 3979 return 0; 3980 } 3981 3982 static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) {} 3983 3984 #endif /* CONFIG_FAIR_GROUP_SCHED */ 3985 3986 #ifdef CONFIG_NO_HZ_COMMON 3987 static inline void migrate_se_pelt_lag(struct sched_entity *se) 3988 { 3989 u64 throttled = 0, now, lut; 3990 struct cfs_rq *cfs_rq; 3991 struct rq *rq; 3992 bool is_idle; 3993 3994 if (load_avg_is_decayed(&se->avg)) 3995 return; 3996 3997 cfs_rq = cfs_rq_of(se); 3998 rq = rq_of(cfs_rq); 3999 4000 rcu_read_lock(); 4001 is_idle = is_idle_task(rcu_dereference(rq->curr)); 4002 rcu_read_unlock(); 4003 4004 /* 4005 * The lag estimation comes with a cost we don't want to pay all the 4006 * time. Hence, limiting to the case where the source CPU is idle and 4007 * we know we are at the greatest risk to have an outdated clock. 4008 */ 4009 if (!is_idle) 4010 return; 4011 4012 /* 4013 * Estimated "now" is: last_update_time + cfs_idle_lag + rq_idle_lag, where: 4014 * 4015 * last_update_time (the cfs_rq's last_update_time) 4016 * = cfs_rq_clock_pelt()@cfs_rq_idle 4017 * = rq_clock_pelt()@cfs_rq_idle 4018 * - cfs->throttled_clock_pelt_time@cfs_rq_idle 4019 * 4020 * cfs_idle_lag (delta between rq's update and cfs_rq's update) 4021 * = rq_clock_pelt()@rq_idle - rq_clock_pelt()@cfs_rq_idle 4022 * 4023 * rq_idle_lag (delta between now and rq's update) 4024 * = sched_clock_cpu() - rq_clock()@rq_idle 4025 * 4026 * We can then write: 4027 * 4028 * now = rq_clock_pelt()@rq_idle - cfs->throttled_clock_pelt_time + 4029 * sched_clock_cpu() - rq_clock()@rq_idle 4030 * Where: 4031 * rq_clock_pelt()@rq_idle is rq->clock_pelt_idle 4032 * rq_clock()@rq_idle is rq->clock_idle 4033 * cfs->throttled_clock_pelt_time@cfs_rq_idle 4034 * is cfs_rq->throttled_pelt_idle 4035 */ 4036 4037 #ifdef CONFIG_CFS_BANDWIDTH 4038 throttled = u64_u32_load(cfs_rq->throttled_pelt_idle); 4039 /* The clock has been stopped for throttling */ 4040 if (throttled == U64_MAX) 4041 return; 4042 #endif 4043 now = u64_u32_load(rq->clock_pelt_idle); 4044 /* 4045 * Paired with _update_idle_rq_clock_pelt(). It ensures at the worst case 4046 * is observed the old clock_pelt_idle value and the new clock_idle, 4047 * which lead to an underestimation. The opposite would lead to an 4048 * overestimation. 4049 */ 4050 smp_rmb(); 4051 lut = cfs_rq_last_update_time(cfs_rq); 4052 4053 now -= throttled; 4054 if (now < lut) 4055 /* 4056 * cfs_rq->avg.last_update_time is more recent than our 4057 * estimation, let's use it. 4058 */ 4059 now = lut; 4060 else 4061 now += sched_clock_cpu(cpu_of(rq)) - u64_u32_load(rq->clock_idle); 4062 4063 __update_load_avg_blocked_se(now, se); 4064 } 4065 #else 4066 static void migrate_se_pelt_lag(struct sched_entity *se) {} 4067 #endif 4068 4069 /** 4070 * update_cfs_rq_load_avg - update the cfs_rq's load/util averages 4071 * @now: current time, as per cfs_rq_clock_pelt() 4072 * @cfs_rq: cfs_rq to update 4073 * 4074 * The cfs_rq avg is the direct sum of all its entities (blocked and runnable) 4075 * avg. The immediate corollary is that all (fair) tasks must be attached. 4076 * 4077 * cfs_rq->avg is used for task_h_load() and update_cfs_share() for example. 4078 * 4079 * Return: true if the load decayed or we removed load. 4080 * 4081 * Since both these conditions indicate a changed cfs_rq->avg.load we should 4082 * call update_tg_load_avg() when this function returns true. 4083 */ 4084 static inline int 4085 update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) 4086 { 4087 unsigned long removed_load = 0, removed_util = 0, removed_runnable = 0; 4088 struct sched_avg *sa = &cfs_rq->avg; 4089 int decayed = 0; 4090 4091 if (cfs_rq->removed.nr) { 4092 unsigned long r; 4093 u32 divider = get_pelt_divider(&cfs_rq->avg); 4094 4095 raw_spin_lock(&cfs_rq->removed.lock); 4096 swap(cfs_rq->removed.util_avg, removed_util); 4097 swap(cfs_rq->removed.load_avg, removed_load); 4098 swap(cfs_rq->removed.runnable_avg, removed_runnable); 4099 cfs_rq->removed.nr = 0; 4100 raw_spin_unlock(&cfs_rq->removed.lock); 4101 4102 r = removed_load; 4103 sub_positive(&sa->load_avg, r); 4104 sub_positive(&sa->load_sum, r * divider); 4105 /* See sa->util_sum below */ 4106 sa->load_sum = max_t(u32, sa->load_sum, sa->load_avg * PELT_MIN_DIVIDER); 4107 4108 r = removed_util; 4109 sub_positive(&sa->util_avg, r); 4110 sub_positive(&sa->util_sum, r * divider); 4111 /* 4112 * Because of rounding, se->util_sum might ends up being +1 more than 4113 * cfs->util_sum. Although this is not a problem by itself, detaching 4114 * a lot of tasks with the rounding problem between 2 updates of 4115 * util_avg (~1ms) can make cfs->util_sum becoming null whereas 4116 * cfs_util_avg is not. 4117 * Check that util_sum is still above its lower bound for the new 4118 * util_avg. Given that period_contrib might have moved since the last 4119 * sync, we are only sure that util_sum must be above or equal to 4120 * util_avg * minimum possible divider 4121 */ 4122 sa->util_sum = max_t(u32, sa->util_sum, sa->util_avg * PELT_MIN_DIVIDER); 4123 4124 r = removed_runnable; 4125 sub_positive(&sa->runnable_avg, r); 4126 sub_positive(&sa->runnable_sum, r * divider); 4127 /* See sa->util_sum above */ 4128 sa->runnable_sum = max_t(u32, sa->runnable_sum, 4129 sa->runnable_avg * PELT_MIN_DIVIDER); 4130 4131 /* 4132 * removed_runnable is the unweighted version of removed_load so we 4133 * can use it to estimate removed_load_sum. 4134 */ 4135 add_tg_cfs_propagate(cfs_rq, 4136 -(long)(removed_runnable * divider) >> SCHED_CAPACITY_SHIFT); 4137 4138 decayed = 1; 4139 } 4140 4141 decayed |= __update_load_avg_cfs_rq(now, cfs_rq); 4142 u64_u32_store_copy(sa->last_update_time, 4143 cfs_rq->last_update_time_copy, 4144 sa->last_update_time); 4145 return decayed; 4146 } 4147 4148 /** 4149 * attach_entity_load_avg - attach this entity to its cfs_rq load avg 4150 * @cfs_rq: cfs_rq to attach to 4151 * @se: sched_entity to attach 4152 * 4153 * Must call update_cfs_rq_load_avg() before this, since we rely on 4154 * cfs_rq->avg.last_update_time being current. 4155 */ 4156 static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 4157 { 4158 /* 4159 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 4160 * See ___update_load_avg() for details. 4161 */ 4162 u32 divider = get_pelt_divider(&cfs_rq->avg); 4163 4164 /* 4165 * When we attach the @se to the @cfs_rq, we must align the decay 4166 * window because without that, really weird and wonderful things can 4167 * happen. 4168 * 4169 * XXX illustrate 4170 */ 4171 se->avg.last_update_time = cfs_rq->avg.last_update_time; 4172 se->avg.period_contrib = cfs_rq->avg.period_contrib; 4173 4174 /* 4175 * Hell(o) Nasty stuff.. we need to recompute _sum based on the new 4176 * period_contrib. This isn't strictly correct, but since we're 4177 * entirely outside of the PELT hierarchy, nobody cares if we truncate 4178 * _sum a little. 4179 */ 4180 se->avg.util_sum = se->avg.util_avg * divider; 4181 4182 se->avg.runnable_sum = se->avg.runnable_avg * divider; 4183 4184 se->avg.load_sum = se->avg.load_avg * divider; 4185 if (se_weight(se) < se->avg.load_sum) 4186 se->avg.load_sum = div_u64(se->avg.load_sum, se_weight(se)); 4187 else 4188 se->avg.load_sum = 1; 4189 4190 enqueue_load_avg(cfs_rq, se); 4191 cfs_rq->avg.util_avg += se->avg.util_avg; 4192 cfs_rq->avg.util_sum += se->avg.util_sum; 4193 cfs_rq->avg.runnable_avg += se->avg.runnable_avg; 4194 cfs_rq->avg.runnable_sum += se->avg.runnable_sum; 4195 4196 add_tg_cfs_propagate(cfs_rq, se->avg.load_sum); 4197 4198 cfs_rq_util_change(cfs_rq, 0); 4199 4200 trace_pelt_cfs_tp(cfs_rq); 4201 } 4202 4203 /** 4204 * detach_entity_load_avg - detach this entity from its cfs_rq load avg 4205 * @cfs_rq: cfs_rq to detach from 4206 * @se: sched_entity to detach 4207 * 4208 * Must call update_cfs_rq_load_avg() before this, since we rely on 4209 * cfs_rq->avg.last_update_time being current. 4210 */ 4211 static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 4212 { 4213 dequeue_load_avg(cfs_rq, se); 4214 sub_positive(&cfs_rq->avg.util_avg, se->avg.util_avg); 4215 sub_positive(&cfs_rq->avg.util_sum, se->avg.util_sum); 4216 /* See update_cfs_rq_load_avg() */ 4217 cfs_rq->avg.util_sum = max_t(u32, cfs_rq->avg.util_sum, 4218 cfs_rq->avg.util_avg * PELT_MIN_DIVIDER); 4219 4220 sub_positive(&cfs_rq->avg.runnable_avg, se->avg.runnable_avg); 4221 sub_positive(&cfs_rq->avg.runnable_sum, se->avg.runnable_sum); 4222 /* See update_cfs_rq_load_avg() */ 4223 cfs_rq->avg.runnable_sum = max_t(u32, cfs_rq->avg.runnable_sum, 4224 cfs_rq->avg.runnable_avg * PELT_MIN_DIVIDER); 4225 4226 add_tg_cfs_propagate(cfs_rq, -se->avg.load_sum); 4227 4228 cfs_rq_util_change(cfs_rq, 0); 4229 4230 trace_pelt_cfs_tp(cfs_rq); 4231 } 4232 4233 /* 4234 * Optional action to be done while updating the load average 4235 */ 4236 #define UPDATE_TG 0x1 4237 #define SKIP_AGE_LOAD 0x2 4238 #define DO_ATTACH 0x4 4239 #define DO_DETACH 0x8 4240 4241 /* Update task and its cfs_rq load average */ 4242 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 4243 { 4244 u64 now = cfs_rq_clock_pelt(cfs_rq); 4245 int decayed; 4246 4247 /* 4248 * Track task load average for carrying it to new CPU after migrated, and 4249 * track group sched_entity load average for task_h_load calc in migration 4250 */ 4251 if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD)) 4252 __update_load_avg_se(now, cfs_rq, se); 4253 4254 decayed = update_cfs_rq_load_avg(now, cfs_rq); 4255 decayed |= propagate_entity_load_avg(se); 4256 4257 if (!se->avg.last_update_time && (flags & DO_ATTACH)) { 4258 4259 /* 4260 * DO_ATTACH means we're here from enqueue_entity(). 4261 * !last_update_time means we've passed through 4262 * migrate_task_rq_fair() indicating we migrated. 4263 * 4264 * IOW we're enqueueing a task on a new CPU. 4265 */ 4266 attach_entity_load_avg(cfs_rq, se); 4267 update_tg_load_avg(cfs_rq); 4268 4269 } else if (flags & DO_DETACH) { 4270 /* 4271 * DO_DETACH means we're here from dequeue_entity() 4272 * and we are migrating task out of the CPU. 4273 */ 4274 detach_entity_load_avg(cfs_rq, se); 4275 update_tg_load_avg(cfs_rq); 4276 } else if (decayed) { 4277 cfs_rq_util_change(cfs_rq, 0); 4278 4279 if (flags & UPDATE_TG) 4280 update_tg_load_avg(cfs_rq); 4281 } 4282 } 4283 4284 /* 4285 * Synchronize entity load avg of dequeued entity without locking 4286 * the previous rq. 4287 */ 4288 static void sync_entity_load_avg(struct sched_entity *se) 4289 { 4290 struct cfs_rq *cfs_rq = cfs_rq_of(se); 4291 u64 last_update_time; 4292 4293 last_update_time = cfs_rq_last_update_time(cfs_rq); 4294 __update_load_avg_blocked_se(last_update_time, se); 4295 } 4296 4297 /* 4298 * Task first catches up with cfs_rq, and then subtract 4299 * itself from the cfs_rq (task must be off the queue now). 4300 */ 4301 static void remove_entity_load_avg(struct sched_entity *se) 4302 { 4303 struct cfs_rq *cfs_rq = cfs_rq_of(se); 4304 unsigned long flags; 4305 4306 /* 4307 * tasks cannot exit without having gone through wake_up_new_task() -> 4308 * enqueue_task_fair() which will have added things to the cfs_rq, 4309 * so we can remove unconditionally. 4310 */ 4311 4312 sync_entity_load_avg(se); 4313 4314 raw_spin_lock_irqsave(&cfs_rq->removed.lock, flags); 4315 ++cfs_rq->removed.nr; 4316 cfs_rq->removed.util_avg += se->avg.util_avg; 4317 cfs_rq->removed.load_avg += se->avg.load_avg; 4318 cfs_rq->removed.runnable_avg += se->avg.runnable_avg; 4319 raw_spin_unlock_irqrestore(&cfs_rq->removed.lock, flags); 4320 } 4321 4322 static inline unsigned long cfs_rq_runnable_avg(struct cfs_rq *cfs_rq) 4323 { 4324 return cfs_rq->avg.runnable_avg; 4325 } 4326 4327 static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq) 4328 { 4329 return cfs_rq->avg.load_avg; 4330 } 4331 4332 static int newidle_balance(struct rq *this_rq, struct rq_flags *rf); 4333 4334 static inline unsigned long task_util(struct task_struct *p) 4335 { 4336 return READ_ONCE(p->se.avg.util_avg); 4337 } 4338 4339 static inline unsigned long _task_util_est(struct task_struct *p) 4340 { 4341 struct util_est ue = READ_ONCE(p->se.avg.util_est); 4342 4343 return max(ue.ewma, (ue.enqueued & ~UTIL_AVG_UNCHANGED)); 4344 } 4345 4346 static inline unsigned long task_util_est(struct task_struct *p) 4347 { 4348 return max(task_util(p), _task_util_est(p)); 4349 } 4350 4351 #ifdef CONFIG_UCLAMP_TASK 4352 static inline unsigned long uclamp_task_util(struct task_struct *p, 4353 unsigned long uclamp_min, 4354 unsigned long uclamp_max) 4355 { 4356 return clamp(task_util_est(p), uclamp_min, uclamp_max); 4357 } 4358 #else 4359 static inline unsigned long uclamp_task_util(struct task_struct *p, 4360 unsigned long uclamp_min, 4361 unsigned long uclamp_max) 4362 { 4363 return task_util_est(p); 4364 } 4365 #endif 4366 4367 static inline void util_est_enqueue(struct cfs_rq *cfs_rq, 4368 struct task_struct *p) 4369 { 4370 unsigned int enqueued; 4371 4372 if (!sched_feat(UTIL_EST)) 4373 return; 4374 4375 /* Update root cfs_rq's estimated utilization */ 4376 enqueued = cfs_rq->avg.util_est.enqueued; 4377 enqueued += _task_util_est(p); 4378 WRITE_ONCE(cfs_rq->avg.util_est.enqueued, enqueued); 4379 4380 trace_sched_util_est_cfs_tp(cfs_rq); 4381 } 4382 4383 static inline void util_est_dequeue(struct cfs_rq *cfs_rq, 4384 struct task_struct *p) 4385 { 4386 unsigned int enqueued; 4387 4388 if (!sched_feat(UTIL_EST)) 4389 return; 4390 4391 /* Update root cfs_rq's estimated utilization */ 4392 enqueued = cfs_rq->avg.util_est.enqueued; 4393 enqueued -= min_t(unsigned int, enqueued, _task_util_est(p)); 4394 WRITE_ONCE(cfs_rq->avg.util_est.enqueued, enqueued); 4395 4396 trace_sched_util_est_cfs_tp(cfs_rq); 4397 } 4398 4399 #define UTIL_EST_MARGIN (SCHED_CAPACITY_SCALE / 100) 4400 4401 /* 4402 * Check if a (signed) value is within a specified (unsigned) margin, 4403 * based on the observation that: 4404 * 4405 * abs(x) < y := (unsigned)(x + y - 1) < (2 * y - 1) 4406 * 4407 * NOTE: this only works when value + margin < INT_MAX. 4408 */ 4409 static inline bool within_margin(int value, int margin) 4410 { 4411 return ((unsigned int)(value + margin - 1) < (2 * margin - 1)); 4412 } 4413 4414 static inline void util_est_update(struct cfs_rq *cfs_rq, 4415 struct task_struct *p, 4416 bool task_sleep) 4417 { 4418 long last_ewma_diff, last_enqueued_diff; 4419 struct util_est ue; 4420 4421 if (!sched_feat(UTIL_EST)) 4422 return; 4423 4424 /* 4425 * Skip update of task's estimated utilization when the task has not 4426 * yet completed an activation, e.g. being migrated. 4427 */ 4428 if (!task_sleep) 4429 return; 4430 4431 /* 4432 * If the PELT values haven't changed since enqueue time, 4433 * skip the util_est update. 4434 */ 4435 ue = p->se.avg.util_est; 4436 if (ue.enqueued & UTIL_AVG_UNCHANGED) 4437 return; 4438 4439 last_enqueued_diff = ue.enqueued; 4440 4441 /* 4442 * Reset EWMA on utilization increases, the moving average is used only 4443 * to smooth utilization decreases. 4444 */ 4445 ue.enqueued = task_util(p); 4446 if (sched_feat(UTIL_EST_FASTUP)) { 4447 if (ue.ewma < ue.enqueued) { 4448 ue.ewma = ue.enqueued; 4449 goto done; 4450 } 4451 } 4452 4453 /* 4454 * Skip update of task's estimated utilization when its members are 4455 * already ~1% close to its last activation value. 4456 */ 4457 last_ewma_diff = ue.enqueued - ue.ewma; 4458 last_enqueued_diff -= ue.enqueued; 4459 if (within_margin(last_ewma_diff, UTIL_EST_MARGIN)) { 4460 if (!within_margin(last_enqueued_diff, UTIL_EST_MARGIN)) 4461 goto done; 4462 4463 return; 4464 } 4465 4466 /* 4467 * To avoid overestimation of actual task utilization, skip updates if 4468 * we cannot grant there is idle time in this CPU. 4469 */ 4470 if (task_util(p) > capacity_orig_of(cpu_of(rq_of(cfs_rq)))) 4471 return; 4472 4473 /* 4474 * Update Task's estimated utilization 4475 * 4476 * When *p completes an activation we can consolidate another sample 4477 * of the task size. This is done by storing the current PELT value 4478 * as ue.enqueued and by using this value to update the Exponential 4479 * Weighted Moving Average (EWMA): 4480 * 4481 * ewma(t) = w * task_util(p) + (1-w) * ewma(t-1) 4482 * = w * task_util(p) + ewma(t-1) - w * ewma(t-1) 4483 * = w * (task_util(p) - ewma(t-1)) + ewma(t-1) 4484 * = w * ( last_ewma_diff ) + ewma(t-1) 4485 * = w * (last_ewma_diff + ewma(t-1) / w) 4486 * 4487 * Where 'w' is the weight of new samples, which is configured to be 4488 * 0.25, thus making w=1/4 ( >>= UTIL_EST_WEIGHT_SHIFT) 4489 */ 4490 ue.ewma <<= UTIL_EST_WEIGHT_SHIFT; 4491 ue.ewma += last_ewma_diff; 4492 ue.ewma >>= UTIL_EST_WEIGHT_SHIFT; 4493 done: 4494 ue.enqueued |= UTIL_AVG_UNCHANGED; 4495 WRITE_ONCE(p->se.avg.util_est, ue); 4496 4497 trace_sched_util_est_se_tp(&p->se); 4498 } 4499 4500 static inline int util_fits_cpu(unsigned long util, 4501 unsigned long uclamp_min, 4502 unsigned long uclamp_max, 4503 int cpu) 4504 { 4505 unsigned long capacity_orig, capacity_orig_thermal; 4506 unsigned long capacity = capacity_of(cpu); 4507 bool fits, uclamp_max_fits; 4508 4509 /* 4510 * Check if the real util fits without any uclamp boost/cap applied. 4511 */ 4512 fits = fits_capacity(util, capacity); 4513 4514 if (!uclamp_is_used()) 4515 return fits; 4516 4517 /* 4518 * We must use capacity_orig_of() for comparing against uclamp_min and 4519 * uclamp_max. We only care about capacity pressure (by using 4520 * capacity_of()) for comparing against the real util. 4521 * 4522 * If a task is boosted to 1024 for example, we don't want a tiny 4523 * pressure to skew the check whether it fits a CPU or not. 4524 * 4525 * Similarly if a task is capped to capacity_orig_of(little_cpu), it 4526 * should fit a little cpu even if there's some pressure. 4527 * 4528 * Only exception is for thermal pressure since it has a direct impact 4529 * on available OPP of the system. 4530 * 4531 * We honour it for uclamp_min only as a drop in performance level 4532 * could result in not getting the requested minimum performance level. 4533 * 4534 * For uclamp_max, we can tolerate a drop in performance level as the 4535 * goal is to cap the task. So it's okay if it's getting less. 4536 */ 4537 capacity_orig = capacity_orig_of(cpu); 4538 capacity_orig_thermal = capacity_orig - arch_scale_thermal_pressure(cpu); 4539 4540 /* 4541 * We want to force a task to fit a cpu as implied by uclamp_max. 4542 * But we do have some corner cases to cater for.. 4543 * 4544 * 4545 * C=z 4546 * | ___ 4547 * | C=y | | 4548 * |_ _ _ _ _ _ _ _ _ ___ _ _ _ | _ | _ _ _ _ _ uclamp_max 4549 * | C=x | | | | 4550 * | ___ | | | | 4551 * | | | | | | | (util somewhere in this region) 4552 * | | | | | | | 4553 * | | | | | | | 4554 * +---------------------------------------- 4555 * cpu0 cpu1 cpu2 4556 * 4557 * In the above example if a task is capped to a specific performance 4558 * point, y, then when: 4559 * 4560 * * util = 80% of x then it does not fit on cpu0 and should migrate 4561 * to cpu1 4562 * * util = 80% of y then it is forced to fit on cpu1 to honour 4563 * uclamp_max request. 4564 * 4565 * which is what we're enforcing here. A task always fits if 4566 * uclamp_max <= capacity_orig. But when uclamp_max > capacity_orig, 4567 * the normal upmigration rules should withhold still. 4568 * 4569 * Only exception is when we are on max capacity, then we need to be 4570 * careful not to block overutilized state. This is so because: 4571 * 4572 * 1. There's no concept of capping at max_capacity! We can't go 4573 * beyond this performance level anyway. 4574 * 2. The system is being saturated when we're operating near 4575 * max capacity, it doesn't make sense to block overutilized. 4576 */ 4577 uclamp_max_fits = (capacity_orig == SCHED_CAPACITY_SCALE) && (uclamp_max == SCHED_CAPACITY_SCALE); 4578 uclamp_max_fits = !uclamp_max_fits && (uclamp_max <= capacity_orig); 4579 fits = fits || uclamp_max_fits; 4580 4581 /* 4582 * 4583 * C=z 4584 * | ___ (region a, capped, util >= uclamp_max) 4585 * | C=y | | 4586 * |_ _ _ _ _ _ _ _ _ ___ _ _ _ | _ | _ _ _ _ _ uclamp_max 4587 * | C=x | | | | 4588 * | ___ | | | | (region b, uclamp_min <= util <= uclamp_max) 4589 * |_ _ _|_ _|_ _ _ _| _ | _ _ _| _ | _ _ _ _ _ uclamp_min 4590 * | | | | | | | 4591 * | | | | | | | (region c, boosted, util < uclamp_min) 4592 * +---------------------------------------- 4593 * cpu0 cpu1 cpu2 4594 * 4595 * a) If util > uclamp_max, then we're capped, we don't care about 4596 * actual fitness value here. We only care if uclamp_max fits 4597 * capacity without taking margin/pressure into account. 4598 * See comment above. 4599 * 4600 * b) If uclamp_min <= util <= uclamp_max, then the normal 4601 * fits_capacity() rules apply. Except we need to ensure that we 4602 * enforce we remain within uclamp_max, see comment above. 4603 * 4604 * c) If util < uclamp_min, then we are boosted. Same as (b) but we 4605 * need to take into account the boosted value fits the CPU without 4606 * taking margin/pressure into account. 4607 * 4608 * Cases (a) and (b) are handled in the 'fits' variable already. We 4609 * just need to consider an extra check for case (c) after ensuring we 4610 * handle the case uclamp_min > uclamp_max. 4611 */ 4612 uclamp_min = min(uclamp_min, uclamp_max); 4613 if (fits && (util < uclamp_min) && (uclamp_min > capacity_orig_thermal)) 4614 return -1; 4615 4616 return fits; 4617 } 4618 4619 static inline int task_fits_cpu(struct task_struct *p, int cpu) 4620 { 4621 unsigned long uclamp_min = uclamp_eff_value(p, UCLAMP_MIN); 4622 unsigned long uclamp_max = uclamp_eff_value(p, UCLAMP_MAX); 4623 unsigned long util = task_util_est(p); 4624 /* 4625 * Return true only if the cpu fully fits the task requirements, which 4626 * include the utilization but also the performance hints. 4627 */ 4628 return (util_fits_cpu(util, uclamp_min, uclamp_max, cpu) > 0); 4629 } 4630 4631 static inline void update_misfit_status(struct task_struct *p, struct rq *rq) 4632 { 4633 if (!sched_asym_cpucap_active()) 4634 return; 4635 4636 if (!p || p->nr_cpus_allowed == 1) { 4637 rq->misfit_task_load = 0; 4638 return; 4639 } 4640 4641 if (task_fits_cpu(p, cpu_of(rq))) { 4642 rq->misfit_task_load = 0; 4643 return; 4644 } 4645 4646 /* 4647 * Make sure that misfit_task_load will not be null even if 4648 * task_h_load() returns 0. 4649 */ 4650 rq->misfit_task_load = max_t(unsigned long, task_h_load(p), 1); 4651 } 4652 4653 #else /* CONFIG_SMP */ 4654 4655 static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq) 4656 { 4657 return true; 4658 } 4659 4660 #define UPDATE_TG 0x0 4661 #define SKIP_AGE_LOAD 0x0 4662 #define DO_ATTACH 0x0 4663 #define DO_DETACH 0x0 4664 4665 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int not_used1) 4666 { 4667 cfs_rq_util_change(cfs_rq, 0); 4668 } 4669 4670 static inline void remove_entity_load_avg(struct sched_entity *se) {} 4671 4672 static inline void 4673 attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} 4674 static inline void 4675 detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} 4676 4677 static inline int newidle_balance(struct rq *rq, struct rq_flags *rf) 4678 { 4679 return 0; 4680 } 4681 4682 static inline void 4683 util_est_enqueue(struct cfs_rq *cfs_rq, struct task_struct *p) {} 4684 4685 static inline void 4686 util_est_dequeue(struct cfs_rq *cfs_rq, struct task_struct *p) {} 4687 4688 static inline void 4689 util_est_update(struct cfs_rq *cfs_rq, struct task_struct *p, 4690 bool task_sleep) {} 4691 static inline void update_misfit_status(struct task_struct *p, struct rq *rq) {} 4692 4693 #endif /* CONFIG_SMP */ 4694 4695 static void check_spread(struct cfs_rq *cfs_rq, struct sched_entity *se) 4696 { 4697 #ifdef CONFIG_SCHED_DEBUG 4698 s64 d = se->vruntime - cfs_rq->min_vruntime; 4699 4700 if (d < 0) 4701 d = -d; 4702 4703 if (d > 3*sysctl_sched_latency) 4704 schedstat_inc(cfs_rq->nr_spread_over); 4705 #endif 4706 } 4707 4708 static inline bool entity_is_long_sleeper(struct sched_entity *se) 4709 { 4710 struct cfs_rq *cfs_rq; 4711 u64 sleep_time; 4712 4713 if (se->exec_start == 0) 4714 return false; 4715 4716 cfs_rq = cfs_rq_of(se); 4717 4718 sleep_time = rq_clock_task(rq_of(cfs_rq)); 4719 4720 /* Happen while migrating because of clock task divergence */ 4721 if (sleep_time <= se->exec_start) 4722 return false; 4723 4724 sleep_time -= se->exec_start; 4725 if (sleep_time > ((1ULL << 63) / scale_load_down(NICE_0_LOAD))) 4726 return true; 4727 4728 return false; 4729 } 4730 4731 static void 4732 place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial) 4733 { 4734 u64 vruntime = cfs_rq->min_vruntime; 4735 4736 /* 4737 * The 'current' period is already promised to the current tasks, 4738 * however the extra weight of the new task will slow them down a 4739 * little, place the new task so that it fits in the slot that 4740 * stays open at the end. 4741 */ 4742 if (initial && sched_feat(START_DEBIT)) 4743 vruntime += sched_vslice(cfs_rq, se); 4744 4745 /* sleeps up to a single latency don't count. */ 4746 if (!initial) { 4747 unsigned long thresh; 4748 4749 if (se_is_idle(se)) 4750 thresh = sysctl_sched_min_granularity; 4751 else 4752 thresh = sysctl_sched_latency; 4753 4754 /* 4755 * Halve their sleep time's effect, to allow 4756 * for a gentler effect of sleepers: 4757 */ 4758 if (sched_feat(GENTLE_FAIR_SLEEPERS)) 4759 thresh >>= 1; 4760 4761 vruntime -= thresh; 4762 } 4763 4764 /* 4765 * Pull vruntime of the entity being placed to the base level of 4766 * cfs_rq, to prevent boosting it if placed backwards. 4767 * However, min_vruntime can advance much faster than real time, with 4768 * the extreme being when an entity with the minimal weight always runs 4769 * on the cfs_rq. If the waking entity slept for a long time, its 4770 * vruntime difference from min_vruntime may overflow s64 and their 4771 * comparison may get inversed, so ignore the entity's original 4772 * vruntime in that case. 4773 * The maximal vruntime speedup is given by the ratio of normal to 4774 * minimal weight: scale_load_down(NICE_0_LOAD) / MIN_SHARES. 4775 * When placing a migrated waking entity, its exec_start has been set 4776 * from a different rq. In order to take into account a possible 4777 * divergence between new and prev rq's clocks task because of irq and 4778 * stolen time, we take an additional margin. 4779 * So, cutting off on the sleep time of 4780 * 2^63 / scale_load_down(NICE_0_LOAD) ~ 104 days 4781 * should be safe. 4782 */ 4783 if (entity_is_long_sleeper(se)) 4784 se->vruntime = vruntime; 4785 else 4786 se->vruntime = max_vruntime(se->vruntime, vruntime); 4787 } 4788 4789 static void check_enqueue_throttle(struct cfs_rq *cfs_rq); 4790 4791 static inline bool cfs_bandwidth_used(void); 4792 4793 /* 4794 * MIGRATION 4795 * 4796 * dequeue 4797 * update_curr() 4798 * update_min_vruntime() 4799 * vruntime -= min_vruntime 4800 * 4801 * enqueue 4802 * update_curr() 4803 * update_min_vruntime() 4804 * vruntime += min_vruntime 4805 * 4806 * this way the vruntime transition between RQs is done when both 4807 * min_vruntime are up-to-date. 4808 * 4809 * WAKEUP (remote) 4810 * 4811 * ->migrate_task_rq_fair() (p->state == TASK_WAKING) 4812 * vruntime -= min_vruntime 4813 * 4814 * enqueue 4815 * update_curr() 4816 * update_min_vruntime() 4817 * vruntime += min_vruntime 4818 * 4819 * this way we don't have the most up-to-date min_vruntime on the originating 4820 * CPU and an up-to-date min_vruntime on the destination CPU. 4821 */ 4822 4823 static void 4824 enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 4825 { 4826 bool renorm = !(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_MIGRATED); 4827 bool curr = cfs_rq->curr == se; 4828 4829 /* 4830 * If we're the current task, we must renormalise before calling 4831 * update_curr(). 4832 */ 4833 if (renorm && curr) 4834 se->vruntime += cfs_rq->min_vruntime; 4835 4836 update_curr(cfs_rq); 4837 4838 /* 4839 * Otherwise, renormalise after, such that we're placed at the current 4840 * moment in time, instead of some random moment in the past. Being 4841 * placed in the past could significantly boost this task to the 4842 * fairness detriment of existing tasks. 4843 */ 4844 if (renorm && !curr) 4845 se->vruntime += cfs_rq->min_vruntime; 4846 4847 /* 4848 * When enqueuing a sched_entity, we must: 4849 * - Update loads to have both entity and cfs_rq synced with now. 4850 * - For group_entity, update its runnable_weight to reflect the new 4851 * h_nr_running of its group cfs_rq. 4852 * - For group_entity, update its weight to reflect the new share of 4853 * its group cfs_rq 4854 * - Add its new weight to cfs_rq->load.weight 4855 */ 4856 update_load_avg(cfs_rq, se, UPDATE_TG | DO_ATTACH); 4857 se_update_runnable(se); 4858 update_cfs_group(se); 4859 account_entity_enqueue(cfs_rq, se); 4860 4861 if (flags & ENQUEUE_WAKEUP) 4862 place_entity(cfs_rq, se, 0); 4863 /* Entity has migrated, no longer consider this task hot */ 4864 if (flags & ENQUEUE_MIGRATED) 4865 se->exec_start = 0; 4866 4867 check_schedstat_required(); 4868 update_stats_enqueue_fair(cfs_rq, se, flags); 4869 check_spread(cfs_rq, se); 4870 if (!curr) 4871 __enqueue_entity(cfs_rq, se); 4872 se->on_rq = 1; 4873 4874 if (cfs_rq->nr_running == 1) { 4875 check_enqueue_throttle(cfs_rq); 4876 if (!throttled_hierarchy(cfs_rq)) 4877 list_add_leaf_cfs_rq(cfs_rq); 4878 } 4879 } 4880 4881 static void __clear_buddies_last(struct sched_entity *se) 4882 { 4883 for_each_sched_entity(se) { 4884 struct cfs_rq *cfs_rq = cfs_rq_of(se); 4885 if (cfs_rq->last != se) 4886 break; 4887 4888 cfs_rq->last = NULL; 4889 } 4890 } 4891 4892 static void __clear_buddies_next(struct sched_entity *se) 4893 { 4894 for_each_sched_entity(se) { 4895 struct cfs_rq *cfs_rq = cfs_rq_of(se); 4896 if (cfs_rq->next != se) 4897 break; 4898 4899 cfs_rq->next = NULL; 4900 } 4901 } 4902 4903 static void __clear_buddies_skip(struct sched_entity *se) 4904 { 4905 for_each_sched_entity(se) { 4906 struct cfs_rq *cfs_rq = cfs_rq_of(se); 4907 if (cfs_rq->skip != se) 4908 break; 4909 4910 cfs_rq->skip = NULL; 4911 } 4912 } 4913 4914 static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) 4915 { 4916 if (cfs_rq->last == se) 4917 __clear_buddies_last(se); 4918 4919 if (cfs_rq->next == se) 4920 __clear_buddies_next(se); 4921 4922 if (cfs_rq->skip == se) 4923 __clear_buddies_skip(se); 4924 } 4925 4926 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq); 4927 4928 static void 4929 dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 4930 { 4931 int action = UPDATE_TG; 4932 4933 if (entity_is_task(se) && task_on_rq_migrating(task_of(se))) 4934 action |= DO_DETACH; 4935 4936 /* 4937 * Update run-time statistics of the 'current'. 4938 */ 4939 update_curr(cfs_rq); 4940 4941 /* 4942 * When dequeuing a sched_entity, we must: 4943 * - Update loads to have both entity and cfs_rq synced with now. 4944 * - For group_entity, update its runnable_weight to reflect the new 4945 * h_nr_running of its group cfs_rq. 4946 * - Subtract its previous weight from cfs_rq->load.weight. 4947 * - For group entity, update its weight to reflect the new share 4948 * of its group cfs_rq. 4949 */ 4950 update_load_avg(cfs_rq, se, action); 4951 se_update_runnable(se); 4952 4953 update_stats_dequeue_fair(cfs_rq, se, flags); 4954 4955 clear_buddies(cfs_rq, se); 4956 4957 if (se != cfs_rq->curr) 4958 __dequeue_entity(cfs_rq, se); 4959 se->on_rq = 0; 4960 account_entity_dequeue(cfs_rq, se); 4961 4962 /* 4963 * Normalize after update_curr(); which will also have moved 4964 * min_vruntime if @se is the one holding it back. But before doing 4965 * update_min_vruntime() again, which will discount @se's position and 4966 * can move min_vruntime forward still more. 4967 */ 4968 if (!(flags & DEQUEUE_SLEEP)) 4969 se->vruntime -= cfs_rq->min_vruntime; 4970 4971 /* return excess runtime on last dequeue */ 4972 return_cfs_rq_runtime(cfs_rq); 4973 4974 update_cfs_group(se); 4975 4976 /* 4977 * Now advance min_vruntime if @se was the entity holding it back, 4978 * except when: DEQUEUE_SAVE && !DEQUEUE_MOVE, in this case we'll be 4979 * put back on, and if we advance min_vruntime, we'll be placed back 4980 * further than we started -- ie. we'll be penalized. 4981 */ 4982 if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) != DEQUEUE_SAVE) 4983 update_min_vruntime(cfs_rq); 4984 4985 if (cfs_rq->nr_running == 0) 4986 update_idle_cfs_rq_clock_pelt(cfs_rq); 4987 } 4988 4989 /* 4990 * Preempt the current task with a newly woken task if needed: 4991 */ 4992 static void 4993 check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr) 4994 { 4995 unsigned long ideal_runtime, delta_exec; 4996 struct sched_entity *se; 4997 s64 delta; 4998 4999 /* 5000 * When many tasks blow up the sched_period; it is possible that 5001 * sched_slice() reports unusually large results (when many tasks are 5002 * very light for example). Therefore impose a maximum. 5003 */ 5004 ideal_runtime = min_t(u64, sched_slice(cfs_rq, curr), sysctl_sched_latency); 5005 5006 delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime; 5007 if (delta_exec > ideal_runtime) { 5008 resched_curr(rq_of(cfs_rq)); 5009 /* 5010 * The current task ran long enough, ensure it doesn't get 5011 * re-elected due to buddy favours. 5012 */ 5013 clear_buddies(cfs_rq, curr); 5014 return; 5015 } 5016 5017 /* 5018 * Ensure that a task that missed wakeup preemption by a 5019 * narrow margin doesn't have to wait for a full slice. 5020 * This also mitigates buddy induced latencies under load. 5021 */ 5022 if (delta_exec < sysctl_sched_min_granularity) 5023 return; 5024 5025 se = __pick_first_entity(cfs_rq); 5026 delta = curr->vruntime - se->vruntime; 5027 5028 if (delta < 0) 5029 return; 5030 5031 if (delta > ideal_runtime) 5032 resched_curr(rq_of(cfs_rq)); 5033 } 5034 5035 static void 5036 set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 5037 { 5038 clear_buddies(cfs_rq, se); 5039 5040 /* 'current' is not kept within the tree. */ 5041 if (se->on_rq) { 5042 /* 5043 * Any task has to be enqueued before it get to execute on 5044 * a CPU. So account for the time it spent waiting on the 5045 * runqueue. 5046 */ 5047 update_stats_wait_end_fair(cfs_rq, se); 5048 __dequeue_entity(cfs_rq, se); 5049 update_load_avg(cfs_rq, se, UPDATE_TG); 5050 } 5051 5052 update_stats_curr_start(cfs_rq, se); 5053 cfs_rq->curr = se; 5054 5055 /* 5056 * Track our maximum slice length, if the CPU's load is at 5057 * least twice that of our own weight (i.e. dont track it 5058 * when there are only lesser-weight tasks around): 5059 */ 5060 if (schedstat_enabled() && 5061 rq_of(cfs_rq)->cfs.load.weight >= 2*se->load.weight) { 5062 struct sched_statistics *stats; 5063 5064 stats = __schedstats_from_se(se); 5065 __schedstat_set(stats->slice_max, 5066 max((u64)stats->slice_max, 5067 se->sum_exec_runtime - se->prev_sum_exec_runtime)); 5068 } 5069 5070 se->prev_sum_exec_runtime = se->sum_exec_runtime; 5071 } 5072 5073 static int 5074 wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se); 5075 5076 /* 5077 * Pick the next process, keeping these things in mind, in this order: 5078 * 1) keep things fair between processes/task groups 5079 * 2) pick the "next" process, since someone really wants that to run 5080 * 3) pick the "last" process, for cache locality 5081 * 4) do not run the "skip" process, if something else is available 5082 */ 5083 static struct sched_entity * 5084 pick_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *curr) 5085 { 5086 struct sched_entity *left = __pick_first_entity(cfs_rq); 5087 struct sched_entity *se; 5088 5089 /* 5090 * If curr is set we have to see if its left of the leftmost entity 5091 * still in the tree, provided there was anything in the tree at all. 5092 */ 5093 if (!left || (curr && entity_before(curr, left))) 5094 left = curr; 5095 5096 se = left; /* ideally we run the leftmost entity */ 5097 5098 /* 5099 * Avoid running the skip buddy, if running something else can 5100 * be done without getting too unfair. 5101 */ 5102 if (cfs_rq->skip && cfs_rq->skip == se) { 5103 struct sched_entity *second; 5104 5105 if (se == curr) { 5106 second = __pick_first_entity(cfs_rq); 5107 } else { 5108 second = __pick_next_entity(se); 5109 if (!second || (curr && entity_before(curr, second))) 5110 second = curr; 5111 } 5112 5113 if (second && wakeup_preempt_entity(second, left) < 1) 5114 se = second; 5115 } 5116 5117 if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, left) < 1) { 5118 /* 5119 * Someone really wants this to run. If it's not unfair, run it. 5120 */ 5121 se = cfs_rq->next; 5122 } else if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, left) < 1) { 5123 /* 5124 * Prefer last buddy, try to return the CPU to a preempted task. 5125 */ 5126 se = cfs_rq->last; 5127 } 5128 5129 return se; 5130 } 5131 5132 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq); 5133 5134 static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) 5135 { 5136 /* 5137 * If still on the runqueue then deactivate_task() 5138 * was not called and update_curr() has to be done: 5139 */ 5140 if (prev->on_rq) 5141 update_curr(cfs_rq); 5142 5143 /* throttle cfs_rqs exceeding runtime */ 5144 check_cfs_rq_runtime(cfs_rq); 5145 5146 check_spread(cfs_rq, prev); 5147 5148 if (prev->on_rq) { 5149 update_stats_wait_start_fair(cfs_rq, prev); 5150 /* Put 'current' back into the tree. */ 5151 __enqueue_entity(cfs_rq, prev); 5152 /* in !on_rq case, update occurred at dequeue */ 5153 update_load_avg(cfs_rq, prev, 0); 5154 } 5155 cfs_rq->curr = NULL; 5156 } 5157 5158 static void 5159 entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) 5160 { 5161 /* 5162 * Update run-time statistics of the 'current'. 5163 */ 5164 update_curr(cfs_rq); 5165 5166 /* 5167 * Ensure that runnable average is periodically updated. 5168 */ 5169 update_load_avg(cfs_rq, curr, UPDATE_TG); 5170 update_cfs_group(curr); 5171 5172 #ifdef CONFIG_SCHED_HRTICK 5173 /* 5174 * queued ticks are scheduled to match the slice, so don't bother 5175 * validating it and just reschedule. 5176 */ 5177 if (queued) { 5178 resched_curr(rq_of(cfs_rq)); 5179 return; 5180 } 5181 /* 5182 * don't let the period tick interfere with the hrtick preemption 5183 */ 5184 if (!sched_feat(DOUBLE_TICK) && 5185 hrtimer_active(&rq_of(cfs_rq)->hrtick_timer)) 5186 return; 5187 #endif 5188 5189 if (cfs_rq->nr_running > 1) 5190 check_preempt_tick(cfs_rq, curr); 5191 } 5192 5193 5194 /************************************************** 5195 * CFS bandwidth control machinery 5196 */ 5197 5198 #ifdef CONFIG_CFS_BANDWIDTH 5199 5200 #ifdef CONFIG_JUMP_LABEL 5201 static struct static_key __cfs_bandwidth_used; 5202 5203 static inline bool cfs_bandwidth_used(void) 5204 { 5205 return static_key_false(&__cfs_bandwidth_used); 5206 } 5207 5208 void cfs_bandwidth_usage_inc(void) 5209 { 5210 static_key_slow_inc_cpuslocked(&__cfs_bandwidth_used); 5211 } 5212 5213 void cfs_bandwidth_usage_dec(void) 5214 { 5215 static_key_slow_dec_cpuslocked(&__cfs_bandwidth_used); 5216 } 5217 #else /* CONFIG_JUMP_LABEL */ 5218 static bool cfs_bandwidth_used(void) 5219 { 5220 return true; 5221 } 5222 5223 void cfs_bandwidth_usage_inc(void) {} 5224 void cfs_bandwidth_usage_dec(void) {} 5225 #endif /* CONFIG_JUMP_LABEL */ 5226 5227 /* 5228 * default period for cfs group bandwidth. 5229 * default: 0.1s, units: nanoseconds 5230 */ 5231 static inline u64 default_cfs_period(void) 5232 { 5233 return 100000000ULL; 5234 } 5235 5236 static inline u64 sched_cfs_bandwidth_slice(void) 5237 { 5238 return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC; 5239 } 5240 5241 /* 5242 * Replenish runtime according to assigned quota. We use sched_clock_cpu 5243 * directly instead of rq->clock to avoid adding additional synchronization 5244 * around rq->lock. 5245 * 5246 * requires cfs_b->lock 5247 */ 5248 void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b) 5249 { 5250 s64 runtime; 5251 5252 if (unlikely(cfs_b->quota == RUNTIME_INF)) 5253 return; 5254 5255 cfs_b->runtime += cfs_b->quota; 5256 runtime = cfs_b->runtime_snap - cfs_b->runtime; 5257 if (runtime > 0) { 5258 cfs_b->burst_time += runtime; 5259 cfs_b->nr_burst++; 5260 } 5261 5262 cfs_b->runtime = min(cfs_b->runtime, cfs_b->quota + cfs_b->burst); 5263 cfs_b->runtime_snap = cfs_b->runtime; 5264 } 5265 5266 static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) 5267 { 5268 return &tg->cfs_bandwidth; 5269 } 5270 5271 /* returns 0 on failure to allocate runtime */ 5272 static int __assign_cfs_rq_runtime(struct cfs_bandwidth *cfs_b, 5273 struct cfs_rq *cfs_rq, u64 target_runtime) 5274 { 5275 u64 min_amount, amount = 0; 5276 5277 lockdep_assert_held(&cfs_b->lock); 5278 5279 /* note: this is a positive sum as runtime_remaining <= 0 */ 5280 min_amount = target_runtime - cfs_rq->runtime_remaining; 5281 5282 if (cfs_b->quota == RUNTIME_INF) 5283 amount = min_amount; 5284 else { 5285 start_cfs_bandwidth(cfs_b); 5286 5287 if (cfs_b->runtime > 0) { 5288 amount = min(cfs_b->runtime, min_amount); 5289 cfs_b->runtime -= amount; 5290 cfs_b->idle = 0; 5291 } 5292 } 5293 5294 cfs_rq->runtime_remaining += amount; 5295 5296 return cfs_rq->runtime_remaining > 0; 5297 } 5298 5299 /* returns 0 on failure to allocate runtime */ 5300 static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq) 5301 { 5302 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 5303 int ret; 5304 5305 raw_spin_lock(&cfs_b->lock); 5306 ret = __assign_cfs_rq_runtime(cfs_b, cfs_rq, sched_cfs_bandwidth_slice()); 5307 raw_spin_unlock(&cfs_b->lock); 5308 5309 return ret; 5310 } 5311 5312 static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) 5313 { 5314 /* dock delta_exec before expiring quota (as it could span periods) */ 5315 cfs_rq->runtime_remaining -= delta_exec; 5316 5317 if (likely(cfs_rq->runtime_remaining > 0)) 5318 return; 5319 5320 if (cfs_rq->throttled) 5321 return; 5322 /* 5323 * if we're unable to extend our runtime we resched so that the active 5324 * hierarchy can be throttled 5325 */ 5326 if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr)) 5327 resched_curr(rq_of(cfs_rq)); 5328 } 5329 5330 static __always_inline 5331 void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) 5332 { 5333 if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled) 5334 return; 5335 5336 __account_cfs_rq_runtime(cfs_rq, delta_exec); 5337 } 5338 5339 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) 5340 { 5341 return cfs_bandwidth_used() && cfs_rq->throttled; 5342 } 5343 5344 /* check whether cfs_rq, or any parent, is throttled */ 5345 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) 5346 { 5347 return cfs_bandwidth_used() && cfs_rq->throttle_count; 5348 } 5349 5350 /* 5351 * Ensure that neither of the group entities corresponding to src_cpu or 5352 * dest_cpu are members of a throttled hierarchy when performing group 5353 * load-balance operations. 5354 */ 5355 static inline int throttled_lb_pair(struct task_group *tg, 5356 int src_cpu, int dest_cpu) 5357 { 5358 struct cfs_rq *src_cfs_rq, *dest_cfs_rq; 5359 5360 src_cfs_rq = tg->cfs_rq[src_cpu]; 5361 dest_cfs_rq = tg->cfs_rq[dest_cpu]; 5362 5363 return throttled_hierarchy(src_cfs_rq) || 5364 throttled_hierarchy(dest_cfs_rq); 5365 } 5366 5367 static int tg_unthrottle_up(struct task_group *tg, void *data) 5368 { 5369 struct rq *rq = data; 5370 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 5371 5372 cfs_rq->throttle_count--; 5373 if (!cfs_rq->throttle_count) { 5374 cfs_rq->throttled_clock_pelt_time += rq_clock_pelt(rq) - 5375 cfs_rq->throttled_clock_pelt; 5376 5377 /* Add cfs_rq with load or one or more already running entities to the list */ 5378 if (!cfs_rq_is_decayed(cfs_rq)) 5379 list_add_leaf_cfs_rq(cfs_rq); 5380 } 5381 5382 return 0; 5383 } 5384 5385 static int tg_throttle_down(struct task_group *tg, void *data) 5386 { 5387 struct rq *rq = data; 5388 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 5389 5390 /* group is entering throttled state, stop time */ 5391 if (!cfs_rq->throttle_count) { 5392 cfs_rq->throttled_clock_pelt = rq_clock_pelt(rq); 5393 list_del_leaf_cfs_rq(cfs_rq); 5394 } 5395 cfs_rq->throttle_count++; 5396 5397 return 0; 5398 } 5399 5400 static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) 5401 { 5402 struct rq *rq = rq_of(cfs_rq); 5403 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 5404 struct sched_entity *se; 5405 long task_delta, idle_task_delta, dequeue = 1; 5406 5407 raw_spin_lock(&cfs_b->lock); 5408 /* This will start the period timer if necessary */ 5409 if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, 1)) { 5410 /* 5411 * We have raced with bandwidth becoming available, and if we 5412 * actually throttled the timer might not unthrottle us for an 5413 * entire period. We additionally needed to make sure that any 5414 * subsequent check_cfs_rq_runtime calls agree not to throttle 5415 * us, as we may commit to do cfs put_prev+pick_next, so we ask 5416 * for 1ns of runtime rather than just check cfs_b. 5417 */ 5418 dequeue = 0; 5419 } else { 5420 list_add_tail_rcu(&cfs_rq->throttled_list, 5421 &cfs_b->throttled_cfs_rq); 5422 } 5423 raw_spin_unlock(&cfs_b->lock); 5424 5425 if (!dequeue) 5426 return false; /* Throttle no longer required. */ 5427 5428 se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))]; 5429 5430 /* freeze hierarchy runnable averages while throttled */ 5431 rcu_read_lock(); 5432 walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq); 5433 rcu_read_unlock(); 5434 5435 task_delta = cfs_rq->h_nr_running; 5436 idle_task_delta = cfs_rq->idle_h_nr_running; 5437 for_each_sched_entity(se) { 5438 struct cfs_rq *qcfs_rq = cfs_rq_of(se); 5439 /* throttled entity or throttle-on-deactivate */ 5440 if (!se->on_rq) 5441 goto done; 5442 5443 dequeue_entity(qcfs_rq, se, DEQUEUE_SLEEP); 5444 5445 if (cfs_rq_is_idle(group_cfs_rq(se))) 5446 idle_task_delta = cfs_rq->h_nr_running; 5447 5448 qcfs_rq->h_nr_running -= task_delta; 5449 qcfs_rq->idle_h_nr_running -= idle_task_delta; 5450 5451 if (qcfs_rq->load.weight) { 5452 /* Avoid re-evaluating load for this entity: */ 5453 se = parent_entity(se); 5454 break; 5455 } 5456 } 5457 5458 for_each_sched_entity(se) { 5459 struct cfs_rq *qcfs_rq = cfs_rq_of(se); 5460 /* throttled entity or throttle-on-deactivate */ 5461 if (!se->on_rq) 5462 goto done; 5463 5464 update_load_avg(qcfs_rq, se, 0); 5465 se_update_runnable(se); 5466 5467 if (cfs_rq_is_idle(group_cfs_rq(se))) 5468 idle_task_delta = cfs_rq->h_nr_running; 5469 5470 qcfs_rq->h_nr_running -= task_delta; 5471 qcfs_rq->idle_h_nr_running -= idle_task_delta; 5472 } 5473 5474 /* At this point se is NULL and we are at root level*/ 5475 sub_nr_running(rq, task_delta); 5476 5477 done: 5478 /* 5479 * Note: distribution will already see us throttled via the 5480 * throttled-list. rq->lock protects completion. 5481 */ 5482 cfs_rq->throttled = 1; 5483 cfs_rq->throttled_clock = rq_clock(rq); 5484 return true; 5485 } 5486 5487 void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) 5488 { 5489 struct rq *rq = rq_of(cfs_rq); 5490 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 5491 struct sched_entity *se; 5492 long task_delta, idle_task_delta; 5493 5494 se = cfs_rq->tg->se[cpu_of(rq)]; 5495 5496 cfs_rq->throttled = 0; 5497 5498 update_rq_clock(rq); 5499 5500 raw_spin_lock(&cfs_b->lock); 5501 cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock; 5502 list_del_rcu(&cfs_rq->throttled_list); 5503 raw_spin_unlock(&cfs_b->lock); 5504 5505 /* update hierarchical throttle state */ 5506 walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq); 5507 5508 if (!cfs_rq->load.weight) { 5509 if (!cfs_rq->on_list) 5510 return; 5511 /* 5512 * Nothing to run but something to decay (on_list)? 5513 * Complete the branch. 5514 */ 5515 for_each_sched_entity(se) { 5516 if (list_add_leaf_cfs_rq(cfs_rq_of(se))) 5517 break; 5518 } 5519 goto unthrottle_throttle; 5520 } 5521 5522 task_delta = cfs_rq->h_nr_running; 5523 idle_task_delta = cfs_rq->idle_h_nr_running; 5524 for_each_sched_entity(se) { 5525 struct cfs_rq *qcfs_rq = cfs_rq_of(se); 5526 5527 if (se->on_rq) 5528 break; 5529 enqueue_entity(qcfs_rq, se, ENQUEUE_WAKEUP); 5530 5531 if (cfs_rq_is_idle(group_cfs_rq(se))) 5532 idle_task_delta = cfs_rq->h_nr_running; 5533 5534 qcfs_rq->h_nr_running += task_delta; 5535 qcfs_rq->idle_h_nr_running += idle_task_delta; 5536 5537 /* end evaluation on encountering a throttled cfs_rq */ 5538 if (cfs_rq_throttled(qcfs_rq)) 5539 goto unthrottle_throttle; 5540 } 5541 5542 for_each_sched_entity(se) { 5543 struct cfs_rq *qcfs_rq = cfs_rq_of(se); 5544 5545 update_load_avg(qcfs_rq, se, UPDATE_TG); 5546 se_update_runnable(se); 5547 5548 if (cfs_rq_is_idle(group_cfs_rq(se))) 5549 idle_task_delta = cfs_rq->h_nr_running; 5550 5551 qcfs_rq->h_nr_running += task_delta; 5552 qcfs_rq->idle_h_nr_running += idle_task_delta; 5553 5554 /* end evaluation on encountering a throttled cfs_rq */ 5555 if (cfs_rq_throttled(qcfs_rq)) 5556 goto unthrottle_throttle; 5557 } 5558 5559 /* At this point se is NULL and we are at root level*/ 5560 add_nr_running(rq, task_delta); 5561 5562 unthrottle_throttle: 5563 assert_list_leaf_cfs_rq(rq); 5564 5565 /* Determine whether we need to wake up potentially idle CPU: */ 5566 if (rq->curr == rq->idle && rq->cfs.nr_running) 5567 resched_curr(rq); 5568 } 5569 5570 #ifdef CONFIG_SMP 5571 static void __cfsb_csd_unthrottle(void *arg) 5572 { 5573 struct cfs_rq *cursor, *tmp; 5574 struct rq *rq = arg; 5575 struct rq_flags rf; 5576 5577 rq_lock(rq, &rf); 5578 5579 /* 5580 * Iterating over the list can trigger several call to 5581 * update_rq_clock() in unthrottle_cfs_rq(). 5582 * Do it once and skip the potential next ones. 5583 */ 5584 update_rq_clock(rq); 5585 rq_clock_start_loop_update(rq); 5586 5587 /* 5588 * Since we hold rq lock we're safe from concurrent manipulation of 5589 * the CSD list. However, this RCU critical section annotates the 5590 * fact that we pair with sched_free_group_rcu(), so that we cannot 5591 * race with group being freed in the window between removing it 5592 * from the list and advancing to the next entry in the list. 5593 */ 5594 rcu_read_lock(); 5595 5596 list_for_each_entry_safe(cursor, tmp, &rq->cfsb_csd_list, 5597 throttled_csd_list) { 5598 list_del_init(&cursor->throttled_csd_list); 5599 5600 if (cfs_rq_throttled(cursor)) 5601 unthrottle_cfs_rq(cursor); 5602 } 5603 5604 rcu_read_unlock(); 5605 5606 rq_clock_stop_loop_update(rq); 5607 rq_unlock(rq, &rf); 5608 } 5609 5610 static inline void __unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq) 5611 { 5612 struct rq *rq = rq_of(cfs_rq); 5613 bool first; 5614 5615 if (rq == this_rq()) { 5616 unthrottle_cfs_rq(cfs_rq); 5617 return; 5618 } 5619 5620 /* Already enqueued */ 5621 if (SCHED_WARN_ON(!list_empty(&cfs_rq->throttled_csd_list))) 5622 return; 5623 5624 first = list_empty(&rq->cfsb_csd_list); 5625 list_add_tail(&cfs_rq->throttled_csd_list, &rq->cfsb_csd_list); 5626 if (first) 5627 smp_call_function_single_async(cpu_of(rq), &rq->cfsb_csd); 5628 } 5629 #else 5630 static inline void __unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq) 5631 { 5632 unthrottle_cfs_rq(cfs_rq); 5633 } 5634 #endif 5635 5636 static void unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq) 5637 { 5638 lockdep_assert_rq_held(rq_of(cfs_rq)); 5639 5640 if (SCHED_WARN_ON(!cfs_rq_throttled(cfs_rq) || 5641 cfs_rq->runtime_remaining <= 0)) 5642 return; 5643 5644 __unthrottle_cfs_rq_async(cfs_rq); 5645 } 5646 5647 static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b) 5648 { 5649 struct cfs_rq *local_unthrottle = NULL; 5650 int this_cpu = smp_processor_id(); 5651 u64 runtime, remaining = 1; 5652 bool throttled = false; 5653 struct cfs_rq *cfs_rq; 5654 struct rq_flags rf; 5655 struct rq *rq; 5656 5657 rcu_read_lock(); 5658 list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq, 5659 throttled_list) { 5660 rq = rq_of(cfs_rq); 5661 5662 if (!remaining) { 5663 throttled = true; 5664 break; 5665 } 5666 5667 rq_lock_irqsave(rq, &rf); 5668 if (!cfs_rq_throttled(cfs_rq)) 5669 goto next; 5670 5671 #ifdef CONFIG_SMP 5672 /* Already queued for async unthrottle */ 5673 if (!list_empty(&cfs_rq->throttled_csd_list)) 5674 goto next; 5675 #endif 5676 5677 /* By the above checks, this should never be true */ 5678 SCHED_WARN_ON(cfs_rq->runtime_remaining > 0); 5679 5680 raw_spin_lock(&cfs_b->lock); 5681 runtime = -cfs_rq->runtime_remaining + 1; 5682 if (runtime > cfs_b->runtime) 5683 runtime = cfs_b->runtime; 5684 cfs_b->runtime -= runtime; 5685 remaining = cfs_b->runtime; 5686 raw_spin_unlock(&cfs_b->lock); 5687 5688 cfs_rq->runtime_remaining += runtime; 5689 5690 /* we check whether we're throttled above */ 5691 if (cfs_rq->runtime_remaining > 0) { 5692 if (cpu_of(rq) != this_cpu || 5693 SCHED_WARN_ON(local_unthrottle)) 5694 unthrottle_cfs_rq_async(cfs_rq); 5695 else 5696 local_unthrottle = cfs_rq; 5697 } else { 5698 throttled = true; 5699 } 5700 5701 next: 5702 rq_unlock_irqrestore(rq, &rf); 5703 } 5704 rcu_read_unlock(); 5705 5706 if (local_unthrottle) { 5707 rq = cpu_rq(this_cpu); 5708 rq_lock_irqsave(rq, &rf); 5709 if (cfs_rq_throttled(local_unthrottle)) 5710 unthrottle_cfs_rq(local_unthrottle); 5711 rq_unlock_irqrestore(rq, &rf); 5712 } 5713 5714 return throttled; 5715 } 5716 5717 /* 5718 * Responsible for refilling a task_group's bandwidth and unthrottling its 5719 * cfs_rqs as appropriate. If there has been no activity within the last 5720 * period the timer is deactivated until scheduling resumes; cfs_b->idle is 5721 * used to track this state. 5722 */ 5723 static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, unsigned long flags) 5724 { 5725 int throttled; 5726 5727 /* no need to continue the timer with no bandwidth constraint */ 5728 if (cfs_b->quota == RUNTIME_INF) 5729 goto out_deactivate; 5730 5731 throttled = !list_empty(&cfs_b->throttled_cfs_rq); 5732 cfs_b->nr_periods += overrun; 5733 5734 /* Refill extra burst quota even if cfs_b->idle */ 5735 __refill_cfs_bandwidth_runtime(cfs_b); 5736 5737 /* 5738 * idle depends on !throttled (for the case of a large deficit), and if 5739 * we're going inactive then everything else can be deferred 5740 */ 5741 if (cfs_b->idle && !throttled) 5742 goto out_deactivate; 5743 5744 if (!throttled) { 5745 /* mark as potentially idle for the upcoming period */ 5746 cfs_b->idle = 1; 5747 return 0; 5748 } 5749 5750 /* account preceding periods in which throttling occurred */ 5751 cfs_b->nr_throttled += overrun; 5752 5753 /* 5754 * This check is repeated as we release cfs_b->lock while we unthrottle. 5755 */ 5756 while (throttled && cfs_b->runtime > 0) { 5757 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 5758 /* we can't nest cfs_b->lock while distributing bandwidth */ 5759 throttled = distribute_cfs_runtime(cfs_b); 5760 raw_spin_lock_irqsave(&cfs_b->lock, flags); 5761 } 5762 5763 /* 5764 * While we are ensured activity in the period following an 5765 * unthrottle, this also covers the case in which the new bandwidth is 5766 * insufficient to cover the existing bandwidth deficit. (Forcing the 5767 * timer to remain active while there are any throttled entities.) 5768 */ 5769 cfs_b->idle = 0; 5770 5771 return 0; 5772 5773 out_deactivate: 5774 return 1; 5775 } 5776 5777 /* a cfs_rq won't donate quota below this amount */ 5778 static const u64 min_cfs_rq_runtime = 1 * NSEC_PER_MSEC; 5779 /* minimum remaining period time to redistribute slack quota */ 5780 static const u64 min_bandwidth_expiration = 2 * NSEC_PER_MSEC; 5781 /* how long we wait to gather additional slack before distributing */ 5782 static const u64 cfs_bandwidth_slack_period = 5 * NSEC_PER_MSEC; 5783 5784 /* 5785 * Are we near the end of the current quota period? 5786 * 5787 * Requires cfs_b->lock for hrtimer_expires_remaining to be safe against the 5788 * hrtimer base being cleared by hrtimer_start. In the case of 5789 * migrate_hrtimers, base is never cleared, so we are fine. 5790 */ 5791 static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire) 5792 { 5793 struct hrtimer *refresh_timer = &cfs_b->period_timer; 5794 s64 remaining; 5795 5796 /* if the call-back is running a quota refresh is already occurring */ 5797 if (hrtimer_callback_running(refresh_timer)) 5798 return 1; 5799 5800 /* is a quota refresh about to occur? */ 5801 remaining = ktime_to_ns(hrtimer_expires_remaining(refresh_timer)); 5802 if (remaining < (s64)min_expire) 5803 return 1; 5804 5805 return 0; 5806 } 5807 5808 static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b) 5809 { 5810 u64 min_left = cfs_bandwidth_slack_period + min_bandwidth_expiration; 5811 5812 /* if there's a quota refresh soon don't bother with slack */ 5813 if (runtime_refresh_within(cfs_b, min_left)) 5814 return; 5815 5816 /* don't push forwards an existing deferred unthrottle */ 5817 if (cfs_b->slack_started) 5818 return; 5819 cfs_b->slack_started = true; 5820 5821 hrtimer_start(&cfs_b->slack_timer, 5822 ns_to_ktime(cfs_bandwidth_slack_period), 5823 HRTIMER_MODE_REL); 5824 } 5825 5826 /* we know any runtime found here is valid as update_curr() precedes return */ 5827 static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq) 5828 { 5829 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 5830 s64 slack_runtime = cfs_rq->runtime_remaining - min_cfs_rq_runtime; 5831 5832 if (slack_runtime <= 0) 5833 return; 5834 5835 raw_spin_lock(&cfs_b->lock); 5836 if (cfs_b->quota != RUNTIME_INF) { 5837 cfs_b->runtime += slack_runtime; 5838 5839 /* we are under rq->lock, defer unthrottling using a timer */ 5840 if (cfs_b->runtime > sched_cfs_bandwidth_slice() && 5841 !list_empty(&cfs_b->throttled_cfs_rq)) 5842 start_cfs_slack_bandwidth(cfs_b); 5843 } 5844 raw_spin_unlock(&cfs_b->lock); 5845 5846 /* even if it's not valid for return we don't want to try again */ 5847 cfs_rq->runtime_remaining -= slack_runtime; 5848 } 5849 5850 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) 5851 { 5852 if (!cfs_bandwidth_used()) 5853 return; 5854 5855 if (!cfs_rq->runtime_enabled || cfs_rq->nr_running) 5856 return; 5857 5858 __return_cfs_rq_runtime(cfs_rq); 5859 } 5860 5861 /* 5862 * This is done with a timer (instead of inline with bandwidth return) since 5863 * it's necessary to juggle rq->locks to unthrottle their respective cfs_rqs. 5864 */ 5865 static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) 5866 { 5867 u64 runtime = 0, slice = sched_cfs_bandwidth_slice(); 5868 unsigned long flags; 5869 5870 /* confirm we're still not at a refresh boundary */ 5871 raw_spin_lock_irqsave(&cfs_b->lock, flags); 5872 cfs_b->slack_started = false; 5873 5874 if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) { 5875 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 5876 return; 5877 } 5878 5879 if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice) 5880 runtime = cfs_b->runtime; 5881 5882 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 5883 5884 if (!runtime) 5885 return; 5886 5887 distribute_cfs_runtime(cfs_b); 5888 } 5889 5890 /* 5891 * When a group wakes up we want to make sure that its quota is not already 5892 * expired/exceeded, otherwise it may be allowed to steal additional ticks of 5893 * runtime as update_curr() throttling can not trigger until it's on-rq. 5894 */ 5895 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) 5896 { 5897 if (!cfs_bandwidth_used()) 5898 return; 5899 5900 /* an active group must be handled by the update_curr()->put() path */ 5901 if (!cfs_rq->runtime_enabled || cfs_rq->curr) 5902 return; 5903 5904 /* ensure the group is not already throttled */ 5905 if (cfs_rq_throttled(cfs_rq)) 5906 return; 5907 5908 /* update runtime allocation */ 5909 account_cfs_rq_runtime(cfs_rq, 0); 5910 if (cfs_rq->runtime_remaining <= 0) 5911 throttle_cfs_rq(cfs_rq); 5912 } 5913 5914 static void sync_throttle(struct task_group *tg, int cpu) 5915 { 5916 struct cfs_rq *pcfs_rq, *cfs_rq; 5917 5918 if (!cfs_bandwidth_used()) 5919 return; 5920 5921 if (!tg->parent) 5922 return; 5923 5924 cfs_rq = tg->cfs_rq[cpu]; 5925 pcfs_rq = tg->parent->cfs_rq[cpu]; 5926 5927 cfs_rq->throttle_count = pcfs_rq->throttle_count; 5928 cfs_rq->throttled_clock_pelt = rq_clock_pelt(cpu_rq(cpu)); 5929 } 5930 5931 /* conditionally throttle active cfs_rq's from put_prev_entity() */ 5932 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) 5933 { 5934 if (!cfs_bandwidth_used()) 5935 return false; 5936 5937 if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0)) 5938 return false; 5939 5940 /* 5941 * it's possible for a throttled entity to be forced into a running 5942 * state (e.g. set_curr_task), in this case we're finished. 5943 */ 5944 if (cfs_rq_throttled(cfs_rq)) 5945 return true; 5946 5947 return throttle_cfs_rq(cfs_rq); 5948 } 5949 5950 static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer) 5951 { 5952 struct cfs_bandwidth *cfs_b = 5953 container_of(timer, struct cfs_bandwidth, slack_timer); 5954 5955 do_sched_cfs_slack_timer(cfs_b); 5956 5957 return HRTIMER_NORESTART; 5958 } 5959 5960 extern const u64 max_cfs_quota_period; 5961 5962 static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer) 5963 { 5964 struct cfs_bandwidth *cfs_b = 5965 container_of(timer, struct cfs_bandwidth, period_timer); 5966 unsigned long flags; 5967 int overrun; 5968 int idle = 0; 5969 int count = 0; 5970 5971 raw_spin_lock_irqsave(&cfs_b->lock, flags); 5972 for (;;) { 5973 overrun = hrtimer_forward_now(timer, cfs_b->period); 5974 if (!overrun) 5975 break; 5976 5977 idle = do_sched_cfs_period_timer(cfs_b, overrun, flags); 5978 5979 if (++count > 3) { 5980 u64 new, old = ktime_to_ns(cfs_b->period); 5981 5982 /* 5983 * Grow period by a factor of 2 to avoid losing precision. 5984 * Precision loss in the quota/period ratio can cause __cfs_schedulable 5985 * to fail. 5986 */ 5987 new = old * 2; 5988 if (new < max_cfs_quota_period) { 5989 cfs_b->period = ns_to_ktime(new); 5990 cfs_b->quota *= 2; 5991 cfs_b->burst *= 2; 5992 5993 pr_warn_ratelimited( 5994 "cfs_period_timer[cpu%d]: period too short, scaling up (new cfs_period_us = %lld, cfs_quota_us = %lld)\n", 5995 smp_processor_id(), 5996 div_u64(new, NSEC_PER_USEC), 5997 div_u64(cfs_b->quota, NSEC_PER_USEC)); 5998 } else { 5999 pr_warn_ratelimited( 6000 "cfs_period_timer[cpu%d]: period too short, but cannot scale up without losing precision (cfs_period_us = %lld, cfs_quota_us = %lld)\n", 6001 smp_processor_id(), 6002 div_u64(old, NSEC_PER_USEC), 6003 div_u64(cfs_b->quota, NSEC_PER_USEC)); 6004 } 6005 6006 /* reset count so we don't come right back in here */ 6007 count = 0; 6008 } 6009 } 6010 if (idle) 6011 cfs_b->period_active = 0; 6012 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 6013 6014 return idle ? HRTIMER_NORESTART : HRTIMER_RESTART; 6015 } 6016 6017 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) 6018 { 6019 raw_spin_lock_init(&cfs_b->lock); 6020 cfs_b->runtime = 0; 6021 cfs_b->quota = RUNTIME_INF; 6022 cfs_b->period = ns_to_ktime(default_cfs_period()); 6023 cfs_b->burst = 0; 6024 6025 INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq); 6026 hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED); 6027 cfs_b->period_timer.function = sched_cfs_period_timer; 6028 6029 /* Add a random offset so that timers interleave */ 6030 hrtimer_set_expires(&cfs_b->period_timer, 6031 get_random_u32_below(cfs_b->period)); 6032 hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 6033 cfs_b->slack_timer.function = sched_cfs_slack_timer; 6034 cfs_b->slack_started = false; 6035 } 6036 6037 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) 6038 { 6039 cfs_rq->runtime_enabled = 0; 6040 INIT_LIST_HEAD(&cfs_rq->throttled_list); 6041 #ifdef CONFIG_SMP 6042 INIT_LIST_HEAD(&cfs_rq->throttled_csd_list); 6043 #endif 6044 } 6045 6046 void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b) 6047 { 6048 lockdep_assert_held(&cfs_b->lock); 6049 6050 if (cfs_b->period_active) 6051 return; 6052 6053 cfs_b->period_active = 1; 6054 hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period); 6055 hrtimer_start_expires(&cfs_b->period_timer, HRTIMER_MODE_ABS_PINNED); 6056 } 6057 6058 static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) 6059 { 6060 int __maybe_unused i; 6061 6062 /* init_cfs_bandwidth() was not called */ 6063 if (!cfs_b->throttled_cfs_rq.next) 6064 return; 6065 6066 hrtimer_cancel(&cfs_b->period_timer); 6067 hrtimer_cancel(&cfs_b->slack_timer); 6068 6069 /* 6070 * It is possible that we still have some cfs_rq's pending on a CSD 6071 * list, though this race is very rare. In order for this to occur, we 6072 * must have raced with the last task leaving the group while there 6073 * exist throttled cfs_rq(s), and the period_timer must have queued the 6074 * CSD item but the remote cpu has not yet processed it. To handle this, 6075 * we can simply flush all pending CSD work inline here. We're 6076 * guaranteed at this point that no additional cfs_rq of this group can 6077 * join a CSD list. 6078 */ 6079 #ifdef CONFIG_SMP 6080 for_each_possible_cpu(i) { 6081 struct rq *rq = cpu_rq(i); 6082 unsigned long flags; 6083 6084 if (list_empty(&rq->cfsb_csd_list)) 6085 continue; 6086 6087 local_irq_save(flags); 6088 __cfsb_csd_unthrottle(rq); 6089 local_irq_restore(flags); 6090 } 6091 #endif 6092 } 6093 6094 /* 6095 * Both these CPU hotplug callbacks race against unregister_fair_sched_group() 6096 * 6097 * The race is harmless, since modifying bandwidth settings of unhooked group 6098 * bits doesn't do much. 6099 */ 6100 6101 /* cpu online callback */ 6102 static void __maybe_unused update_runtime_enabled(struct rq *rq) 6103 { 6104 struct task_group *tg; 6105 6106 lockdep_assert_rq_held(rq); 6107 6108 rcu_read_lock(); 6109 list_for_each_entry_rcu(tg, &task_groups, list) { 6110 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 6111 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 6112 6113 raw_spin_lock(&cfs_b->lock); 6114 cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF; 6115 raw_spin_unlock(&cfs_b->lock); 6116 } 6117 rcu_read_unlock(); 6118 } 6119 6120 /* cpu offline callback */ 6121 static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq) 6122 { 6123 struct task_group *tg; 6124 6125 lockdep_assert_rq_held(rq); 6126 6127 /* 6128 * The rq clock has already been updated in the 6129 * set_rq_offline(), so we should skip updating 6130 * the rq clock again in unthrottle_cfs_rq(). 6131 */ 6132 rq_clock_start_loop_update(rq); 6133 6134 rcu_read_lock(); 6135 list_for_each_entry_rcu(tg, &task_groups, list) { 6136 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 6137 6138 if (!cfs_rq->runtime_enabled) 6139 continue; 6140 6141 /* 6142 * clock_task is not advancing so we just need to make sure 6143 * there's some valid quota amount 6144 */ 6145 cfs_rq->runtime_remaining = 1; 6146 /* 6147 * Offline rq is schedulable till CPU is completely disabled 6148 * in take_cpu_down(), so we prevent new cfs throttling here. 6149 */ 6150 cfs_rq->runtime_enabled = 0; 6151 6152 if (cfs_rq_throttled(cfs_rq)) 6153 unthrottle_cfs_rq(cfs_rq); 6154 } 6155 rcu_read_unlock(); 6156 6157 rq_clock_stop_loop_update(rq); 6158 } 6159 6160 #else /* CONFIG_CFS_BANDWIDTH */ 6161 6162 static inline bool cfs_bandwidth_used(void) 6163 { 6164 return false; 6165 } 6166 6167 static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) {} 6168 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) { return false; } 6169 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {} 6170 static inline void sync_throttle(struct task_group *tg, int cpu) {} 6171 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} 6172 6173 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) 6174 { 6175 return 0; 6176 } 6177 6178 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) 6179 { 6180 return 0; 6181 } 6182 6183 static inline int throttled_lb_pair(struct task_group *tg, 6184 int src_cpu, int dest_cpu) 6185 { 6186 return 0; 6187 } 6188 6189 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {} 6190 6191 #ifdef CONFIG_FAIR_GROUP_SCHED 6192 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} 6193 #endif 6194 6195 static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) 6196 { 6197 return NULL; 6198 } 6199 static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {} 6200 static inline void update_runtime_enabled(struct rq *rq) {} 6201 static inline void unthrottle_offline_cfs_rqs(struct rq *rq) {} 6202 6203 #endif /* CONFIG_CFS_BANDWIDTH */ 6204 6205 /************************************************** 6206 * CFS operations on tasks: 6207 */ 6208 6209 #ifdef CONFIG_SCHED_HRTICK 6210 static void hrtick_start_fair(struct rq *rq, struct task_struct *p) 6211 { 6212 struct sched_entity *se = &p->se; 6213 struct cfs_rq *cfs_rq = cfs_rq_of(se); 6214 6215 SCHED_WARN_ON(task_rq(p) != rq); 6216 6217 if (rq->cfs.h_nr_running > 1) { 6218 u64 slice = sched_slice(cfs_rq, se); 6219 u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime; 6220 s64 delta = slice - ran; 6221 6222 if (delta < 0) { 6223 if (task_current(rq, p)) 6224 resched_curr(rq); 6225 return; 6226 } 6227 hrtick_start(rq, delta); 6228 } 6229 } 6230 6231 /* 6232 * called from enqueue/dequeue and updates the hrtick when the 6233 * current task is from our class and nr_running is low enough 6234 * to matter. 6235 */ 6236 static void hrtick_update(struct rq *rq) 6237 { 6238 struct task_struct *curr = rq->curr; 6239 6240 if (!hrtick_enabled_fair(rq) || curr->sched_class != &fair_sched_class) 6241 return; 6242 6243 if (cfs_rq_of(&curr->se)->nr_running < sched_nr_latency) 6244 hrtick_start_fair(rq, curr); 6245 } 6246 #else /* !CONFIG_SCHED_HRTICK */ 6247 static inline void 6248 hrtick_start_fair(struct rq *rq, struct task_struct *p) 6249 { 6250 } 6251 6252 static inline void hrtick_update(struct rq *rq) 6253 { 6254 } 6255 #endif 6256 6257 #ifdef CONFIG_SMP 6258 static inline bool cpu_overutilized(int cpu) 6259 { 6260 unsigned long rq_util_min = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MIN); 6261 unsigned long rq_util_max = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MAX); 6262 6263 /* Return true only if the utilization doesn't fit CPU's capacity */ 6264 return !util_fits_cpu(cpu_util_cfs(cpu), rq_util_min, rq_util_max, cpu); 6265 } 6266 6267 static inline void update_overutilized_status(struct rq *rq) 6268 { 6269 if (!READ_ONCE(rq->rd->overutilized) && cpu_overutilized(rq->cpu)) { 6270 WRITE_ONCE(rq->rd->overutilized, SG_OVERUTILIZED); 6271 trace_sched_overutilized_tp(rq->rd, SG_OVERUTILIZED); 6272 } 6273 } 6274 #else 6275 static inline void update_overutilized_status(struct rq *rq) { } 6276 #endif 6277 6278 /* Runqueue only has SCHED_IDLE tasks enqueued */ 6279 static int sched_idle_rq(struct rq *rq) 6280 { 6281 return unlikely(rq->nr_running == rq->cfs.idle_h_nr_running && 6282 rq->nr_running); 6283 } 6284 6285 /* 6286 * Returns true if cfs_rq only has SCHED_IDLE entities enqueued. Note the use 6287 * of idle_nr_running, which does not consider idle descendants of normal 6288 * entities. 6289 */ 6290 static bool sched_idle_cfs_rq(struct cfs_rq *cfs_rq) 6291 { 6292 return cfs_rq->nr_running && 6293 cfs_rq->nr_running == cfs_rq->idle_nr_running; 6294 } 6295 6296 #ifdef CONFIG_SMP 6297 static int sched_idle_cpu(int cpu) 6298 { 6299 return sched_idle_rq(cpu_rq(cpu)); 6300 } 6301 #endif 6302 6303 /* 6304 * The enqueue_task method is called before nr_running is 6305 * increased. Here we update the fair scheduling stats and 6306 * then put the task into the rbtree: 6307 */ 6308 static void 6309 enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) 6310 { 6311 struct cfs_rq *cfs_rq; 6312 struct sched_entity *se = &p->se; 6313 int idle_h_nr_running = task_has_idle_policy(p); 6314 int task_new = !(flags & ENQUEUE_WAKEUP); 6315 6316 /* 6317 * The code below (indirectly) updates schedutil which looks at 6318 * the cfs_rq utilization to select a frequency. 6319 * Let's add the task's estimated utilization to the cfs_rq's 6320 * estimated utilization, before we update schedutil. 6321 */ 6322 util_est_enqueue(&rq->cfs, p); 6323 6324 /* 6325 * If in_iowait is set, the code below may not trigger any cpufreq 6326 * utilization updates, so do it here explicitly with the IOWAIT flag 6327 * passed. 6328 */ 6329 if (p->in_iowait) 6330 cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT); 6331 6332 for_each_sched_entity(se) { 6333 if (se->on_rq) 6334 break; 6335 cfs_rq = cfs_rq_of(se); 6336 enqueue_entity(cfs_rq, se, flags); 6337 6338 cfs_rq->h_nr_running++; 6339 cfs_rq->idle_h_nr_running += idle_h_nr_running; 6340 6341 if (cfs_rq_is_idle(cfs_rq)) 6342 idle_h_nr_running = 1; 6343 6344 /* end evaluation on encountering a throttled cfs_rq */ 6345 if (cfs_rq_throttled(cfs_rq)) 6346 goto enqueue_throttle; 6347 6348 flags = ENQUEUE_WAKEUP; 6349 } 6350 6351 for_each_sched_entity(se) { 6352 cfs_rq = cfs_rq_of(se); 6353 6354 update_load_avg(cfs_rq, se, UPDATE_TG); 6355 se_update_runnable(se); 6356 update_cfs_group(se); 6357 6358 cfs_rq->h_nr_running++; 6359 cfs_rq->idle_h_nr_running += idle_h_nr_running; 6360 6361 if (cfs_rq_is_idle(cfs_rq)) 6362 idle_h_nr_running = 1; 6363 6364 /* end evaluation on encountering a throttled cfs_rq */ 6365 if (cfs_rq_throttled(cfs_rq)) 6366 goto enqueue_throttle; 6367 } 6368 6369 /* At this point se is NULL and we are at root level*/ 6370 add_nr_running(rq, 1); 6371 6372 /* 6373 * Since new tasks are assigned an initial util_avg equal to 6374 * half of the spare capacity of their CPU, tiny tasks have the 6375 * ability to cross the overutilized threshold, which will 6376 * result in the load balancer ruining all the task placement 6377 * done by EAS. As a way to mitigate that effect, do not account 6378 * for the first enqueue operation of new tasks during the 6379 * overutilized flag detection. 6380 * 6381 * A better way of solving this problem would be to wait for 6382 * the PELT signals of tasks to converge before taking them 6383 * into account, but that is not straightforward to implement, 6384 * and the following generally works well enough in practice. 6385 */ 6386 if (!task_new) 6387 update_overutilized_status(rq); 6388 6389 enqueue_throttle: 6390 assert_list_leaf_cfs_rq(rq); 6391 6392 hrtick_update(rq); 6393 } 6394 6395 static void set_next_buddy(struct sched_entity *se); 6396 6397 /* 6398 * The dequeue_task method is called before nr_running is 6399 * decreased. We remove the task from the rbtree and 6400 * update the fair scheduling stats: 6401 */ 6402 static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) 6403 { 6404 struct cfs_rq *cfs_rq; 6405 struct sched_entity *se = &p->se; 6406 int task_sleep = flags & DEQUEUE_SLEEP; 6407 int idle_h_nr_running = task_has_idle_policy(p); 6408 bool was_sched_idle = sched_idle_rq(rq); 6409 6410 util_est_dequeue(&rq->cfs, p); 6411 6412 for_each_sched_entity(se) { 6413 cfs_rq = cfs_rq_of(se); 6414 dequeue_entity(cfs_rq, se, flags); 6415 6416 cfs_rq->h_nr_running--; 6417 cfs_rq->idle_h_nr_running -= idle_h_nr_running; 6418 6419 if (cfs_rq_is_idle(cfs_rq)) 6420 idle_h_nr_running = 1; 6421 6422 /* end evaluation on encountering a throttled cfs_rq */ 6423 if (cfs_rq_throttled(cfs_rq)) 6424 goto dequeue_throttle; 6425 6426 /* Don't dequeue parent if it has other entities besides us */ 6427 if (cfs_rq->load.weight) { 6428 /* Avoid re-evaluating load for this entity: */ 6429 se = parent_entity(se); 6430 /* 6431 * Bias pick_next to pick a task from this cfs_rq, as 6432 * p is sleeping when it is within its sched_slice. 6433 */ 6434 if (task_sleep && se && !throttled_hierarchy(cfs_rq)) 6435 set_next_buddy(se); 6436 break; 6437 } 6438 flags |= DEQUEUE_SLEEP; 6439 } 6440 6441 for_each_sched_entity(se) { 6442 cfs_rq = cfs_rq_of(se); 6443 6444 update_load_avg(cfs_rq, se, UPDATE_TG); 6445 se_update_runnable(se); 6446 update_cfs_group(se); 6447 6448 cfs_rq->h_nr_running--; 6449 cfs_rq->idle_h_nr_running -= idle_h_nr_running; 6450 6451 if (cfs_rq_is_idle(cfs_rq)) 6452 idle_h_nr_running = 1; 6453 6454 /* end evaluation on encountering a throttled cfs_rq */ 6455 if (cfs_rq_throttled(cfs_rq)) 6456 goto dequeue_throttle; 6457 6458 } 6459 6460 /* At this point se is NULL and we are at root level*/ 6461 sub_nr_running(rq, 1); 6462 6463 /* balance early to pull high priority tasks */ 6464 if (unlikely(!was_sched_idle && sched_idle_rq(rq))) 6465 rq->next_balance = jiffies; 6466 6467 dequeue_throttle: 6468 util_est_update(&rq->cfs, p, task_sleep); 6469 hrtick_update(rq); 6470 } 6471 6472 #ifdef CONFIG_SMP 6473 6474 /* Working cpumask for: load_balance, load_balance_newidle. */ 6475 static DEFINE_PER_CPU(cpumask_var_t, load_balance_mask); 6476 static DEFINE_PER_CPU(cpumask_var_t, select_rq_mask); 6477 6478 #ifdef CONFIG_NO_HZ_COMMON 6479 6480 static struct { 6481 cpumask_var_t idle_cpus_mask; 6482 atomic_t nr_cpus; 6483 int has_blocked; /* Idle CPUS has blocked load */ 6484 int needs_update; /* Newly idle CPUs need their next_balance collated */ 6485 unsigned long next_balance; /* in jiffy units */ 6486 unsigned long next_blocked; /* Next update of blocked load in jiffies */ 6487 } nohz ____cacheline_aligned; 6488 6489 #endif /* CONFIG_NO_HZ_COMMON */ 6490 6491 static unsigned long cpu_load(struct rq *rq) 6492 { 6493 return cfs_rq_load_avg(&rq->cfs); 6494 } 6495 6496 /* 6497 * cpu_load_without - compute CPU load without any contributions from *p 6498 * @cpu: the CPU which load is requested 6499 * @p: the task which load should be discounted 6500 * 6501 * The load of a CPU is defined by the load of tasks currently enqueued on that 6502 * CPU as well as tasks which are currently sleeping after an execution on that 6503 * CPU. 6504 * 6505 * This method returns the load of the specified CPU by discounting the load of 6506 * the specified task, whenever the task is currently contributing to the CPU 6507 * load. 6508 */ 6509 static unsigned long cpu_load_without(struct rq *rq, struct task_struct *p) 6510 { 6511 struct cfs_rq *cfs_rq; 6512 unsigned int load; 6513 6514 /* Task has no contribution or is new */ 6515 if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 6516 return cpu_load(rq); 6517 6518 cfs_rq = &rq->cfs; 6519 load = READ_ONCE(cfs_rq->avg.load_avg); 6520 6521 /* Discount task's util from CPU's util */ 6522 lsub_positive(&load, task_h_load(p)); 6523 6524 return load; 6525 } 6526 6527 static unsigned long cpu_runnable(struct rq *rq) 6528 { 6529 return cfs_rq_runnable_avg(&rq->cfs); 6530 } 6531 6532 static unsigned long cpu_runnable_without(struct rq *rq, struct task_struct *p) 6533 { 6534 struct cfs_rq *cfs_rq; 6535 unsigned int runnable; 6536 6537 /* Task has no contribution or is new */ 6538 if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 6539 return cpu_runnable(rq); 6540 6541 cfs_rq = &rq->cfs; 6542 runnable = READ_ONCE(cfs_rq->avg.runnable_avg); 6543 6544 /* Discount task's runnable from CPU's runnable */ 6545 lsub_positive(&runnable, p->se.avg.runnable_avg); 6546 6547 return runnable; 6548 } 6549 6550 static unsigned long capacity_of(int cpu) 6551 { 6552 return cpu_rq(cpu)->cpu_capacity; 6553 } 6554 6555 static void record_wakee(struct task_struct *p) 6556 { 6557 /* 6558 * Only decay a single time; tasks that have less then 1 wakeup per 6559 * jiffy will not have built up many flips. 6560 */ 6561 if (time_after(jiffies, current->wakee_flip_decay_ts + HZ)) { 6562 current->wakee_flips >>= 1; 6563 current->wakee_flip_decay_ts = jiffies; 6564 } 6565 6566 if (current->last_wakee != p) { 6567 current->last_wakee = p; 6568 current->wakee_flips++; 6569 } 6570 } 6571 6572 /* 6573 * Detect M:N waker/wakee relationships via a switching-frequency heuristic. 6574 * 6575 * A waker of many should wake a different task than the one last awakened 6576 * at a frequency roughly N times higher than one of its wakees. 6577 * 6578 * In order to determine whether we should let the load spread vs consolidating 6579 * to shared cache, we look for a minimum 'flip' frequency of llc_size in one 6580 * partner, and a factor of lls_size higher frequency in the other. 6581 * 6582 * With both conditions met, we can be relatively sure that the relationship is 6583 * non-monogamous, with partner count exceeding socket size. 6584 * 6585 * Waker/wakee being client/server, worker/dispatcher, interrupt source or 6586 * whatever is irrelevant, spread criteria is apparent partner count exceeds 6587 * socket size. 6588 */ 6589 static int wake_wide(struct task_struct *p) 6590 { 6591 unsigned int master = current->wakee_flips; 6592 unsigned int slave = p->wakee_flips; 6593 int factor = __this_cpu_read(sd_llc_size); 6594 6595 if (master < slave) 6596 swap(master, slave); 6597 if (slave < factor || master < slave * factor) 6598 return 0; 6599 return 1; 6600 } 6601 6602 /* 6603 * The purpose of wake_affine() is to quickly determine on which CPU we can run 6604 * soonest. For the purpose of speed we only consider the waking and previous 6605 * CPU. 6606 * 6607 * wake_affine_idle() - only considers 'now', it check if the waking CPU is 6608 * cache-affine and is (or will be) idle. 6609 * 6610 * wake_affine_weight() - considers the weight to reflect the average 6611 * scheduling latency of the CPUs. This seems to work 6612 * for the overloaded case. 6613 */ 6614 static int 6615 wake_affine_idle(int this_cpu, int prev_cpu, int sync) 6616 { 6617 /* 6618 * If this_cpu is idle, it implies the wakeup is from interrupt 6619 * context. Only allow the move if cache is shared. Otherwise an 6620 * interrupt intensive workload could force all tasks onto one 6621 * node depending on the IO topology or IRQ affinity settings. 6622 * 6623 * If the prev_cpu is idle and cache affine then avoid a migration. 6624 * There is no guarantee that the cache hot data from an interrupt 6625 * is more important than cache hot data on the prev_cpu and from 6626 * a cpufreq perspective, it's better to have higher utilisation 6627 * on one CPU. 6628 */ 6629 if (available_idle_cpu(this_cpu) && cpus_share_cache(this_cpu, prev_cpu)) 6630 return available_idle_cpu(prev_cpu) ? prev_cpu : this_cpu; 6631 6632 if (sync && cpu_rq(this_cpu)->nr_running == 1) 6633 return this_cpu; 6634 6635 if (available_idle_cpu(prev_cpu)) 6636 return prev_cpu; 6637 6638 return nr_cpumask_bits; 6639 } 6640 6641 static int 6642 wake_affine_weight(struct sched_domain *sd, struct task_struct *p, 6643 int this_cpu, int prev_cpu, int sync) 6644 { 6645 s64 this_eff_load, prev_eff_load; 6646 unsigned long task_load; 6647 6648 this_eff_load = cpu_load(cpu_rq(this_cpu)); 6649 6650 if (sync) { 6651 unsigned long current_load = task_h_load(current); 6652 6653 if (current_load > this_eff_load) 6654 return this_cpu; 6655 6656 this_eff_load -= current_load; 6657 } 6658 6659 task_load = task_h_load(p); 6660 6661 this_eff_load += task_load; 6662 if (sched_feat(WA_BIAS)) 6663 this_eff_load *= 100; 6664 this_eff_load *= capacity_of(prev_cpu); 6665 6666 prev_eff_load = cpu_load(cpu_rq(prev_cpu)); 6667 prev_eff_load -= task_load; 6668 if (sched_feat(WA_BIAS)) 6669 prev_eff_load *= 100 + (sd->imbalance_pct - 100) / 2; 6670 prev_eff_load *= capacity_of(this_cpu); 6671 6672 /* 6673 * If sync, adjust the weight of prev_eff_load such that if 6674 * prev_eff == this_eff that select_idle_sibling() will consider 6675 * stacking the wakee on top of the waker if no other CPU is 6676 * idle. 6677 */ 6678 if (sync) 6679 prev_eff_load += 1; 6680 6681 return this_eff_load < prev_eff_load ? this_cpu : nr_cpumask_bits; 6682 } 6683 6684 static int wake_affine(struct sched_domain *sd, struct task_struct *p, 6685 int this_cpu, int prev_cpu, int sync) 6686 { 6687 int target = nr_cpumask_bits; 6688 6689 if (sched_feat(WA_IDLE)) 6690 target = wake_affine_idle(this_cpu, prev_cpu, sync); 6691 6692 if (sched_feat(WA_WEIGHT) && target == nr_cpumask_bits) 6693 target = wake_affine_weight(sd, p, this_cpu, prev_cpu, sync); 6694 6695 schedstat_inc(p->stats.nr_wakeups_affine_attempts); 6696 if (target != this_cpu) 6697 return prev_cpu; 6698 6699 schedstat_inc(sd->ttwu_move_affine); 6700 schedstat_inc(p->stats.nr_wakeups_affine); 6701 return target; 6702 } 6703 6704 static struct sched_group * 6705 find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu); 6706 6707 /* 6708 * find_idlest_group_cpu - find the idlest CPU among the CPUs in the group. 6709 */ 6710 static int 6711 find_idlest_group_cpu(struct sched_group *group, struct task_struct *p, int this_cpu) 6712 { 6713 unsigned long load, min_load = ULONG_MAX; 6714 unsigned int min_exit_latency = UINT_MAX; 6715 u64 latest_idle_timestamp = 0; 6716 int least_loaded_cpu = this_cpu; 6717 int shallowest_idle_cpu = -1; 6718 int i; 6719 6720 /* Check if we have any choice: */ 6721 if (group->group_weight == 1) 6722 return cpumask_first(sched_group_span(group)); 6723 6724 /* Traverse only the allowed CPUs */ 6725 for_each_cpu_and(i, sched_group_span(group), p->cpus_ptr) { 6726 struct rq *rq = cpu_rq(i); 6727 6728 if (!sched_core_cookie_match(rq, p)) 6729 continue; 6730 6731 if (sched_idle_cpu(i)) 6732 return i; 6733 6734 if (available_idle_cpu(i)) { 6735 struct cpuidle_state *idle = idle_get_state(rq); 6736 if (idle && idle->exit_latency < min_exit_latency) { 6737 /* 6738 * We give priority to a CPU whose idle state 6739 * has the smallest exit latency irrespective 6740 * of any idle timestamp. 6741 */ 6742 min_exit_latency = idle->exit_latency; 6743 latest_idle_timestamp = rq->idle_stamp; 6744 shallowest_idle_cpu = i; 6745 } else if ((!idle || idle->exit_latency == min_exit_latency) && 6746 rq->idle_stamp > latest_idle_timestamp) { 6747 /* 6748 * If equal or no active idle state, then 6749 * the most recently idled CPU might have 6750 * a warmer cache. 6751 */ 6752 latest_idle_timestamp = rq->idle_stamp; 6753 shallowest_idle_cpu = i; 6754 } 6755 } else if (shallowest_idle_cpu == -1) { 6756 load = cpu_load(cpu_rq(i)); 6757 if (load < min_load) { 6758 min_load = load; 6759 least_loaded_cpu = i; 6760 } 6761 } 6762 } 6763 6764 return shallowest_idle_cpu != -1 ? shallowest_idle_cpu : least_loaded_cpu; 6765 } 6766 6767 static inline int find_idlest_cpu(struct sched_domain *sd, struct task_struct *p, 6768 int cpu, int prev_cpu, int sd_flag) 6769 { 6770 int new_cpu = cpu; 6771 6772 if (!cpumask_intersects(sched_domain_span(sd), p->cpus_ptr)) 6773 return prev_cpu; 6774 6775 /* 6776 * We need task's util for cpu_util_without, sync it up to 6777 * prev_cpu's last_update_time. 6778 */ 6779 if (!(sd_flag & SD_BALANCE_FORK)) 6780 sync_entity_load_avg(&p->se); 6781 6782 while (sd) { 6783 struct sched_group *group; 6784 struct sched_domain *tmp; 6785 int weight; 6786 6787 if (!(sd->flags & sd_flag)) { 6788 sd = sd->child; 6789 continue; 6790 } 6791 6792 group = find_idlest_group(sd, p, cpu); 6793 if (!group) { 6794 sd = sd->child; 6795 continue; 6796 } 6797 6798 new_cpu = find_idlest_group_cpu(group, p, cpu); 6799 if (new_cpu == cpu) { 6800 /* Now try balancing at a lower domain level of 'cpu': */ 6801 sd = sd->child; 6802 continue; 6803 } 6804 6805 /* Now try balancing at a lower domain level of 'new_cpu': */ 6806 cpu = new_cpu; 6807 weight = sd->span_weight; 6808 sd = NULL; 6809 for_each_domain(cpu, tmp) { 6810 if (weight <= tmp->span_weight) 6811 break; 6812 if (tmp->flags & sd_flag) 6813 sd = tmp; 6814 } 6815 } 6816 6817 return new_cpu; 6818 } 6819 6820 static inline int __select_idle_cpu(int cpu, struct task_struct *p) 6821 { 6822 if ((available_idle_cpu(cpu) || sched_idle_cpu(cpu)) && 6823 sched_cpu_cookie_match(cpu_rq(cpu), p)) 6824 return cpu; 6825 6826 return -1; 6827 } 6828 6829 #ifdef CONFIG_SCHED_SMT 6830 DEFINE_STATIC_KEY_FALSE(sched_smt_present); 6831 EXPORT_SYMBOL_GPL(sched_smt_present); 6832 6833 static inline void set_idle_cores(int cpu, int val) 6834 { 6835 struct sched_domain_shared *sds; 6836 6837 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); 6838 if (sds) 6839 WRITE_ONCE(sds->has_idle_cores, val); 6840 } 6841 6842 static inline bool test_idle_cores(int cpu) 6843 { 6844 struct sched_domain_shared *sds; 6845 6846 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); 6847 if (sds) 6848 return READ_ONCE(sds->has_idle_cores); 6849 6850 return false; 6851 } 6852 6853 /* 6854 * Scans the local SMT mask to see if the entire core is idle, and records this 6855 * information in sd_llc_shared->has_idle_cores. 6856 * 6857 * Since SMT siblings share all cache levels, inspecting this limited remote 6858 * state should be fairly cheap. 6859 */ 6860 void __update_idle_core(struct rq *rq) 6861 { 6862 int core = cpu_of(rq); 6863 int cpu; 6864 6865 rcu_read_lock(); 6866 if (test_idle_cores(core)) 6867 goto unlock; 6868 6869 for_each_cpu(cpu, cpu_smt_mask(core)) { 6870 if (cpu == core) 6871 continue; 6872 6873 if (!available_idle_cpu(cpu)) 6874 goto unlock; 6875 } 6876 6877 set_idle_cores(core, 1); 6878 unlock: 6879 rcu_read_unlock(); 6880 } 6881 6882 /* 6883 * Scan the entire LLC domain for idle cores; this dynamically switches off if 6884 * there are no idle cores left in the system; tracked through 6885 * sd_llc->shared->has_idle_cores and enabled through update_idle_core() above. 6886 */ 6887 static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu) 6888 { 6889 bool idle = true; 6890 int cpu; 6891 6892 for_each_cpu(cpu, cpu_smt_mask(core)) { 6893 if (!available_idle_cpu(cpu)) { 6894 idle = false; 6895 if (*idle_cpu == -1) { 6896 if (sched_idle_cpu(cpu) && cpumask_test_cpu(cpu, p->cpus_ptr)) { 6897 *idle_cpu = cpu; 6898 break; 6899 } 6900 continue; 6901 } 6902 break; 6903 } 6904 if (*idle_cpu == -1 && cpumask_test_cpu(cpu, p->cpus_ptr)) 6905 *idle_cpu = cpu; 6906 } 6907 6908 if (idle) 6909 return core; 6910 6911 cpumask_andnot(cpus, cpus, cpu_smt_mask(core)); 6912 return -1; 6913 } 6914 6915 /* 6916 * Scan the local SMT mask for idle CPUs. 6917 */ 6918 static int select_idle_smt(struct task_struct *p, int target) 6919 { 6920 int cpu; 6921 6922 for_each_cpu_and(cpu, cpu_smt_mask(target), p->cpus_ptr) { 6923 if (cpu == target) 6924 continue; 6925 if (available_idle_cpu(cpu) || sched_idle_cpu(cpu)) 6926 return cpu; 6927 } 6928 6929 return -1; 6930 } 6931 6932 #else /* CONFIG_SCHED_SMT */ 6933 6934 static inline void set_idle_cores(int cpu, int val) 6935 { 6936 } 6937 6938 static inline bool test_idle_cores(int cpu) 6939 { 6940 return false; 6941 } 6942 6943 static inline int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu) 6944 { 6945 return __select_idle_cpu(core, p); 6946 } 6947 6948 static inline int select_idle_smt(struct task_struct *p, int target) 6949 { 6950 return -1; 6951 } 6952 6953 #endif /* CONFIG_SCHED_SMT */ 6954 6955 /* 6956 * Scan the LLC domain for idle CPUs; this is dynamically regulated by 6957 * comparing the average scan cost (tracked in sd->avg_scan_cost) against the 6958 * average idle time for this rq (as found in rq->avg_idle). 6959 */ 6960 static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool has_idle_core, int target) 6961 { 6962 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 6963 int i, cpu, idle_cpu = -1, nr = INT_MAX; 6964 struct sched_domain_shared *sd_share; 6965 struct rq *this_rq = this_rq(); 6966 int this = smp_processor_id(); 6967 struct sched_domain *this_sd = NULL; 6968 u64 time = 0; 6969 6970 cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr); 6971 6972 if (sched_feat(SIS_PROP) && !has_idle_core) { 6973 u64 avg_cost, avg_idle, span_avg; 6974 unsigned long now = jiffies; 6975 6976 this_sd = rcu_dereference(*this_cpu_ptr(&sd_llc)); 6977 if (!this_sd) 6978 return -1; 6979 6980 /* 6981 * If we're busy, the assumption that the last idle period 6982 * predicts the future is flawed; age away the remaining 6983 * predicted idle time. 6984 */ 6985 if (unlikely(this_rq->wake_stamp < now)) { 6986 while (this_rq->wake_stamp < now && this_rq->wake_avg_idle) { 6987 this_rq->wake_stamp++; 6988 this_rq->wake_avg_idle >>= 1; 6989 } 6990 } 6991 6992 avg_idle = this_rq->wake_avg_idle; 6993 avg_cost = this_sd->avg_scan_cost + 1; 6994 6995 span_avg = sd->span_weight * avg_idle; 6996 if (span_avg > 4*avg_cost) 6997 nr = div_u64(span_avg, avg_cost); 6998 else 6999 nr = 4; 7000 7001 time = cpu_clock(this); 7002 } 7003 7004 if (sched_feat(SIS_UTIL)) { 7005 sd_share = rcu_dereference(per_cpu(sd_llc_shared, target)); 7006 if (sd_share) { 7007 /* because !--nr is the condition to stop scan */ 7008 nr = READ_ONCE(sd_share->nr_idle_scan) + 1; 7009 /* overloaded LLC is unlikely to have idle cpu/core */ 7010 if (nr == 1) 7011 return -1; 7012 } 7013 } 7014 7015 for_each_cpu_wrap(cpu, cpus, target + 1) { 7016 if (has_idle_core) { 7017 i = select_idle_core(p, cpu, cpus, &idle_cpu); 7018 if ((unsigned int)i < nr_cpumask_bits) 7019 return i; 7020 7021 } else { 7022 if (!--nr) 7023 return -1; 7024 idle_cpu = __select_idle_cpu(cpu, p); 7025 if ((unsigned int)idle_cpu < nr_cpumask_bits) 7026 break; 7027 } 7028 } 7029 7030 if (has_idle_core) 7031 set_idle_cores(target, false); 7032 7033 if (sched_feat(SIS_PROP) && this_sd && !has_idle_core) { 7034 time = cpu_clock(this) - time; 7035 7036 /* 7037 * Account for the scan cost of wakeups against the average 7038 * idle time. 7039 */ 7040 this_rq->wake_avg_idle -= min(this_rq->wake_avg_idle, time); 7041 7042 update_avg(&this_sd->avg_scan_cost, time); 7043 } 7044 7045 return idle_cpu; 7046 } 7047 7048 /* 7049 * Scan the asym_capacity domain for idle CPUs; pick the first idle one on which 7050 * the task fits. If no CPU is big enough, but there are idle ones, try to 7051 * maximize capacity. 7052 */ 7053 static int 7054 select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) 7055 { 7056 unsigned long task_util, util_min, util_max, best_cap = 0; 7057 int fits, best_fits = 0; 7058 int cpu, best_cpu = -1; 7059 struct cpumask *cpus; 7060 7061 cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 7062 cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr); 7063 7064 task_util = task_util_est(p); 7065 util_min = uclamp_eff_value(p, UCLAMP_MIN); 7066 util_max = uclamp_eff_value(p, UCLAMP_MAX); 7067 7068 for_each_cpu_wrap(cpu, cpus, target + 1) { 7069 unsigned long cpu_cap = capacity_of(cpu); 7070 7071 if (!available_idle_cpu(cpu) && !sched_idle_cpu(cpu)) 7072 continue; 7073 7074 fits = util_fits_cpu(task_util, util_min, util_max, cpu); 7075 7076 /* This CPU fits with all requirements */ 7077 if (fits > 0) 7078 return cpu; 7079 /* 7080 * Only the min performance hint (i.e. uclamp_min) doesn't fit. 7081 * Look for the CPU with best capacity. 7082 */ 7083 else if (fits < 0) 7084 cpu_cap = capacity_orig_of(cpu) - thermal_load_avg(cpu_rq(cpu)); 7085 7086 /* 7087 * First, select CPU which fits better (-1 being better than 0). 7088 * Then, select the one with best capacity at same level. 7089 */ 7090 if ((fits < best_fits) || 7091 ((fits == best_fits) && (cpu_cap > best_cap))) { 7092 best_cap = cpu_cap; 7093 best_cpu = cpu; 7094 best_fits = fits; 7095 } 7096 } 7097 7098 return best_cpu; 7099 } 7100 7101 static inline bool asym_fits_cpu(unsigned long util, 7102 unsigned long util_min, 7103 unsigned long util_max, 7104 int cpu) 7105 { 7106 if (sched_asym_cpucap_active()) 7107 /* 7108 * Return true only if the cpu fully fits the task requirements 7109 * which include the utilization and the performance hints. 7110 */ 7111 return (util_fits_cpu(util, util_min, util_max, cpu) > 0); 7112 7113 return true; 7114 } 7115 7116 /* 7117 * Try and locate an idle core/thread in the LLC cache domain. 7118 */ 7119 static int select_idle_sibling(struct task_struct *p, int prev, int target) 7120 { 7121 bool has_idle_core = false; 7122 struct sched_domain *sd; 7123 unsigned long task_util, util_min, util_max; 7124 int i, recent_used_cpu; 7125 7126 /* 7127 * On asymmetric system, update task utilization because we will check 7128 * that the task fits with cpu's capacity. 7129 */ 7130 if (sched_asym_cpucap_active()) { 7131 sync_entity_load_avg(&p->se); 7132 task_util = task_util_est(p); 7133 util_min = uclamp_eff_value(p, UCLAMP_MIN); 7134 util_max = uclamp_eff_value(p, UCLAMP_MAX); 7135 } 7136 7137 /* 7138 * per-cpu select_rq_mask usage 7139 */ 7140 lockdep_assert_irqs_disabled(); 7141 7142 if ((available_idle_cpu(target) || sched_idle_cpu(target)) && 7143 asym_fits_cpu(task_util, util_min, util_max, target)) 7144 return target; 7145 7146 /* 7147 * If the previous CPU is cache affine and idle, don't be stupid: 7148 */ 7149 if (prev != target && cpus_share_cache(prev, target) && 7150 (available_idle_cpu(prev) || sched_idle_cpu(prev)) && 7151 asym_fits_cpu(task_util, util_min, util_max, prev)) 7152 return prev; 7153 7154 /* 7155 * Allow a per-cpu kthread to stack with the wakee if the 7156 * kworker thread and the tasks previous CPUs are the same. 7157 * The assumption is that the wakee queued work for the 7158 * per-cpu kthread that is now complete and the wakeup is 7159 * essentially a sync wakeup. An obvious example of this 7160 * pattern is IO completions. 7161 */ 7162 if (is_per_cpu_kthread(current) && 7163 in_task() && 7164 prev == smp_processor_id() && 7165 this_rq()->nr_running <= 1 && 7166 asym_fits_cpu(task_util, util_min, util_max, prev)) { 7167 return prev; 7168 } 7169 7170 /* Check a recently used CPU as a potential idle candidate: */ 7171 recent_used_cpu = p->recent_used_cpu; 7172 p->recent_used_cpu = prev; 7173 if (recent_used_cpu != prev && 7174 recent_used_cpu != target && 7175 cpus_share_cache(recent_used_cpu, target) && 7176 (available_idle_cpu(recent_used_cpu) || sched_idle_cpu(recent_used_cpu)) && 7177 cpumask_test_cpu(recent_used_cpu, p->cpus_ptr) && 7178 asym_fits_cpu(task_util, util_min, util_max, recent_used_cpu)) { 7179 return recent_used_cpu; 7180 } 7181 7182 /* 7183 * For asymmetric CPU capacity systems, our domain of interest is 7184 * sd_asym_cpucapacity rather than sd_llc. 7185 */ 7186 if (sched_asym_cpucap_active()) { 7187 sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, target)); 7188 /* 7189 * On an asymmetric CPU capacity system where an exclusive 7190 * cpuset defines a symmetric island (i.e. one unique 7191 * capacity_orig value through the cpuset), the key will be set 7192 * but the CPUs within that cpuset will not have a domain with 7193 * SD_ASYM_CPUCAPACITY. These should follow the usual symmetric 7194 * capacity path. 7195 */ 7196 if (sd) { 7197 i = select_idle_capacity(p, sd, target); 7198 return ((unsigned)i < nr_cpumask_bits) ? i : target; 7199 } 7200 } 7201 7202 sd = rcu_dereference(per_cpu(sd_llc, target)); 7203 if (!sd) 7204 return target; 7205 7206 if (sched_smt_active()) { 7207 has_idle_core = test_idle_cores(target); 7208 7209 if (!has_idle_core && cpus_share_cache(prev, target)) { 7210 i = select_idle_smt(p, prev); 7211 if ((unsigned int)i < nr_cpumask_bits) 7212 return i; 7213 } 7214 } 7215 7216 i = select_idle_cpu(p, sd, has_idle_core, target); 7217 if ((unsigned)i < nr_cpumask_bits) 7218 return i; 7219 7220 return target; 7221 } 7222 7223 /** 7224 * cpu_util() - Estimates the amount of CPU capacity used by CFS tasks. 7225 * @cpu: the CPU to get the utilization for 7226 * @p: task for which the CPU utilization should be predicted or NULL 7227 * @dst_cpu: CPU @p migrates to, -1 if @p moves from @cpu or @p == NULL 7228 * @boost: 1 to enable boosting, otherwise 0 7229 * 7230 * The unit of the return value must be the same as the one of CPU capacity 7231 * so that CPU utilization can be compared with CPU capacity. 7232 * 7233 * CPU utilization is the sum of running time of runnable tasks plus the 7234 * recent utilization of currently non-runnable tasks on that CPU. 7235 * It represents the amount of CPU capacity currently used by CFS tasks in 7236 * the range [0..max CPU capacity] with max CPU capacity being the CPU 7237 * capacity at f_max. 7238 * 7239 * The estimated CPU utilization is defined as the maximum between CPU 7240 * utilization and sum of the estimated utilization of the currently 7241 * runnable tasks on that CPU. It preserves a utilization "snapshot" of 7242 * previously-executed tasks, which helps better deduce how busy a CPU will 7243 * be when a long-sleeping task wakes up. The contribution to CPU utilization 7244 * of such a task would be significantly decayed at this point of time. 7245 * 7246 * Boosted CPU utilization is defined as max(CPU runnable, CPU utilization). 7247 * CPU contention for CFS tasks can be detected by CPU runnable > CPU 7248 * utilization. Boosting is implemented in cpu_util() so that internal 7249 * users (e.g. EAS) can use it next to external users (e.g. schedutil), 7250 * latter via cpu_util_cfs_boost(). 7251 * 7252 * CPU utilization can be higher than the current CPU capacity 7253 * (f_curr/f_max * max CPU capacity) or even the max CPU capacity because 7254 * of rounding errors as well as task migrations or wakeups of new tasks. 7255 * CPU utilization has to be capped to fit into the [0..max CPU capacity] 7256 * range. Otherwise a group of CPUs (CPU0 util = 121% + CPU1 util = 80%) 7257 * could be seen as over-utilized even though CPU1 has 20% of spare CPU 7258 * capacity. CPU utilization is allowed to overshoot current CPU capacity 7259 * though since this is useful for predicting the CPU capacity required 7260 * after task migrations (scheduler-driven DVFS). 7261 * 7262 * Return: (Boosted) (estimated) utilization for the specified CPU. 7263 */ 7264 static unsigned long 7265 cpu_util(int cpu, struct task_struct *p, int dst_cpu, int boost) 7266 { 7267 struct cfs_rq *cfs_rq = &cpu_rq(cpu)->cfs; 7268 unsigned long util = READ_ONCE(cfs_rq->avg.util_avg); 7269 unsigned long runnable; 7270 7271 if (boost) { 7272 runnable = READ_ONCE(cfs_rq->avg.runnable_avg); 7273 util = max(util, runnable); 7274 } 7275 7276 /* 7277 * If @dst_cpu is -1 or @p migrates from @cpu to @dst_cpu remove its 7278 * contribution. If @p migrates from another CPU to @cpu add its 7279 * contribution. In all the other cases @cpu is not impacted by the 7280 * migration so its util_avg is already correct. 7281 */ 7282 if (p && task_cpu(p) == cpu && dst_cpu != cpu) 7283 lsub_positive(&util, task_util(p)); 7284 else if (p && task_cpu(p) != cpu && dst_cpu == cpu) 7285 util += task_util(p); 7286 7287 if (sched_feat(UTIL_EST)) { 7288 unsigned long util_est; 7289 7290 util_est = READ_ONCE(cfs_rq->avg.util_est.enqueued); 7291 7292 if (boost) 7293 util_est = max(util_est, runnable); 7294 7295 /* 7296 * During wake-up @p isn't enqueued yet and doesn't contribute 7297 * to any cpu_rq(cpu)->cfs.avg.util_est.enqueued. 7298 * If @dst_cpu == @cpu add it to "simulate" cpu_util after @p 7299 * has been enqueued. 7300 * 7301 * During exec (@dst_cpu = -1) @p is enqueued and does 7302 * contribute to cpu_rq(cpu)->cfs.util_est.enqueued. 7303 * Remove it to "simulate" cpu_util without @p's contribution. 7304 * 7305 * Despite the task_on_rq_queued(@p) check there is still a 7306 * small window for a possible race when an exec 7307 * select_task_rq_fair() races with LB's detach_task(). 7308 * 7309 * detach_task() 7310 * deactivate_task() 7311 * p->on_rq = TASK_ON_RQ_MIGRATING; 7312 * -------------------------------- A 7313 * dequeue_task() \ 7314 * dequeue_task_fair() + Race Time 7315 * util_est_dequeue() / 7316 * -------------------------------- B 7317 * 7318 * The additional check "current == p" is required to further 7319 * reduce the race window. 7320 */ 7321 if (dst_cpu == cpu) 7322 util_est += _task_util_est(p); 7323 else if (p && unlikely(task_on_rq_queued(p) || current == p)) 7324 lsub_positive(&util_est, _task_util_est(p)); 7325 7326 util = max(util, util_est); 7327 } 7328 7329 return min(util, capacity_orig_of(cpu)); 7330 } 7331 7332 unsigned long cpu_util_cfs(int cpu) 7333 { 7334 return cpu_util(cpu, NULL, -1, 0); 7335 } 7336 7337 unsigned long cpu_util_cfs_boost(int cpu) 7338 { 7339 return cpu_util(cpu, NULL, -1, 1); 7340 } 7341 7342 /* 7343 * cpu_util_without: compute cpu utilization without any contributions from *p 7344 * @cpu: the CPU which utilization is requested 7345 * @p: the task which utilization should be discounted 7346 * 7347 * The utilization of a CPU is defined by the utilization of tasks currently 7348 * enqueued on that CPU as well as tasks which are currently sleeping after an 7349 * execution on that CPU. 7350 * 7351 * This method returns the utilization of the specified CPU by discounting the 7352 * utilization of the specified task, whenever the task is currently 7353 * contributing to the CPU utilization. 7354 */ 7355 static unsigned long cpu_util_without(int cpu, struct task_struct *p) 7356 { 7357 /* Task has no contribution or is new */ 7358 if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 7359 p = NULL; 7360 7361 return cpu_util(cpu, p, -1, 0); 7362 } 7363 7364 /* 7365 * energy_env - Utilization landscape for energy estimation. 7366 * @task_busy_time: Utilization contribution by the task for which we test the 7367 * placement. Given by eenv_task_busy_time(). 7368 * @pd_busy_time: Utilization of the whole perf domain without the task 7369 * contribution. Given by eenv_pd_busy_time(). 7370 * @cpu_cap: Maximum CPU capacity for the perf domain. 7371 * @pd_cap: Entire perf domain capacity. (pd->nr_cpus * cpu_cap). 7372 */ 7373 struct energy_env { 7374 unsigned long task_busy_time; 7375 unsigned long pd_busy_time; 7376 unsigned long cpu_cap; 7377 unsigned long pd_cap; 7378 }; 7379 7380 /* 7381 * Compute the task busy time for compute_energy(). This time cannot be 7382 * injected directly into effective_cpu_util() because of the IRQ scaling. 7383 * The latter only makes sense with the most recent CPUs where the task has 7384 * run. 7385 */ 7386 static inline void eenv_task_busy_time(struct energy_env *eenv, 7387 struct task_struct *p, int prev_cpu) 7388 { 7389 unsigned long busy_time, max_cap = arch_scale_cpu_capacity(prev_cpu); 7390 unsigned long irq = cpu_util_irq(cpu_rq(prev_cpu)); 7391 7392 if (unlikely(irq >= max_cap)) 7393 busy_time = max_cap; 7394 else 7395 busy_time = scale_irq_capacity(task_util_est(p), irq, max_cap); 7396 7397 eenv->task_busy_time = busy_time; 7398 } 7399 7400 /* 7401 * Compute the perf_domain (PD) busy time for compute_energy(). Based on the 7402 * utilization for each @pd_cpus, it however doesn't take into account 7403 * clamping since the ratio (utilization / cpu_capacity) is already enough to 7404 * scale the EM reported power consumption at the (eventually clamped) 7405 * cpu_capacity. 7406 * 7407 * The contribution of the task @p for which we want to estimate the 7408 * energy cost is removed (by cpu_util()) and must be calculated 7409 * separately (see eenv_task_busy_time). This ensures: 7410 * 7411 * - A stable PD utilization, no matter which CPU of that PD we want to place 7412 * the task on. 7413 * 7414 * - A fair comparison between CPUs as the task contribution (task_util()) 7415 * will always be the same no matter which CPU utilization we rely on 7416 * (util_avg or util_est). 7417 * 7418 * Set @eenv busy time for the PD that spans @pd_cpus. This busy time can't 7419 * exceed @eenv->pd_cap. 7420 */ 7421 static inline void eenv_pd_busy_time(struct energy_env *eenv, 7422 struct cpumask *pd_cpus, 7423 struct task_struct *p) 7424 { 7425 unsigned long busy_time = 0; 7426 int cpu; 7427 7428 for_each_cpu(cpu, pd_cpus) { 7429 unsigned long util = cpu_util(cpu, p, -1, 0); 7430 7431 busy_time += effective_cpu_util(cpu, util, ENERGY_UTIL, NULL); 7432 } 7433 7434 eenv->pd_busy_time = min(eenv->pd_cap, busy_time); 7435 } 7436 7437 /* 7438 * Compute the maximum utilization for compute_energy() when the task @p 7439 * is placed on the cpu @dst_cpu. 7440 * 7441 * Returns the maximum utilization among @eenv->cpus. This utilization can't 7442 * exceed @eenv->cpu_cap. 7443 */ 7444 static inline unsigned long 7445 eenv_pd_max_util(struct energy_env *eenv, struct cpumask *pd_cpus, 7446 struct task_struct *p, int dst_cpu) 7447 { 7448 unsigned long max_util = 0; 7449 int cpu; 7450 7451 for_each_cpu(cpu, pd_cpus) { 7452 struct task_struct *tsk = (cpu == dst_cpu) ? p : NULL; 7453 unsigned long util = cpu_util(cpu, p, dst_cpu, 1); 7454 unsigned long eff_util; 7455 7456 /* 7457 * Performance domain frequency: utilization clamping 7458 * must be considered since it affects the selection 7459 * of the performance domain frequency. 7460 * NOTE: in case RT tasks are running, by default the 7461 * FREQUENCY_UTIL's utilization can be max OPP. 7462 */ 7463 eff_util = effective_cpu_util(cpu, util, FREQUENCY_UTIL, tsk); 7464 max_util = max(max_util, eff_util); 7465 } 7466 7467 return min(max_util, eenv->cpu_cap); 7468 } 7469 7470 /* 7471 * compute_energy(): Use the Energy Model to estimate the energy that @pd would 7472 * consume for a given utilization landscape @eenv. When @dst_cpu < 0, the task 7473 * contribution is ignored. 7474 */ 7475 static inline unsigned long 7476 compute_energy(struct energy_env *eenv, struct perf_domain *pd, 7477 struct cpumask *pd_cpus, struct task_struct *p, int dst_cpu) 7478 { 7479 unsigned long max_util = eenv_pd_max_util(eenv, pd_cpus, p, dst_cpu); 7480 unsigned long busy_time = eenv->pd_busy_time; 7481 7482 if (dst_cpu >= 0) 7483 busy_time = min(eenv->pd_cap, busy_time + eenv->task_busy_time); 7484 7485 return em_cpu_energy(pd->em_pd, max_util, busy_time, eenv->cpu_cap); 7486 } 7487 7488 /* 7489 * find_energy_efficient_cpu(): Find most energy-efficient target CPU for the 7490 * waking task. find_energy_efficient_cpu() looks for the CPU with maximum 7491 * spare capacity in each performance domain and uses it as a potential 7492 * candidate to execute the task. Then, it uses the Energy Model to figure 7493 * out which of the CPU candidates is the most energy-efficient. 7494 * 7495 * The rationale for this heuristic is as follows. In a performance domain, 7496 * all the most energy efficient CPU candidates (according to the Energy 7497 * Model) are those for which we'll request a low frequency. When there are 7498 * several CPUs for which the frequency request will be the same, we don't 7499 * have enough data to break the tie between them, because the Energy Model 7500 * only includes active power costs. With this model, if we assume that 7501 * frequency requests follow utilization (e.g. using schedutil), the CPU with 7502 * the maximum spare capacity in a performance domain is guaranteed to be among 7503 * the best candidates of the performance domain. 7504 * 7505 * In practice, it could be preferable from an energy standpoint to pack 7506 * small tasks on a CPU in order to let other CPUs go in deeper idle states, 7507 * but that could also hurt our chances to go cluster idle, and we have no 7508 * ways to tell with the current Energy Model if this is actually a good 7509 * idea or not. So, find_energy_efficient_cpu() basically favors 7510 * cluster-packing, and spreading inside a cluster. That should at least be 7511 * a good thing for latency, and this is consistent with the idea that most 7512 * of the energy savings of EAS come from the asymmetry of the system, and 7513 * not so much from breaking the tie between identical CPUs. That's also the 7514 * reason why EAS is enabled in the topology code only for systems where 7515 * SD_ASYM_CPUCAPACITY is set. 7516 * 7517 * NOTE: Forkees are not accepted in the energy-aware wake-up path because 7518 * they don't have any useful utilization data yet and it's not possible to 7519 * forecast their impact on energy consumption. Consequently, they will be 7520 * placed by find_idlest_cpu() on the least loaded CPU, which might turn out 7521 * to be energy-inefficient in some use-cases. The alternative would be to 7522 * bias new tasks towards specific types of CPUs first, or to try to infer 7523 * their util_avg from the parent task, but those heuristics could hurt 7524 * other use-cases too. So, until someone finds a better way to solve this, 7525 * let's keep things simple by re-using the existing slow path. 7526 */ 7527 static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu) 7528 { 7529 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 7530 unsigned long prev_delta = ULONG_MAX, best_delta = ULONG_MAX; 7531 unsigned long p_util_min = uclamp_is_used() ? uclamp_eff_value(p, UCLAMP_MIN) : 0; 7532 unsigned long p_util_max = uclamp_is_used() ? uclamp_eff_value(p, UCLAMP_MAX) : 1024; 7533 struct root_domain *rd = this_rq()->rd; 7534 int cpu, best_energy_cpu, target = -1; 7535 int prev_fits = -1, best_fits = -1; 7536 unsigned long best_thermal_cap = 0; 7537 unsigned long prev_thermal_cap = 0; 7538 struct sched_domain *sd; 7539 struct perf_domain *pd; 7540 struct energy_env eenv; 7541 7542 rcu_read_lock(); 7543 pd = rcu_dereference(rd->pd); 7544 if (!pd || READ_ONCE(rd->overutilized)) 7545 goto unlock; 7546 7547 /* 7548 * Energy-aware wake-up happens on the lowest sched_domain starting 7549 * from sd_asym_cpucapacity spanning over this_cpu and prev_cpu. 7550 */ 7551 sd = rcu_dereference(*this_cpu_ptr(&sd_asym_cpucapacity)); 7552 while (sd && !cpumask_test_cpu(prev_cpu, sched_domain_span(sd))) 7553 sd = sd->parent; 7554 if (!sd) 7555 goto unlock; 7556 7557 target = prev_cpu; 7558 7559 sync_entity_load_avg(&p->se); 7560 if (!uclamp_task_util(p, p_util_min, p_util_max)) 7561 goto unlock; 7562 7563 eenv_task_busy_time(&eenv, p, prev_cpu); 7564 7565 for (; pd; pd = pd->next) { 7566 unsigned long util_min = p_util_min, util_max = p_util_max; 7567 unsigned long cpu_cap, cpu_thermal_cap, util; 7568 unsigned long cur_delta, max_spare_cap = 0; 7569 unsigned long rq_util_min, rq_util_max; 7570 unsigned long prev_spare_cap = 0; 7571 int max_spare_cap_cpu = -1; 7572 unsigned long base_energy; 7573 int fits, max_fits = -1; 7574 7575 cpumask_and(cpus, perf_domain_span(pd), cpu_online_mask); 7576 7577 if (cpumask_empty(cpus)) 7578 continue; 7579 7580 /* Account thermal pressure for the energy estimation */ 7581 cpu = cpumask_first(cpus); 7582 cpu_thermal_cap = arch_scale_cpu_capacity(cpu); 7583 cpu_thermal_cap -= arch_scale_thermal_pressure(cpu); 7584 7585 eenv.cpu_cap = cpu_thermal_cap; 7586 eenv.pd_cap = 0; 7587 7588 for_each_cpu(cpu, cpus) { 7589 struct rq *rq = cpu_rq(cpu); 7590 7591 eenv.pd_cap += cpu_thermal_cap; 7592 7593 if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) 7594 continue; 7595 7596 if (!cpumask_test_cpu(cpu, p->cpus_ptr)) 7597 continue; 7598 7599 util = cpu_util(cpu, p, cpu, 0); 7600 cpu_cap = capacity_of(cpu); 7601 7602 /* 7603 * Skip CPUs that cannot satisfy the capacity request. 7604 * IOW, placing the task there would make the CPU 7605 * overutilized. Take uclamp into account to see how 7606 * much capacity we can get out of the CPU; this is 7607 * aligned with sched_cpu_util(). 7608 */ 7609 if (uclamp_is_used() && !uclamp_rq_is_idle(rq)) { 7610 /* 7611 * Open code uclamp_rq_util_with() except for 7612 * the clamp() part. Ie: apply max aggregation 7613 * only. util_fits_cpu() logic requires to 7614 * operate on non clamped util but must use the 7615 * max-aggregated uclamp_{min, max}. 7616 */ 7617 rq_util_min = uclamp_rq_get(rq, UCLAMP_MIN); 7618 rq_util_max = uclamp_rq_get(rq, UCLAMP_MAX); 7619 7620 util_min = max(rq_util_min, p_util_min); 7621 util_max = max(rq_util_max, p_util_max); 7622 } 7623 7624 fits = util_fits_cpu(util, util_min, util_max, cpu); 7625 if (!fits) 7626 continue; 7627 7628 lsub_positive(&cpu_cap, util); 7629 7630 if (cpu == prev_cpu) { 7631 /* Always use prev_cpu as a candidate. */ 7632 prev_spare_cap = cpu_cap; 7633 prev_fits = fits; 7634 } else if ((fits > max_fits) || 7635 ((fits == max_fits) && (cpu_cap > max_spare_cap))) { 7636 /* 7637 * Find the CPU with the maximum spare capacity 7638 * among the remaining CPUs in the performance 7639 * domain. 7640 */ 7641 max_spare_cap = cpu_cap; 7642 max_spare_cap_cpu = cpu; 7643 max_fits = fits; 7644 } 7645 } 7646 7647 if (max_spare_cap_cpu < 0 && prev_spare_cap == 0) 7648 continue; 7649 7650 eenv_pd_busy_time(&eenv, cpus, p); 7651 /* Compute the 'base' energy of the pd, without @p */ 7652 base_energy = compute_energy(&eenv, pd, cpus, p, -1); 7653 7654 /* Evaluate the energy impact of using prev_cpu. */ 7655 if (prev_spare_cap > 0) { 7656 prev_delta = compute_energy(&eenv, pd, cpus, p, 7657 prev_cpu); 7658 /* CPU utilization has changed */ 7659 if (prev_delta < base_energy) 7660 goto unlock; 7661 prev_delta -= base_energy; 7662 prev_thermal_cap = cpu_thermal_cap; 7663 best_delta = min(best_delta, prev_delta); 7664 } 7665 7666 /* Evaluate the energy impact of using max_spare_cap_cpu. */ 7667 if (max_spare_cap_cpu >= 0 && max_spare_cap > prev_spare_cap) { 7668 /* Current best energy cpu fits better */ 7669 if (max_fits < best_fits) 7670 continue; 7671 7672 /* 7673 * Both don't fit performance hint (i.e. uclamp_min) 7674 * but best energy cpu has better capacity. 7675 */ 7676 if ((max_fits < 0) && 7677 (cpu_thermal_cap <= best_thermal_cap)) 7678 continue; 7679 7680 cur_delta = compute_energy(&eenv, pd, cpus, p, 7681 max_spare_cap_cpu); 7682 /* CPU utilization has changed */ 7683 if (cur_delta < base_energy) 7684 goto unlock; 7685 cur_delta -= base_energy; 7686 7687 /* 7688 * Both fit for the task but best energy cpu has lower 7689 * energy impact. 7690 */ 7691 if ((max_fits > 0) && (best_fits > 0) && 7692 (cur_delta >= best_delta)) 7693 continue; 7694 7695 best_delta = cur_delta; 7696 best_energy_cpu = max_spare_cap_cpu; 7697 best_fits = max_fits; 7698 best_thermal_cap = cpu_thermal_cap; 7699 } 7700 } 7701 rcu_read_unlock(); 7702 7703 if ((best_fits > prev_fits) || 7704 ((best_fits > 0) && (best_delta < prev_delta)) || 7705 ((best_fits < 0) && (best_thermal_cap > prev_thermal_cap))) 7706 target = best_energy_cpu; 7707 7708 return target; 7709 7710 unlock: 7711 rcu_read_unlock(); 7712 7713 return target; 7714 } 7715 7716 /* 7717 * select_task_rq_fair: Select target runqueue for the waking task in domains 7718 * that have the relevant SD flag set. In practice, this is SD_BALANCE_WAKE, 7719 * SD_BALANCE_FORK, or SD_BALANCE_EXEC. 7720 * 7721 * Balances load by selecting the idlest CPU in the idlest group, or under 7722 * certain conditions an idle sibling CPU if the domain has SD_WAKE_AFFINE set. 7723 * 7724 * Returns the target CPU number. 7725 */ 7726 static int 7727 select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags) 7728 { 7729 int sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING); 7730 struct sched_domain *tmp, *sd = NULL; 7731 int cpu = smp_processor_id(); 7732 int new_cpu = prev_cpu; 7733 int want_affine = 0; 7734 /* SD_flags and WF_flags share the first nibble */ 7735 int sd_flag = wake_flags & 0xF; 7736 7737 /* 7738 * required for stable ->cpus_allowed 7739 */ 7740 lockdep_assert_held(&p->pi_lock); 7741 if (wake_flags & WF_TTWU) { 7742 record_wakee(p); 7743 7744 if ((wake_flags & WF_CURRENT_CPU) && 7745 cpumask_test_cpu(cpu, p->cpus_ptr)) 7746 return cpu; 7747 7748 if (sched_energy_enabled()) { 7749 new_cpu = find_energy_efficient_cpu(p, prev_cpu); 7750 if (new_cpu >= 0) 7751 return new_cpu; 7752 new_cpu = prev_cpu; 7753 } 7754 7755 want_affine = !wake_wide(p) && cpumask_test_cpu(cpu, p->cpus_ptr); 7756 } 7757 7758 rcu_read_lock(); 7759 for_each_domain(cpu, tmp) { 7760 /* 7761 * If both 'cpu' and 'prev_cpu' are part of this domain, 7762 * cpu is a valid SD_WAKE_AFFINE target. 7763 */ 7764 if (want_affine && (tmp->flags & SD_WAKE_AFFINE) && 7765 cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) { 7766 if (cpu != prev_cpu) 7767 new_cpu = wake_affine(tmp, p, cpu, prev_cpu, sync); 7768 7769 sd = NULL; /* Prefer wake_affine over balance flags */ 7770 break; 7771 } 7772 7773 /* 7774 * Usually only true for WF_EXEC and WF_FORK, as sched_domains 7775 * usually do not have SD_BALANCE_WAKE set. That means wakeup 7776 * will usually go to the fast path. 7777 */ 7778 if (tmp->flags & sd_flag) 7779 sd = tmp; 7780 else if (!want_affine) 7781 break; 7782 } 7783 7784 if (unlikely(sd)) { 7785 /* Slow path */ 7786 new_cpu = find_idlest_cpu(sd, p, cpu, prev_cpu, sd_flag); 7787 } else if (wake_flags & WF_TTWU) { /* XXX always ? */ 7788 /* Fast path */ 7789 new_cpu = select_idle_sibling(p, prev_cpu, new_cpu); 7790 } 7791 rcu_read_unlock(); 7792 7793 return new_cpu; 7794 } 7795 7796 /* 7797 * Called immediately before a task is migrated to a new CPU; task_cpu(p) and 7798 * cfs_rq_of(p) references at time of call are still valid and identify the 7799 * previous CPU. The caller guarantees p->pi_lock or task_rq(p)->lock is held. 7800 */ 7801 static void migrate_task_rq_fair(struct task_struct *p, int new_cpu) 7802 { 7803 struct sched_entity *se = &p->se; 7804 7805 /* 7806 * As blocked tasks retain absolute vruntime the migration needs to 7807 * deal with this by subtracting the old and adding the new 7808 * min_vruntime -- the latter is done by enqueue_entity() when placing 7809 * the task on the new runqueue. 7810 */ 7811 if (READ_ONCE(p->__state) == TASK_WAKING) { 7812 struct cfs_rq *cfs_rq = cfs_rq_of(se); 7813 7814 se->vruntime -= u64_u32_load(cfs_rq->min_vruntime); 7815 } 7816 7817 if (!task_on_rq_migrating(p)) { 7818 remove_entity_load_avg(se); 7819 7820 /* 7821 * Here, the task's PELT values have been updated according to 7822 * the current rq's clock. But if that clock hasn't been 7823 * updated in a while, a substantial idle time will be missed, 7824 * leading to an inflation after wake-up on the new rq. 7825 * 7826 * Estimate the missing time from the cfs_rq last_update_time 7827 * and update sched_avg to improve the PELT continuity after 7828 * migration. 7829 */ 7830 migrate_se_pelt_lag(se); 7831 } 7832 7833 /* Tell new CPU we are migrated */ 7834 se->avg.last_update_time = 0; 7835 7836 update_scan_period(p, new_cpu); 7837 } 7838 7839 static void task_dead_fair(struct task_struct *p) 7840 { 7841 remove_entity_load_avg(&p->se); 7842 } 7843 7844 static int 7845 balance_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 7846 { 7847 if (rq->nr_running) 7848 return 1; 7849 7850 return newidle_balance(rq, rf) != 0; 7851 } 7852 #endif /* CONFIG_SMP */ 7853 7854 static unsigned long wakeup_gran(struct sched_entity *se) 7855 { 7856 unsigned long gran = sysctl_sched_wakeup_granularity; 7857 7858 /* 7859 * Since its curr running now, convert the gran from real-time 7860 * to virtual-time in his units. 7861 * 7862 * By using 'se' instead of 'curr' we penalize light tasks, so 7863 * they get preempted easier. That is, if 'se' < 'curr' then 7864 * the resulting gran will be larger, therefore penalizing the 7865 * lighter, if otoh 'se' > 'curr' then the resulting gran will 7866 * be smaller, again penalizing the lighter task. 7867 * 7868 * This is especially important for buddies when the leftmost 7869 * task is higher priority than the buddy. 7870 */ 7871 return calc_delta_fair(gran, se); 7872 } 7873 7874 /* 7875 * Should 'se' preempt 'curr'. 7876 * 7877 * |s1 7878 * |s2 7879 * |s3 7880 * g 7881 * |<--->|c 7882 * 7883 * w(c, s1) = -1 7884 * w(c, s2) = 0 7885 * w(c, s3) = 1 7886 * 7887 */ 7888 static int 7889 wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se) 7890 { 7891 s64 gran, vdiff = curr->vruntime - se->vruntime; 7892 7893 if (vdiff <= 0) 7894 return -1; 7895 7896 gran = wakeup_gran(se); 7897 if (vdiff > gran) 7898 return 1; 7899 7900 return 0; 7901 } 7902 7903 static void set_last_buddy(struct sched_entity *se) 7904 { 7905 for_each_sched_entity(se) { 7906 if (SCHED_WARN_ON(!se->on_rq)) 7907 return; 7908 if (se_is_idle(se)) 7909 return; 7910 cfs_rq_of(se)->last = se; 7911 } 7912 } 7913 7914 static void set_next_buddy(struct sched_entity *se) 7915 { 7916 for_each_sched_entity(se) { 7917 if (SCHED_WARN_ON(!se->on_rq)) 7918 return; 7919 if (se_is_idle(se)) 7920 return; 7921 cfs_rq_of(se)->next = se; 7922 } 7923 } 7924 7925 static void set_skip_buddy(struct sched_entity *se) 7926 { 7927 for_each_sched_entity(se) 7928 cfs_rq_of(se)->skip = se; 7929 } 7930 7931 /* 7932 * Preempt the current task with a newly woken task if needed: 7933 */ 7934 static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_flags) 7935 { 7936 struct task_struct *curr = rq->curr; 7937 struct sched_entity *se = &curr->se, *pse = &p->se; 7938 struct cfs_rq *cfs_rq = task_cfs_rq(curr); 7939 int scale = cfs_rq->nr_running >= sched_nr_latency; 7940 int next_buddy_marked = 0; 7941 int cse_is_idle, pse_is_idle; 7942 7943 if (unlikely(se == pse)) 7944 return; 7945 7946 /* 7947 * This is possible from callers such as attach_tasks(), in which we 7948 * unconditionally check_preempt_curr() after an enqueue (which may have 7949 * lead to a throttle). This both saves work and prevents false 7950 * next-buddy nomination below. 7951 */ 7952 if (unlikely(throttled_hierarchy(cfs_rq_of(pse)))) 7953 return; 7954 7955 if (sched_feat(NEXT_BUDDY) && scale && !(wake_flags & WF_FORK)) { 7956 set_next_buddy(pse); 7957 next_buddy_marked = 1; 7958 } 7959 7960 /* 7961 * We can come here with TIF_NEED_RESCHED already set from new task 7962 * wake up path. 7963 * 7964 * Note: this also catches the edge-case of curr being in a throttled 7965 * group (e.g. via set_curr_task), since update_curr() (in the 7966 * enqueue of curr) will have resulted in resched being set. This 7967 * prevents us from potentially nominating it as a false LAST_BUDDY 7968 * below. 7969 */ 7970 if (test_tsk_need_resched(curr)) 7971 return; 7972 7973 /* Idle tasks are by definition preempted by non-idle tasks. */ 7974 if (unlikely(task_has_idle_policy(curr)) && 7975 likely(!task_has_idle_policy(p))) 7976 goto preempt; 7977 7978 /* 7979 * Batch and idle tasks do not preempt non-idle tasks (their preemption 7980 * is driven by the tick): 7981 */ 7982 if (unlikely(p->policy != SCHED_NORMAL) || !sched_feat(WAKEUP_PREEMPTION)) 7983 return; 7984 7985 find_matching_se(&se, &pse); 7986 WARN_ON_ONCE(!pse); 7987 7988 cse_is_idle = se_is_idle(se); 7989 pse_is_idle = se_is_idle(pse); 7990 7991 /* 7992 * Preempt an idle group in favor of a non-idle group (and don't preempt 7993 * in the inverse case). 7994 */ 7995 if (cse_is_idle && !pse_is_idle) 7996 goto preempt; 7997 if (cse_is_idle != pse_is_idle) 7998 return; 7999 8000 update_curr(cfs_rq_of(se)); 8001 if (wakeup_preempt_entity(se, pse) == 1) { 8002 /* 8003 * Bias pick_next to pick the sched entity that is 8004 * triggering this preemption. 8005 */ 8006 if (!next_buddy_marked) 8007 set_next_buddy(pse); 8008 goto preempt; 8009 } 8010 8011 return; 8012 8013 preempt: 8014 resched_curr(rq); 8015 /* 8016 * Only set the backward buddy when the current task is still 8017 * on the rq. This can happen when a wakeup gets interleaved 8018 * with schedule on the ->pre_schedule() or idle_balance() 8019 * point, either of which can * drop the rq lock. 8020 * 8021 * Also, during early boot the idle thread is in the fair class, 8022 * for obvious reasons its a bad idea to schedule back to it. 8023 */ 8024 if (unlikely(!se->on_rq || curr == rq->idle)) 8025 return; 8026 8027 if (sched_feat(LAST_BUDDY) && scale && entity_is_task(se)) 8028 set_last_buddy(se); 8029 } 8030 8031 #ifdef CONFIG_SMP 8032 static struct task_struct *pick_task_fair(struct rq *rq) 8033 { 8034 struct sched_entity *se; 8035 struct cfs_rq *cfs_rq; 8036 8037 again: 8038 cfs_rq = &rq->cfs; 8039 if (!cfs_rq->nr_running) 8040 return NULL; 8041 8042 do { 8043 struct sched_entity *curr = cfs_rq->curr; 8044 8045 /* When we pick for a remote RQ, we'll not have done put_prev_entity() */ 8046 if (curr) { 8047 if (curr->on_rq) 8048 update_curr(cfs_rq); 8049 else 8050 curr = NULL; 8051 8052 if (unlikely(check_cfs_rq_runtime(cfs_rq))) 8053 goto again; 8054 } 8055 8056 se = pick_next_entity(cfs_rq, curr); 8057 cfs_rq = group_cfs_rq(se); 8058 } while (cfs_rq); 8059 8060 return task_of(se); 8061 } 8062 #endif 8063 8064 struct task_struct * 8065 pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 8066 { 8067 struct cfs_rq *cfs_rq = &rq->cfs; 8068 struct sched_entity *se; 8069 struct task_struct *p; 8070 int new_tasks; 8071 8072 again: 8073 if (!sched_fair_runnable(rq)) 8074 goto idle; 8075 8076 #ifdef CONFIG_FAIR_GROUP_SCHED 8077 if (!prev || prev->sched_class != &fair_sched_class) 8078 goto simple; 8079 8080 /* 8081 * Because of the set_next_buddy() in dequeue_task_fair() it is rather 8082 * likely that a next task is from the same cgroup as the current. 8083 * 8084 * Therefore attempt to avoid putting and setting the entire cgroup 8085 * hierarchy, only change the part that actually changes. 8086 */ 8087 8088 do { 8089 struct sched_entity *curr = cfs_rq->curr; 8090 8091 /* 8092 * Since we got here without doing put_prev_entity() we also 8093 * have to consider cfs_rq->curr. If it is still a runnable 8094 * entity, update_curr() will update its vruntime, otherwise 8095 * forget we've ever seen it. 8096 */ 8097 if (curr) { 8098 if (curr->on_rq) 8099 update_curr(cfs_rq); 8100 else 8101 curr = NULL; 8102 8103 /* 8104 * This call to check_cfs_rq_runtime() will do the 8105 * throttle and dequeue its entity in the parent(s). 8106 * Therefore the nr_running test will indeed 8107 * be correct. 8108 */ 8109 if (unlikely(check_cfs_rq_runtime(cfs_rq))) { 8110 cfs_rq = &rq->cfs; 8111 8112 if (!cfs_rq->nr_running) 8113 goto idle; 8114 8115 goto simple; 8116 } 8117 } 8118 8119 se = pick_next_entity(cfs_rq, curr); 8120 cfs_rq = group_cfs_rq(se); 8121 } while (cfs_rq); 8122 8123 p = task_of(se); 8124 8125 /* 8126 * Since we haven't yet done put_prev_entity and if the selected task 8127 * is a different task than we started out with, try and touch the 8128 * least amount of cfs_rqs. 8129 */ 8130 if (prev != p) { 8131 struct sched_entity *pse = &prev->se; 8132 8133 while (!(cfs_rq = is_same_group(se, pse))) { 8134 int se_depth = se->depth; 8135 int pse_depth = pse->depth; 8136 8137 if (se_depth <= pse_depth) { 8138 put_prev_entity(cfs_rq_of(pse), pse); 8139 pse = parent_entity(pse); 8140 } 8141 if (se_depth >= pse_depth) { 8142 set_next_entity(cfs_rq_of(se), se); 8143 se = parent_entity(se); 8144 } 8145 } 8146 8147 put_prev_entity(cfs_rq, pse); 8148 set_next_entity(cfs_rq, se); 8149 } 8150 8151 goto done; 8152 simple: 8153 #endif 8154 if (prev) 8155 put_prev_task(rq, prev); 8156 8157 do { 8158 se = pick_next_entity(cfs_rq, NULL); 8159 set_next_entity(cfs_rq, se); 8160 cfs_rq = group_cfs_rq(se); 8161 } while (cfs_rq); 8162 8163 p = task_of(se); 8164 8165 done: __maybe_unused; 8166 #ifdef CONFIG_SMP 8167 /* 8168 * Move the next running task to the front of 8169 * the list, so our cfs_tasks list becomes MRU 8170 * one. 8171 */ 8172 list_move(&p->se.group_node, &rq->cfs_tasks); 8173 #endif 8174 8175 if (hrtick_enabled_fair(rq)) 8176 hrtick_start_fair(rq, p); 8177 8178 update_misfit_status(p, rq); 8179 8180 return p; 8181 8182 idle: 8183 if (!rf) 8184 return NULL; 8185 8186 new_tasks = newidle_balance(rq, rf); 8187 8188 /* 8189 * Because newidle_balance() releases (and re-acquires) rq->lock, it is 8190 * possible for any higher priority task to appear. In that case we 8191 * must re-start the pick_next_entity() loop. 8192 */ 8193 if (new_tasks < 0) 8194 return RETRY_TASK; 8195 8196 if (new_tasks > 0) 8197 goto again; 8198 8199 /* 8200 * rq is about to be idle, check if we need to update the 8201 * lost_idle_time of clock_pelt 8202 */ 8203 update_idle_rq_clock_pelt(rq); 8204 8205 return NULL; 8206 } 8207 8208 static struct task_struct *__pick_next_task_fair(struct rq *rq) 8209 { 8210 return pick_next_task_fair(rq, NULL, NULL); 8211 } 8212 8213 /* 8214 * Account for a descheduled task: 8215 */ 8216 static void put_prev_task_fair(struct rq *rq, struct task_struct *prev) 8217 { 8218 struct sched_entity *se = &prev->se; 8219 struct cfs_rq *cfs_rq; 8220 8221 for_each_sched_entity(se) { 8222 cfs_rq = cfs_rq_of(se); 8223 put_prev_entity(cfs_rq, se); 8224 } 8225 } 8226 8227 /* 8228 * sched_yield() is very simple 8229 * 8230 * The magic of dealing with the ->skip buddy is in pick_next_entity. 8231 */ 8232 static void yield_task_fair(struct rq *rq) 8233 { 8234 struct task_struct *curr = rq->curr; 8235 struct cfs_rq *cfs_rq = task_cfs_rq(curr); 8236 struct sched_entity *se = &curr->se; 8237 8238 /* 8239 * Are we the only task in the tree? 8240 */ 8241 if (unlikely(rq->nr_running == 1)) 8242 return; 8243 8244 clear_buddies(cfs_rq, se); 8245 8246 if (curr->policy != SCHED_BATCH) { 8247 update_rq_clock(rq); 8248 /* 8249 * Update run-time statistics of the 'current'. 8250 */ 8251 update_curr(cfs_rq); 8252 /* 8253 * Tell update_rq_clock() that we've just updated, 8254 * so we don't do microscopic update in schedule() 8255 * and double the fastpath cost. 8256 */ 8257 rq_clock_skip_update(rq); 8258 } 8259 8260 set_skip_buddy(se); 8261 } 8262 8263 static bool yield_to_task_fair(struct rq *rq, struct task_struct *p) 8264 { 8265 struct sched_entity *se = &p->se; 8266 8267 /* throttled hierarchies are not runnable */ 8268 if (!se->on_rq || throttled_hierarchy(cfs_rq_of(se))) 8269 return false; 8270 8271 /* Tell the scheduler that we'd really like pse to run next. */ 8272 set_next_buddy(se); 8273 8274 yield_task_fair(rq); 8275 8276 return true; 8277 } 8278 8279 #ifdef CONFIG_SMP 8280 /************************************************** 8281 * Fair scheduling class load-balancing methods. 8282 * 8283 * BASICS 8284 * 8285 * The purpose of load-balancing is to achieve the same basic fairness the 8286 * per-CPU scheduler provides, namely provide a proportional amount of compute 8287 * time to each task. This is expressed in the following equation: 8288 * 8289 * W_i,n/P_i == W_j,n/P_j for all i,j (1) 8290 * 8291 * Where W_i,n is the n-th weight average for CPU i. The instantaneous weight 8292 * W_i,0 is defined as: 8293 * 8294 * W_i,0 = \Sum_j w_i,j (2) 8295 * 8296 * Where w_i,j is the weight of the j-th runnable task on CPU i. This weight 8297 * is derived from the nice value as per sched_prio_to_weight[]. 8298 * 8299 * The weight average is an exponential decay average of the instantaneous 8300 * weight: 8301 * 8302 * W'_i,n = (2^n - 1) / 2^n * W_i,n + 1 / 2^n * W_i,0 (3) 8303 * 8304 * C_i is the compute capacity of CPU i, typically it is the 8305 * fraction of 'recent' time available for SCHED_OTHER task execution. But it 8306 * can also include other factors [XXX]. 8307 * 8308 * To achieve this balance we define a measure of imbalance which follows 8309 * directly from (1): 8310 * 8311 * imb_i,j = max{ avg(W/C), W_i/C_i } - min{ avg(W/C), W_j/C_j } (4) 8312 * 8313 * We them move tasks around to minimize the imbalance. In the continuous 8314 * function space it is obvious this converges, in the discrete case we get 8315 * a few fun cases generally called infeasible weight scenarios. 8316 * 8317 * [XXX expand on: 8318 * - infeasible weights; 8319 * - local vs global optima in the discrete case. ] 8320 * 8321 * 8322 * SCHED DOMAINS 8323 * 8324 * In order to solve the imbalance equation (4), and avoid the obvious O(n^2) 8325 * for all i,j solution, we create a tree of CPUs that follows the hardware 8326 * topology where each level pairs two lower groups (or better). This results 8327 * in O(log n) layers. Furthermore we reduce the number of CPUs going up the 8328 * tree to only the first of the previous level and we decrease the frequency 8329 * of load-balance at each level inv. proportional to the number of CPUs in 8330 * the groups. 8331 * 8332 * This yields: 8333 * 8334 * log_2 n 1 n 8335 * \Sum { --- * --- * 2^i } = O(n) (5) 8336 * i = 0 2^i 2^i 8337 * `- size of each group 8338 * | | `- number of CPUs doing load-balance 8339 * | `- freq 8340 * `- sum over all levels 8341 * 8342 * Coupled with a limit on how many tasks we can migrate every balance pass, 8343 * this makes (5) the runtime complexity of the balancer. 8344 * 8345 * An important property here is that each CPU is still (indirectly) connected 8346 * to every other CPU in at most O(log n) steps: 8347 * 8348 * The adjacency matrix of the resulting graph is given by: 8349 * 8350 * log_2 n 8351 * A_i,j = \Union (i % 2^k == 0) && i / 2^(k+1) == j / 2^(k+1) (6) 8352 * k = 0 8353 * 8354 * And you'll find that: 8355 * 8356 * A^(log_2 n)_i,j != 0 for all i,j (7) 8357 * 8358 * Showing there's indeed a path between every CPU in at most O(log n) steps. 8359 * The task movement gives a factor of O(m), giving a convergence complexity 8360 * of: 8361 * 8362 * O(nm log n), n := nr_cpus, m := nr_tasks (8) 8363 * 8364 * 8365 * WORK CONSERVING 8366 * 8367 * In order to avoid CPUs going idle while there's still work to do, new idle 8368 * balancing is more aggressive and has the newly idle CPU iterate up the domain 8369 * tree itself instead of relying on other CPUs to bring it work. 8370 * 8371 * This adds some complexity to both (5) and (8) but it reduces the total idle 8372 * time. 8373 * 8374 * [XXX more?] 8375 * 8376 * 8377 * CGROUPS 8378 * 8379 * Cgroups make a horror show out of (2), instead of a simple sum we get: 8380 * 8381 * s_k,i 8382 * W_i,0 = \Sum_j \Prod_k w_k * ----- (9) 8383 * S_k 8384 * 8385 * Where 8386 * 8387 * s_k,i = \Sum_j w_i,j,k and S_k = \Sum_i s_k,i (10) 8388 * 8389 * w_i,j,k is the weight of the j-th runnable task in the k-th cgroup on CPU i. 8390 * 8391 * The big problem is S_k, its a global sum needed to compute a local (W_i) 8392 * property. 8393 * 8394 * [XXX write more on how we solve this.. _after_ merging pjt's patches that 8395 * rewrite all of this once again.] 8396 */ 8397 8398 static unsigned long __read_mostly max_load_balance_interval = HZ/10; 8399 8400 enum fbq_type { regular, remote, all }; 8401 8402 /* 8403 * 'group_type' describes the group of CPUs at the moment of load balancing. 8404 * 8405 * The enum is ordered by pulling priority, with the group with lowest priority 8406 * first so the group_type can simply be compared when selecting the busiest 8407 * group. See update_sd_pick_busiest(). 8408 */ 8409 enum group_type { 8410 /* The group has spare capacity that can be used to run more tasks. */ 8411 group_has_spare = 0, 8412 /* 8413 * The group is fully used and the tasks don't compete for more CPU 8414 * cycles. Nevertheless, some tasks might wait before running. 8415 */ 8416 group_fully_busy, 8417 /* 8418 * One task doesn't fit with CPU's capacity and must be migrated to a 8419 * more powerful CPU. 8420 */ 8421 group_misfit_task, 8422 /* 8423 * SD_ASYM_PACKING only: One local CPU with higher capacity is available, 8424 * and the task should be migrated to it instead of running on the 8425 * current CPU. 8426 */ 8427 group_asym_packing, 8428 /* 8429 * The tasks' affinity constraints previously prevented the scheduler 8430 * from balancing the load across the system. 8431 */ 8432 group_imbalanced, 8433 /* 8434 * The CPU is overloaded and can't provide expected CPU cycles to all 8435 * tasks. 8436 */ 8437 group_overloaded 8438 }; 8439 8440 enum migration_type { 8441 migrate_load = 0, 8442 migrate_util, 8443 migrate_task, 8444 migrate_misfit 8445 }; 8446 8447 #define LBF_ALL_PINNED 0x01 8448 #define LBF_NEED_BREAK 0x02 8449 #define LBF_DST_PINNED 0x04 8450 #define LBF_SOME_PINNED 0x08 8451 #define LBF_ACTIVE_LB 0x10 8452 8453 struct lb_env { 8454 struct sched_domain *sd; 8455 8456 struct rq *src_rq; 8457 int src_cpu; 8458 8459 int dst_cpu; 8460 struct rq *dst_rq; 8461 8462 struct cpumask *dst_grpmask; 8463 int new_dst_cpu; 8464 enum cpu_idle_type idle; 8465 long imbalance; 8466 /* The set of CPUs under consideration for load-balancing */ 8467 struct cpumask *cpus; 8468 8469 unsigned int flags; 8470 8471 unsigned int loop; 8472 unsigned int loop_break; 8473 unsigned int loop_max; 8474 8475 enum fbq_type fbq_type; 8476 enum migration_type migration_type; 8477 struct list_head tasks; 8478 }; 8479 8480 /* 8481 * Is this task likely cache-hot: 8482 */ 8483 static int task_hot(struct task_struct *p, struct lb_env *env) 8484 { 8485 s64 delta; 8486 8487 lockdep_assert_rq_held(env->src_rq); 8488 8489 if (p->sched_class != &fair_sched_class) 8490 return 0; 8491 8492 if (unlikely(task_has_idle_policy(p))) 8493 return 0; 8494 8495 /* SMT siblings share cache */ 8496 if (env->sd->flags & SD_SHARE_CPUCAPACITY) 8497 return 0; 8498 8499 /* 8500 * Buddy candidates are cache hot: 8501 */ 8502 if (sched_feat(CACHE_HOT_BUDDY) && env->dst_rq->nr_running && 8503 (&p->se == cfs_rq_of(&p->se)->next || 8504 &p->se == cfs_rq_of(&p->se)->last)) 8505 return 1; 8506 8507 if (sysctl_sched_migration_cost == -1) 8508 return 1; 8509 8510 /* 8511 * Don't migrate task if the task's cookie does not match 8512 * with the destination CPU's core cookie. 8513 */ 8514 if (!sched_core_cookie_match(cpu_rq(env->dst_cpu), p)) 8515 return 1; 8516 8517 if (sysctl_sched_migration_cost == 0) 8518 return 0; 8519 8520 delta = rq_clock_task(env->src_rq) - p->se.exec_start; 8521 8522 return delta < (s64)sysctl_sched_migration_cost; 8523 } 8524 8525 #ifdef CONFIG_NUMA_BALANCING 8526 /* 8527 * Returns 1, if task migration degrades locality 8528 * Returns 0, if task migration improves locality i.e migration preferred. 8529 * Returns -1, if task migration is not affected by locality. 8530 */ 8531 static int migrate_degrades_locality(struct task_struct *p, struct lb_env *env) 8532 { 8533 struct numa_group *numa_group = rcu_dereference(p->numa_group); 8534 unsigned long src_weight, dst_weight; 8535 int src_nid, dst_nid, dist; 8536 8537 if (!static_branch_likely(&sched_numa_balancing)) 8538 return -1; 8539 8540 if (!p->numa_faults || !(env->sd->flags & SD_NUMA)) 8541 return -1; 8542 8543 src_nid = cpu_to_node(env->src_cpu); 8544 dst_nid = cpu_to_node(env->dst_cpu); 8545 8546 if (src_nid == dst_nid) 8547 return -1; 8548 8549 /* Migrating away from the preferred node is always bad. */ 8550 if (src_nid == p->numa_preferred_nid) { 8551 if (env->src_rq->nr_running > env->src_rq->nr_preferred_running) 8552 return 1; 8553 else 8554 return -1; 8555 } 8556 8557 /* Encourage migration to the preferred node. */ 8558 if (dst_nid == p->numa_preferred_nid) 8559 return 0; 8560 8561 /* Leaving a core idle is often worse than degrading locality. */ 8562 if (env->idle == CPU_IDLE) 8563 return -1; 8564 8565 dist = node_distance(src_nid, dst_nid); 8566 if (numa_group) { 8567 src_weight = group_weight(p, src_nid, dist); 8568 dst_weight = group_weight(p, dst_nid, dist); 8569 } else { 8570 src_weight = task_weight(p, src_nid, dist); 8571 dst_weight = task_weight(p, dst_nid, dist); 8572 } 8573 8574 return dst_weight < src_weight; 8575 } 8576 8577 #else 8578 static inline int migrate_degrades_locality(struct task_struct *p, 8579 struct lb_env *env) 8580 { 8581 return -1; 8582 } 8583 #endif 8584 8585 /* 8586 * can_migrate_task - may task p from runqueue rq be migrated to this_cpu? 8587 */ 8588 static 8589 int can_migrate_task(struct task_struct *p, struct lb_env *env) 8590 { 8591 int tsk_cache_hot; 8592 8593 lockdep_assert_rq_held(env->src_rq); 8594 8595 /* 8596 * We do not migrate tasks that are: 8597 * 1) throttled_lb_pair, or 8598 * 2) cannot be migrated to this CPU due to cpus_ptr, or 8599 * 3) running (obviously), or 8600 * 4) are cache-hot on their current CPU. 8601 */ 8602 if (throttled_lb_pair(task_group(p), env->src_cpu, env->dst_cpu)) 8603 return 0; 8604 8605 /* Disregard pcpu kthreads; they are where they need to be. */ 8606 if (kthread_is_per_cpu(p)) 8607 return 0; 8608 8609 if (!cpumask_test_cpu(env->dst_cpu, p->cpus_ptr)) { 8610 int cpu; 8611 8612 schedstat_inc(p->stats.nr_failed_migrations_affine); 8613 8614 env->flags |= LBF_SOME_PINNED; 8615 8616 /* 8617 * Remember if this task can be migrated to any other CPU in 8618 * our sched_group. We may want to revisit it if we couldn't 8619 * meet load balance goals by pulling other tasks on src_cpu. 8620 * 8621 * Avoid computing new_dst_cpu 8622 * - for NEWLY_IDLE 8623 * - if we have already computed one in current iteration 8624 * - if it's an active balance 8625 */ 8626 if (env->idle == CPU_NEWLY_IDLE || 8627 env->flags & (LBF_DST_PINNED | LBF_ACTIVE_LB)) 8628 return 0; 8629 8630 /* Prevent to re-select dst_cpu via env's CPUs: */ 8631 for_each_cpu_and(cpu, env->dst_grpmask, env->cpus) { 8632 if (cpumask_test_cpu(cpu, p->cpus_ptr)) { 8633 env->flags |= LBF_DST_PINNED; 8634 env->new_dst_cpu = cpu; 8635 break; 8636 } 8637 } 8638 8639 return 0; 8640 } 8641 8642 /* Record that we found at least one task that could run on dst_cpu */ 8643 env->flags &= ~LBF_ALL_PINNED; 8644 8645 if (task_on_cpu(env->src_rq, p)) { 8646 schedstat_inc(p->stats.nr_failed_migrations_running); 8647 return 0; 8648 } 8649 8650 /* 8651 * Aggressive migration if: 8652 * 1) active balance 8653 * 2) destination numa is preferred 8654 * 3) task is cache cold, or 8655 * 4) too many balance attempts have failed. 8656 */ 8657 if (env->flags & LBF_ACTIVE_LB) 8658 return 1; 8659 8660 tsk_cache_hot = migrate_degrades_locality(p, env); 8661 if (tsk_cache_hot == -1) 8662 tsk_cache_hot = task_hot(p, env); 8663 8664 if (tsk_cache_hot <= 0 || 8665 env->sd->nr_balance_failed > env->sd->cache_nice_tries) { 8666 if (tsk_cache_hot == 1) { 8667 schedstat_inc(env->sd->lb_hot_gained[env->idle]); 8668 schedstat_inc(p->stats.nr_forced_migrations); 8669 } 8670 return 1; 8671 } 8672 8673 schedstat_inc(p->stats.nr_failed_migrations_hot); 8674 return 0; 8675 } 8676 8677 /* 8678 * detach_task() -- detach the task for the migration specified in env 8679 */ 8680 static void detach_task(struct task_struct *p, struct lb_env *env) 8681 { 8682 lockdep_assert_rq_held(env->src_rq); 8683 8684 deactivate_task(env->src_rq, p, DEQUEUE_NOCLOCK); 8685 set_task_cpu(p, env->dst_cpu); 8686 } 8687 8688 /* 8689 * detach_one_task() -- tries to dequeue exactly one task from env->src_rq, as 8690 * part of active balancing operations within "domain". 8691 * 8692 * Returns a task if successful and NULL otherwise. 8693 */ 8694 static struct task_struct *detach_one_task(struct lb_env *env) 8695 { 8696 struct task_struct *p; 8697 8698 lockdep_assert_rq_held(env->src_rq); 8699 8700 list_for_each_entry_reverse(p, 8701 &env->src_rq->cfs_tasks, se.group_node) { 8702 if (!can_migrate_task(p, env)) 8703 continue; 8704 8705 detach_task(p, env); 8706 8707 /* 8708 * Right now, this is only the second place where 8709 * lb_gained[env->idle] is updated (other is detach_tasks) 8710 * so we can safely collect stats here rather than 8711 * inside detach_tasks(). 8712 */ 8713 schedstat_inc(env->sd->lb_gained[env->idle]); 8714 return p; 8715 } 8716 return NULL; 8717 } 8718 8719 /* 8720 * detach_tasks() -- tries to detach up to imbalance load/util/tasks from 8721 * busiest_rq, as part of a balancing operation within domain "sd". 8722 * 8723 * Returns number of detached tasks if successful and 0 otherwise. 8724 */ 8725 static int detach_tasks(struct lb_env *env) 8726 { 8727 struct list_head *tasks = &env->src_rq->cfs_tasks; 8728 unsigned long util, load; 8729 struct task_struct *p; 8730 int detached = 0; 8731 8732 lockdep_assert_rq_held(env->src_rq); 8733 8734 /* 8735 * Source run queue has been emptied by another CPU, clear 8736 * LBF_ALL_PINNED flag as we will not test any task. 8737 */ 8738 if (env->src_rq->nr_running <= 1) { 8739 env->flags &= ~LBF_ALL_PINNED; 8740 return 0; 8741 } 8742 8743 if (env->imbalance <= 0) 8744 return 0; 8745 8746 while (!list_empty(tasks)) { 8747 /* 8748 * We don't want to steal all, otherwise we may be treated likewise, 8749 * which could at worst lead to a livelock crash. 8750 */ 8751 if (env->idle != CPU_NOT_IDLE && env->src_rq->nr_running <= 1) 8752 break; 8753 8754 env->loop++; 8755 /* 8756 * We've more or less seen every task there is, call it quits 8757 * unless we haven't found any movable task yet. 8758 */ 8759 if (env->loop > env->loop_max && 8760 !(env->flags & LBF_ALL_PINNED)) 8761 break; 8762 8763 /* take a breather every nr_migrate tasks */ 8764 if (env->loop > env->loop_break) { 8765 env->loop_break += SCHED_NR_MIGRATE_BREAK; 8766 env->flags |= LBF_NEED_BREAK; 8767 break; 8768 } 8769 8770 p = list_last_entry(tasks, struct task_struct, se.group_node); 8771 8772 if (!can_migrate_task(p, env)) 8773 goto next; 8774 8775 switch (env->migration_type) { 8776 case migrate_load: 8777 /* 8778 * Depending of the number of CPUs and tasks and the 8779 * cgroup hierarchy, task_h_load() can return a null 8780 * value. Make sure that env->imbalance decreases 8781 * otherwise detach_tasks() will stop only after 8782 * detaching up to loop_max tasks. 8783 */ 8784 load = max_t(unsigned long, task_h_load(p), 1); 8785 8786 if (sched_feat(LB_MIN) && 8787 load < 16 && !env->sd->nr_balance_failed) 8788 goto next; 8789 8790 /* 8791 * Make sure that we don't migrate too much load. 8792 * Nevertheless, let relax the constraint if 8793 * scheduler fails to find a good waiting task to 8794 * migrate. 8795 */ 8796 if (shr_bound(load, env->sd->nr_balance_failed) > env->imbalance) 8797 goto next; 8798 8799 env->imbalance -= load; 8800 break; 8801 8802 case migrate_util: 8803 util = task_util_est(p); 8804 8805 if (util > env->imbalance) 8806 goto next; 8807 8808 env->imbalance -= util; 8809 break; 8810 8811 case migrate_task: 8812 env->imbalance--; 8813 break; 8814 8815 case migrate_misfit: 8816 /* This is not a misfit task */ 8817 if (task_fits_cpu(p, env->src_cpu)) 8818 goto next; 8819 8820 env->imbalance = 0; 8821 break; 8822 } 8823 8824 detach_task(p, env); 8825 list_add(&p->se.group_node, &env->tasks); 8826 8827 detached++; 8828 8829 #ifdef CONFIG_PREEMPTION 8830 /* 8831 * NEWIDLE balancing is a source of latency, so preemptible 8832 * kernels will stop after the first task is detached to minimize 8833 * the critical section. 8834 */ 8835 if (env->idle == CPU_NEWLY_IDLE) 8836 break; 8837 #endif 8838 8839 /* 8840 * We only want to steal up to the prescribed amount of 8841 * load/util/tasks. 8842 */ 8843 if (env->imbalance <= 0) 8844 break; 8845 8846 continue; 8847 next: 8848 list_move(&p->se.group_node, tasks); 8849 } 8850 8851 /* 8852 * Right now, this is one of only two places we collect this stat 8853 * so we can safely collect detach_one_task() stats here rather 8854 * than inside detach_one_task(). 8855 */ 8856 schedstat_add(env->sd->lb_gained[env->idle], detached); 8857 8858 return detached; 8859 } 8860 8861 /* 8862 * attach_task() -- attach the task detached by detach_task() to its new rq. 8863 */ 8864 static void attach_task(struct rq *rq, struct task_struct *p) 8865 { 8866 lockdep_assert_rq_held(rq); 8867 8868 WARN_ON_ONCE(task_rq(p) != rq); 8869 activate_task(rq, p, ENQUEUE_NOCLOCK); 8870 check_preempt_curr(rq, p, 0); 8871 } 8872 8873 /* 8874 * attach_one_task() -- attaches the task returned from detach_one_task() to 8875 * its new rq. 8876 */ 8877 static void attach_one_task(struct rq *rq, struct task_struct *p) 8878 { 8879 struct rq_flags rf; 8880 8881 rq_lock(rq, &rf); 8882 update_rq_clock(rq); 8883 attach_task(rq, p); 8884 rq_unlock(rq, &rf); 8885 } 8886 8887 /* 8888 * attach_tasks() -- attaches all tasks detached by detach_tasks() to their 8889 * new rq. 8890 */ 8891 static void attach_tasks(struct lb_env *env) 8892 { 8893 struct list_head *tasks = &env->tasks; 8894 struct task_struct *p; 8895 struct rq_flags rf; 8896 8897 rq_lock(env->dst_rq, &rf); 8898 update_rq_clock(env->dst_rq); 8899 8900 while (!list_empty(tasks)) { 8901 p = list_first_entry(tasks, struct task_struct, se.group_node); 8902 list_del_init(&p->se.group_node); 8903 8904 attach_task(env->dst_rq, p); 8905 } 8906 8907 rq_unlock(env->dst_rq, &rf); 8908 } 8909 8910 #ifdef CONFIG_NO_HZ_COMMON 8911 static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq) 8912 { 8913 if (cfs_rq->avg.load_avg) 8914 return true; 8915 8916 if (cfs_rq->avg.util_avg) 8917 return true; 8918 8919 return false; 8920 } 8921 8922 static inline bool others_have_blocked(struct rq *rq) 8923 { 8924 if (READ_ONCE(rq->avg_rt.util_avg)) 8925 return true; 8926 8927 if (READ_ONCE(rq->avg_dl.util_avg)) 8928 return true; 8929 8930 if (thermal_load_avg(rq)) 8931 return true; 8932 8933 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ 8934 if (READ_ONCE(rq->avg_irq.util_avg)) 8935 return true; 8936 #endif 8937 8938 return false; 8939 } 8940 8941 static inline void update_blocked_load_tick(struct rq *rq) 8942 { 8943 WRITE_ONCE(rq->last_blocked_load_update_tick, jiffies); 8944 } 8945 8946 static inline void update_blocked_load_status(struct rq *rq, bool has_blocked) 8947 { 8948 if (!has_blocked) 8949 rq->has_blocked_load = 0; 8950 } 8951 #else 8952 static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq) { return false; } 8953 static inline bool others_have_blocked(struct rq *rq) { return false; } 8954 static inline void update_blocked_load_tick(struct rq *rq) {} 8955 static inline void update_blocked_load_status(struct rq *rq, bool has_blocked) {} 8956 #endif 8957 8958 static bool __update_blocked_others(struct rq *rq, bool *done) 8959 { 8960 const struct sched_class *curr_class; 8961 u64 now = rq_clock_pelt(rq); 8962 unsigned long thermal_pressure; 8963 bool decayed; 8964 8965 /* 8966 * update_load_avg() can call cpufreq_update_util(). Make sure that RT, 8967 * DL and IRQ signals have been updated before updating CFS. 8968 */ 8969 curr_class = rq->curr->sched_class; 8970 8971 thermal_pressure = arch_scale_thermal_pressure(cpu_of(rq)); 8972 8973 decayed = update_rt_rq_load_avg(now, rq, curr_class == &rt_sched_class) | 8974 update_dl_rq_load_avg(now, rq, curr_class == &dl_sched_class) | 8975 update_thermal_load_avg(rq_clock_thermal(rq), rq, thermal_pressure) | 8976 update_irq_load_avg(rq, 0); 8977 8978 if (others_have_blocked(rq)) 8979 *done = false; 8980 8981 return decayed; 8982 } 8983 8984 #ifdef CONFIG_FAIR_GROUP_SCHED 8985 8986 static bool __update_blocked_fair(struct rq *rq, bool *done) 8987 { 8988 struct cfs_rq *cfs_rq, *pos; 8989 bool decayed = false; 8990 int cpu = cpu_of(rq); 8991 8992 /* 8993 * Iterates the task_group tree in a bottom up fashion, see 8994 * list_add_leaf_cfs_rq() for details. 8995 */ 8996 for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) { 8997 struct sched_entity *se; 8998 8999 if (update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq)) { 9000 update_tg_load_avg(cfs_rq); 9001 9002 if (cfs_rq->nr_running == 0) 9003 update_idle_cfs_rq_clock_pelt(cfs_rq); 9004 9005 if (cfs_rq == &rq->cfs) 9006 decayed = true; 9007 } 9008 9009 /* Propagate pending load changes to the parent, if any: */ 9010 se = cfs_rq->tg->se[cpu]; 9011 if (se && !skip_blocked_update(se)) 9012 update_load_avg(cfs_rq_of(se), se, UPDATE_TG); 9013 9014 /* 9015 * There can be a lot of idle CPU cgroups. Don't let fully 9016 * decayed cfs_rqs linger on the list. 9017 */ 9018 if (cfs_rq_is_decayed(cfs_rq)) 9019 list_del_leaf_cfs_rq(cfs_rq); 9020 9021 /* Don't need periodic decay once load/util_avg are null */ 9022 if (cfs_rq_has_blocked(cfs_rq)) 9023 *done = false; 9024 } 9025 9026 return decayed; 9027 } 9028 9029 /* 9030 * Compute the hierarchical load factor for cfs_rq and all its ascendants. 9031 * This needs to be done in a top-down fashion because the load of a child 9032 * group is a fraction of its parents load. 9033 */ 9034 static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq) 9035 { 9036 struct rq *rq = rq_of(cfs_rq); 9037 struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)]; 9038 unsigned long now = jiffies; 9039 unsigned long load; 9040 9041 if (cfs_rq->last_h_load_update == now) 9042 return; 9043 9044 WRITE_ONCE(cfs_rq->h_load_next, NULL); 9045 for_each_sched_entity(se) { 9046 cfs_rq = cfs_rq_of(se); 9047 WRITE_ONCE(cfs_rq->h_load_next, se); 9048 if (cfs_rq->last_h_load_update == now) 9049 break; 9050 } 9051 9052 if (!se) { 9053 cfs_rq->h_load = cfs_rq_load_avg(cfs_rq); 9054 cfs_rq->last_h_load_update = now; 9055 } 9056 9057 while ((se = READ_ONCE(cfs_rq->h_load_next)) != NULL) { 9058 load = cfs_rq->h_load; 9059 load = div64_ul(load * se->avg.load_avg, 9060 cfs_rq_load_avg(cfs_rq) + 1); 9061 cfs_rq = group_cfs_rq(se); 9062 cfs_rq->h_load = load; 9063 cfs_rq->last_h_load_update = now; 9064 } 9065 } 9066 9067 static unsigned long task_h_load(struct task_struct *p) 9068 { 9069 struct cfs_rq *cfs_rq = task_cfs_rq(p); 9070 9071 update_cfs_rq_h_load(cfs_rq); 9072 return div64_ul(p->se.avg.load_avg * cfs_rq->h_load, 9073 cfs_rq_load_avg(cfs_rq) + 1); 9074 } 9075 #else 9076 static bool __update_blocked_fair(struct rq *rq, bool *done) 9077 { 9078 struct cfs_rq *cfs_rq = &rq->cfs; 9079 bool decayed; 9080 9081 decayed = update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq); 9082 if (cfs_rq_has_blocked(cfs_rq)) 9083 *done = false; 9084 9085 return decayed; 9086 } 9087 9088 static unsigned long task_h_load(struct task_struct *p) 9089 { 9090 return p->se.avg.load_avg; 9091 } 9092 #endif 9093 9094 static void update_blocked_averages(int cpu) 9095 { 9096 bool decayed = false, done = true; 9097 struct rq *rq = cpu_rq(cpu); 9098 struct rq_flags rf; 9099 9100 rq_lock_irqsave(rq, &rf); 9101 update_blocked_load_tick(rq); 9102 update_rq_clock(rq); 9103 9104 decayed |= __update_blocked_others(rq, &done); 9105 decayed |= __update_blocked_fair(rq, &done); 9106 9107 update_blocked_load_status(rq, !done); 9108 if (decayed) 9109 cpufreq_update_util(rq, 0); 9110 rq_unlock_irqrestore(rq, &rf); 9111 } 9112 9113 /********** Helpers for find_busiest_group ************************/ 9114 9115 /* 9116 * sg_lb_stats - stats of a sched_group required for load_balancing 9117 */ 9118 struct sg_lb_stats { 9119 unsigned long avg_load; /*Avg load across the CPUs of the group */ 9120 unsigned long group_load; /* Total load over the CPUs of the group */ 9121 unsigned long group_capacity; 9122 unsigned long group_util; /* Total utilization over the CPUs of the group */ 9123 unsigned long group_runnable; /* Total runnable time over the CPUs of the group */ 9124 unsigned int sum_nr_running; /* Nr of tasks running in the group */ 9125 unsigned int sum_h_nr_running; /* Nr of CFS tasks running in the group */ 9126 unsigned int idle_cpus; 9127 unsigned int group_weight; 9128 enum group_type group_type; 9129 unsigned int group_asym_packing; /* Tasks should be moved to preferred CPU */ 9130 unsigned long group_misfit_task_load; /* A CPU has a task too big for its capacity */ 9131 #ifdef CONFIG_NUMA_BALANCING 9132 unsigned int nr_numa_running; 9133 unsigned int nr_preferred_running; 9134 #endif 9135 }; 9136 9137 /* 9138 * sd_lb_stats - Structure to store the statistics of a sched_domain 9139 * during load balancing. 9140 */ 9141 struct sd_lb_stats { 9142 struct sched_group *busiest; /* Busiest group in this sd */ 9143 struct sched_group *local; /* Local group in this sd */ 9144 unsigned long total_load; /* Total load of all groups in sd */ 9145 unsigned long total_capacity; /* Total capacity of all groups in sd */ 9146 unsigned long avg_load; /* Average load across all groups in sd */ 9147 unsigned int prefer_sibling; /* tasks should go to sibling first */ 9148 9149 struct sg_lb_stats busiest_stat;/* Statistics of the busiest group */ 9150 struct sg_lb_stats local_stat; /* Statistics of the local group */ 9151 }; 9152 9153 static inline void init_sd_lb_stats(struct sd_lb_stats *sds) 9154 { 9155 /* 9156 * Skimp on the clearing to avoid duplicate work. We can avoid clearing 9157 * local_stat because update_sg_lb_stats() does a full clear/assignment. 9158 * We must however set busiest_stat::group_type and 9159 * busiest_stat::idle_cpus to the worst busiest group because 9160 * update_sd_pick_busiest() reads these before assignment. 9161 */ 9162 *sds = (struct sd_lb_stats){ 9163 .busiest = NULL, 9164 .local = NULL, 9165 .total_load = 0UL, 9166 .total_capacity = 0UL, 9167 .busiest_stat = { 9168 .idle_cpus = UINT_MAX, 9169 .group_type = group_has_spare, 9170 }, 9171 }; 9172 } 9173 9174 static unsigned long scale_rt_capacity(int cpu) 9175 { 9176 struct rq *rq = cpu_rq(cpu); 9177 unsigned long max = arch_scale_cpu_capacity(cpu); 9178 unsigned long used, free; 9179 unsigned long irq; 9180 9181 irq = cpu_util_irq(rq); 9182 9183 if (unlikely(irq >= max)) 9184 return 1; 9185 9186 /* 9187 * avg_rt.util_avg and avg_dl.util_avg track binary signals 9188 * (running and not running) with weights 0 and 1024 respectively. 9189 * avg_thermal.load_avg tracks thermal pressure and the weighted 9190 * average uses the actual delta max capacity(load). 9191 */ 9192 used = READ_ONCE(rq->avg_rt.util_avg); 9193 used += READ_ONCE(rq->avg_dl.util_avg); 9194 used += thermal_load_avg(rq); 9195 9196 if (unlikely(used >= max)) 9197 return 1; 9198 9199 free = max - used; 9200 9201 return scale_irq_capacity(free, irq, max); 9202 } 9203 9204 static void update_cpu_capacity(struct sched_domain *sd, int cpu) 9205 { 9206 unsigned long capacity = scale_rt_capacity(cpu); 9207 struct sched_group *sdg = sd->groups; 9208 9209 cpu_rq(cpu)->cpu_capacity_orig = arch_scale_cpu_capacity(cpu); 9210 9211 if (!capacity) 9212 capacity = 1; 9213 9214 cpu_rq(cpu)->cpu_capacity = capacity; 9215 trace_sched_cpu_capacity_tp(cpu_rq(cpu)); 9216 9217 sdg->sgc->capacity = capacity; 9218 sdg->sgc->min_capacity = capacity; 9219 sdg->sgc->max_capacity = capacity; 9220 } 9221 9222 void update_group_capacity(struct sched_domain *sd, int cpu) 9223 { 9224 struct sched_domain *child = sd->child; 9225 struct sched_group *group, *sdg = sd->groups; 9226 unsigned long capacity, min_capacity, max_capacity; 9227 unsigned long interval; 9228 9229 interval = msecs_to_jiffies(sd->balance_interval); 9230 interval = clamp(interval, 1UL, max_load_balance_interval); 9231 sdg->sgc->next_update = jiffies + interval; 9232 9233 if (!child) { 9234 update_cpu_capacity(sd, cpu); 9235 return; 9236 } 9237 9238 capacity = 0; 9239 min_capacity = ULONG_MAX; 9240 max_capacity = 0; 9241 9242 if (child->flags & SD_OVERLAP) { 9243 /* 9244 * SD_OVERLAP domains cannot assume that child groups 9245 * span the current group. 9246 */ 9247 9248 for_each_cpu(cpu, sched_group_span(sdg)) { 9249 unsigned long cpu_cap = capacity_of(cpu); 9250 9251 capacity += cpu_cap; 9252 min_capacity = min(cpu_cap, min_capacity); 9253 max_capacity = max(cpu_cap, max_capacity); 9254 } 9255 } else { 9256 /* 9257 * !SD_OVERLAP domains can assume that child groups 9258 * span the current group. 9259 */ 9260 9261 group = child->groups; 9262 do { 9263 struct sched_group_capacity *sgc = group->sgc; 9264 9265 capacity += sgc->capacity; 9266 min_capacity = min(sgc->min_capacity, min_capacity); 9267 max_capacity = max(sgc->max_capacity, max_capacity); 9268 group = group->next; 9269 } while (group != child->groups); 9270 } 9271 9272 sdg->sgc->capacity = capacity; 9273 sdg->sgc->min_capacity = min_capacity; 9274 sdg->sgc->max_capacity = max_capacity; 9275 } 9276 9277 /* 9278 * Check whether the capacity of the rq has been noticeably reduced by side 9279 * activity. The imbalance_pct is used for the threshold. 9280 * Return true is the capacity is reduced 9281 */ 9282 static inline int 9283 check_cpu_capacity(struct rq *rq, struct sched_domain *sd) 9284 { 9285 return ((rq->cpu_capacity * sd->imbalance_pct) < 9286 (rq->cpu_capacity_orig * 100)); 9287 } 9288 9289 /* 9290 * Check whether a rq has a misfit task and if it looks like we can actually 9291 * help that task: we can migrate the task to a CPU of higher capacity, or 9292 * the task's current CPU is heavily pressured. 9293 */ 9294 static inline int check_misfit_status(struct rq *rq, struct sched_domain *sd) 9295 { 9296 return rq->misfit_task_load && 9297 (rq->cpu_capacity_orig < rq->rd->max_cpu_capacity || 9298 check_cpu_capacity(rq, sd)); 9299 } 9300 9301 /* 9302 * Group imbalance indicates (and tries to solve) the problem where balancing 9303 * groups is inadequate due to ->cpus_ptr constraints. 9304 * 9305 * Imagine a situation of two groups of 4 CPUs each and 4 tasks each with a 9306 * cpumask covering 1 CPU of the first group and 3 CPUs of the second group. 9307 * Something like: 9308 * 9309 * { 0 1 2 3 } { 4 5 6 7 } 9310 * * * * * 9311 * 9312 * If we were to balance group-wise we'd place two tasks in the first group and 9313 * two tasks in the second group. Clearly this is undesired as it will overload 9314 * cpu 3 and leave one of the CPUs in the second group unused. 9315 * 9316 * The current solution to this issue is detecting the skew in the first group 9317 * by noticing the lower domain failed to reach balance and had difficulty 9318 * moving tasks due to affinity constraints. 9319 * 9320 * When this is so detected; this group becomes a candidate for busiest; see 9321 * update_sd_pick_busiest(). And calculate_imbalance() and 9322 * find_busiest_group() avoid some of the usual balance conditions to allow it 9323 * to create an effective group imbalance. 9324 * 9325 * This is a somewhat tricky proposition since the next run might not find the 9326 * group imbalance and decide the groups need to be balanced again. A most 9327 * subtle and fragile situation. 9328 */ 9329 9330 static inline int sg_imbalanced(struct sched_group *group) 9331 { 9332 return group->sgc->imbalance; 9333 } 9334 9335 /* 9336 * group_has_capacity returns true if the group has spare capacity that could 9337 * be used by some tasks. 9338 * We consider that a group has spare capacity if the number of task is 9339 * smaller than the number of CPUs or if the utilization is lower than the 9340 * available capacity for CFS tasks. 9341 * For the latter, we use a threshold to stabilize the state, to take into 9342 * account the variance of the tasks' load and to return true if the available 9343 * capacity in meaningful for the load balancer. 9344 * As an example, an available capacity of 1% can appear but it doesn't make 9345 * any benefit for the load balance. 9346 */ 9347 static inline bool 9348 group_has_capacity(unsigned int imbalance_pct, struct sg_lb_stats *sgs) 9349 { 9350 if (sgs->sum_nr_running < sgs->group_weight) 9351 return true; 9352 9353 if ((sgs->group_capacity * imbalance_pct) < 9354 (sgs->group_runnable * 100)) 9355 return false; 9356 9357 if ((sgs->group_capacity * 100) > 9358 (sgs->group_util * imbalance_pct)) 9359 return true; 9360 9361 return false; 9362 } 9363 9364 /* 9365 * group_is_overloaded returns true if the group has more tasks than it can 9366 * handle. 9367 * group_is_overloaded is not equals to !group_has_capacity because a group 9368 * with the exact right number of tasks, has no more spare capacity but is not 9369 * overloaded so both group_has_capacity and group_is_overloaded return 9370 * false. 9371 */ 9372 static inline bool 9373 group_is_overloaded(unsigned int imbalance_pct, struct sg_lb_stats *sgs) 9374 { 9375 if (sgs->sum_nr_running <= sgs->group_weight) 9376 return false; 9377 9378 if ((sgs->group_capacity * 100) < 9379 (sgs->group_util * imbalance_pct)) 9380 return true; 9381 9382 if ((sgs->group_capacity * imbalance_pct) < 9383 (sgs->group_runnable * 100)) 9384 return true; 9385 9386 return false; 9387 } 9388 9389 static inline enum 9390 group_type group_classify(unsigned int imbalance_pct, 9391 struct sched_group *group, 9392 struct sg_lb_stats *sgs) 9393 { 9394 if (group_is_overloaded(imbalance_pct, sgs)) 9395 return group_overloaded; 9396 9397 if (sg_imbalanced(group)) 9398 return group_imbalanced; 9399 9400 if (sgs->group_asym_packing) 9401 return group_asym_packing; 9402 9403 if (sgs->group_misfit_task_load) 9404 return group_misfit_task; 9405 9406 if (!group_has_capacity(imbalance_pct, sgs)) 9407 return group_fully_busy; 9408 9409 return group_has_spare; 9410 } 9411 9412 /** 9413 * sched_use_asym_prio - Check whether asym_packing priority must be used 9414 * @sd: The scheduling domain of the load balancing 9415 * @cpu: A CPU 9416 * 9417 * Always use CPU priority when balancing load between SMT siblings. When 9418 * balancing load between cores, it is not sufficient that @cpu is idle. Only 9419 * use CPU priority if the whole core is idle. 9420 * 9421 * Returns: True if the priority of @cpu must be followed. False otherwise. 9422 */ 9423 static bool sched_use_asym_prio(struct sched_domain *sd, int cpu) 9424 { 9425 if (!sched_smt_active()) 9426 return true; 9427 9428 return sd->flags & SD_SHARE_CPUCAPACITY || is_core_idle(cpu); 9429 } 9430 9431 /** 9432 * sched_asym - Check if the destination CPU can do asym_packing load balance 9433 * @env: The load balancing environment 9434 * @sds: Load-balancing data with statistics of the local group 9435 * @sgs: Load-balancing statistics of the candidate busiest group 9436 * @group: The candidate busiest group 9437 * 9438 * @env::dst_cpu can do asym_packing if it has higher priority than the 9439 * preferred CPU of @group. 9440 * 9441 * SMT is a special case. If we are balancing load between cores, @env::dst_cpu 9442 * can do asym_packing balance only if all its SMT siblings are idle. Also, it 9443 * can only do it if @group is an SMT group and has exactly on busy CPU. Larger 9444 * imbalances in the number of CPUS are dealt with in find_busiest_group(). 9445 * 9446 * If we are balancing load within an SMT core, or at DIE domain level, always 9447 * proceed. 9448 * 9449 * Return: true if @env::dst_cpu can do with asym_packing load balance. False 9450 * otherwise. 9451 */ 9452 static inline bool 9453 sched_asym(struct lb_env *env, struct sd_lb_stats *sds, struct sg_lb_stats *sgs, 9454 struct sched_group *group) 9455 { 9456 /* Ensure that the whole local core is idle, if applicable. */ 9457 if (!sched_use_asym_prio(env->sd, env->dst_cpu)) 9458 return false; 9459 9460 /* 9461 * CPU priorities does not make sense for SMT cores with more than one 9462 * busy sibling. 9463 */ 9464 if (group->flags & SD_SHARE_CPUCAPACITY) { 9465 if (sgs->group_weight - sgs->idle_cpus != 1) 9466 return false; 9467 } 9468 9469 return sched_asym_prefer(env->dst_cpu, group->asym_prefer_cpu); 9470 } 9471 9472 static inline bool 9473 sched_reduced_capacity(struct rq *rq, struct sched_domain *sd) 9474 { 9475 /* 9476 * When there is more than 1 task, the group_overloaded case already 9477 * takes care of cpu with reduced capacity 9478 */ 9479 if (rq->cfs.h_nr_running != 1) 9480 return false; 9481 9482 return check_cpu_capacity(rq, sd); 9483 } 9484 9485 /** 9486 * update_sg_lb_stats - Update sched_group's statistics for load balancing. 9487 * @env: The load balancing environment. 9488 * @sds: Load-balancing data with statistics of the local group. 9489 * @group: sched_group whose statistics are to be updated. 9490 * @sgs: variable to hold the statistics for this group. 9491 * @sg_status: Holds flag indicating the status of the sched_group 9492 */ 9493 static inline void update_sg_lb_stats(struct lb_env *env, 9494 struct sd_lb_stats *sds, 9495 struct sched_group *group, 9496 struct sg_lb_stats *sgs, 9497 int *sg_status) 9498 { 9499 int i, nr_running, local_group; 9500 9501 memset(sgs, 0, sizeof(*sgs)); 9502 9503 local_group = group == sds->local; 9504 9505 for_each_cpu_and(i, sched_group_span(group), env->cpus) { 9506 struct rq *rq = cpu_rq(i); 9507 unsigned long load = cpu_load(rq); 9508 9509 sgs->group_load += load; 9510 sgs->group_util += cpu_util_cfs(i); 9511 sgs->group_runnable += cpu_runnable(rq); 9512 sgs->sum_h_nr_running += rq->cfs.h_nr_running; 9513 9514 nr_running = rq->nr_running; 9515 sgs->sum_nr_running += nr_running; 9516 9517 if (nr_running > 1) 9518 *sg_status |= SG_OVERLOAD; 9519 9520 if (cpu_overutilized(i)) 9521 *sg_status |= SG_OVERUTILIZED; 9522 9523 #ifdef CONFIG_NUMA_BALANCING 9524 sgs->nr_numa_running += rq->nr_numa_running; 9525 sgs->nr_preferred_running += rq->nr_preferred_running; 9526 #endif 9527 /* 9528 * No need to call idle_cpu() if nr_running is not 0 9529 */ 9530 if (!nr_running && idle_cpu(i)) { 9531 sgs->idle_cpus++; 9532 /* Idle cpu can't have misfit task */ 9533 continue; 9534 } 9535 9536 if (local_group) 9537 continue; 9538 9539 if (env->sd->flags & SD_ASYM_CPUCAPACITY) { 9540 /* Check for a misfit task on the cpu */ 9541 if (sgs->group_misfit_task_load < rq->misfit_task_load) { 9542 sgs->group_misfit_task_load = rq->misfit_task_load; 9543 *sg_status |= SG_OVERLOAD; 9544 } 9545 } else if ((env->idle != CPU_NOT_IDLE) && 9546 sched_reduced_capacity(rq, env->sd)) { 9547 /* Check for a task running on a CPU with reduced capacity */ 9548 if (sgs->group_misfit_task_load < load) 9549 sgs->group_misfit_task_load = load; 9550 } 9551 } 9552 9553 sgs->group_capacity = group->sgc->capacity; 9554 9555 sgs->group_weight = group->group_weight; 9556 9557 /* Check if dst CPU is idle and preferred to this group */ 9558 if (!local_group && env->sd->flags & SD_ASYM_PACKING && 9559 env->idle != CPU_NOT_IDLE && sgs->sum_h_nr_running && 9560 sched_asym(env, sds, sgs, group)) { 9561 sgs->group_asym_packing = 1; 9562 } 9563 9564 sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs); 9565 9566 /* Computing avg_load makes sense only when group is overloaded */ 9567 if (sgs->group_type == group_overloaded) 9568 sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) / 9569 sgs->group_capacity; 9570 } 9571 9572 /** 9573 * update_sd_pick_busiest - return 1 on busiest group 9574 * @env: The load balancing environment. 9575 * @sds: sched_domain statistics 9576 * @sg: sched_group candidate to be checked for being the busiest 9577 * @sgs: sched_group statistics 9578 * 9579 * Determine if @sg is a busier group than the previously selected 9580 * busiest group. 9581 * 9582 * Return: %true if @sg is a busier group than the previously selected 9583 * busiest group. %false otherwise. 9584 */ 9585 static bool update_sd_pick_busiest(struct lb_env *env, 9586 struct sd_lb_stats *sds, 9587 struct sched_group *sg, 9588 struct sg_lb_stats *sgs) 9589 { 9590 struct sg_lb_stats *busiest = &sds->busiest_stat; 9591 9592 /* Make sure that there is at least one task to pull */ 9593 if (!sgs->sum_h_nr_running) 9594 return false; 9595 9596 /* 9597 * Don't try to pull misfit tasks we can't help. 9598 * We can use max_capacity here as reduction in capacity on some 9599 * CPUs in the group should either be possible to resolve 9600 * internally or be covered by avg_load imbalance (eventually). 9601 */ 9602 if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && 9603 (sgs->group_type == group_misfit_task) && 9604 (!capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) || 9605 sds->local_stat.group_type != group_has_spare)) 9606 return false; 9607 9608 if (sgs->group_type > busiest->group_type) 9609 return true; 9610 9611 if (sgs->group_type < busiest->group_type) 9612 return false; 9613 9614 /* 9615 * The candidate and the current busiest group are the same type of 9616 * group. Let check which one is the busiest according to the type. 9617 */ 9618 9619 switch (sgs->group_type) { 9620 case group_overloaded: 9621 /* Select the overloaded group with highest avg_load. */ 9622 if (sgs->avg_load <= busiest->avg_load) 9623 return false; 9624 break; 9625 9626 case group_imbalanced: 9627 /* 9628 * Select the 1st imbalanced group as we don't have any way to 9629 * choose one more than another. 9630 */ 9631 return false; 9632 9633 case group_asym_packing: 9634 /* Prefer to move from lowest priority CPU's work */ 9635 if (sched_asym_prefer(sg->asym_prefer_cpu, sds->busiest->asym_prefer_cpu)) 9636 return false; 9637 break; 9638 9639 case group_misfit_task: 9640 /* 9641 * If we have more than one misfit sg go with the biggest 9642 * misfit. 9643 */ 9644 if (sgs->group_misfit_task_load < busiest->group_misfit_task_load) 9645 return false; 9646 break; 9647 9648 case group_fully_busy: 9649 /* 9650 * Select the fully busy group with highest avg_load. In 9651 * theory, there is no need to pull task from such kind of 9652 * group because tasks have all compute capacity that they need 9653 * but we can still improve the overall throughput by reducing 9654 * contention when accessing shared HW resources. 9655 * 9656 * XXX for now avg_load is not computed and always 0 so we 9657 * select the 1st one, except if @sg is composed of SMT 9658 * siblings. 9659 */ 9660 9661 if (sgs->avg_load < busiest->avg_load) 9662 return false; 9663 9664 if (sgs->avg_load == busiest->avg_load) { 9665 /* 9666 * SMT sched groups need more help than non-SMT groups. 9667 * If @sg happens to also be SMT, either choice is good. 9668 */ 9669 if (sds->busiest->flags & SD_SHARE_CPUCAPACITY) 9670 return false; 9671 } 9672 9673 break; 9674 9675 case group_has_spare: 9676 /* 9677 * Select not overloaded group with lowest number of idle cpus 9678 * and highest number of running tasks. We could also compare 9679 * the spare capacity which is more stable but it can end up 9680 * that the group has less spare capacity but finally more idle 9681 * CPUs which means less opportunity to pull tasks. 9682 */ 9683 if (sgs->idle_cpus > busiest->idle_cpus) 9684 return false; 9685 else if ((sgs->idle_cpus == busiest->idle_cpus) && 9686 (sgs->sum_nr_running <= busiest->sum_nr_running)) 9687 return false; 9688 9689 break; 9690 } 9691 9692 /* 9693 * Candidate sg has no more than one task per CPU and has higher 9694 * per-CPU capacity. Migrating tasks to less capable CPUs may harm 9695 * throughput. Maximize throughput, power/energy consequences are not 9696 * considered. 9697 */ 9698 if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && 9699 (sgs->group_type <= group_fully_busy) && 9700 (capacity_greater(sg->sgc->min_capacity, capacity_of(env->dst_cpu)))) 9701 return false; 9702 9703 return true; 9704 } 9705 9706 #ifdef CONFIG_NUMA_BALANCING 9707 static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs) 9708 { 9709 if (sgs->sum_h_nr_running > sgs->nr_numa_running) 9710 return regular; 9711 if (sgs->sum_h_nr_running > sgs->nr_preferred_running) 9712 return remote; 9713 return all; 9714 } 9715 9716 static inline enum fbq_type fbq_classify_rq(struct rq *rq) 9717 { 9718 if (rq->nr_running > rq->nr_numa_running) 9719 return regular; 9720 if (rq->nr_running > rq->nr_preferred_running) 9721 return remote; 9722 return all; 9723 } 9724 #else 9725 static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs) 9726 { 9727 return all; 9728 } 9729 9730 static inline enum fbq_type fbq_classify_rq(struct rq *rq) 9731 { 9732 return regular; 9733 } 9734 #endif /* CONFIG_NUMA_BALANCING */ 9735 9736 9737 struct sg_lb_stats; 9738 9739 /* 9740 * task_running_on_cpu - return 1 if @p is running on @cpu. 9741 */ 9742 9743 static unsigned int task_running_on_cpu(int cpu, struct task_struct *p) 9744 { 9745 /* Task has no contribution or is new */ 9746 if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 9747 return 0; 9748 9749 if (task_on_rq_queued(p)) 9750 return 1; 9751 9752 return 0; 9753 } 9754 9755 /** 9756 * idle_cpu_without - would a given CPU be idle without p ? 9757 * @cpu: the processor on which idleness is tested. 9758 * @p: task which should be ignored. 9759 * 9760 * Return: 1 if the CPU would be idle. 0 otherwise. 9761 */ 9762 static int idle_cpu_without(int cpu, struct task_struct *p) 9763 { 9764 struct rq *rq = cpu_rq(cpu); 9765 9766 if (rq->curr != rq->idle && rq->curr != p) 9767 return 0; 9768 9769 /* 9770 * rq->nr_running can't be used but an updated version without the 9771 * impact of p on cpu must be used instead. The updated nr_running 9772 * be computed and tested before calling idle_cpu_without(). 9773 */ 9774 9775 #ifdef CONFIG_SMP 9776 if (rq->ttwu_pending) 9777 return 0; 9778 #endif 9779 9780 return 1; 9781 } 9782 9783 /* 9784 * update_sg_wakeup_stats - Update sched_group's statistics for wakeup. 9785 * @sd: The sched_domain level to look for idlest group. 9786 * @group: sched_group whose statistics are to be updated. 9787 * @sgs: variable to hold the statistics for this group. 9788 * @p: The task for which we look for the idlest group/CPU. 9789 */ 9790 static inline void update_sg_wakeup_stats(struct sched_domain *sd, 9791 struct sched_group *group, 9792 struct sg_lb_stats *sgs, 9793 struct task_struct *p) 9794 { 9795 int i, nr_running; 9796 9797 memset(sgs, 0, sizeof(*sgs)); 9798 9799 /* Assume that task can't fit any CPU of the group */ 9800 if (sd->flags & SD_ASYM_CPUCAPACITY) 9801 sgs->group_misfit_task_load = 1; 9802 9803 for_each_cpu(i, sched_group_span(group)) { 9804 struct rq *rq = cpu_rq(i); 9805 unsigned int local; 9806 9807 sgs->group_load += cpu_load_without(rq, p); 9808 sgs->group_util += cpu_util_without(i, p); 9809 sgs->group_runnable += cpu_runnable_without(rq, p); 9810 local = task_running_on_cpu(i, p); 9811 sgs->sum_h_nr_running += rq->cfs.h_nr_running - local; 9812 9813 nr_running = rq->nr_running - local; 9814 sgs->sum_nr_running += nr_running; 9815 9816 /* 9817 * No need to call idle_cpu_without() if nr_running is not 0 9818 */ 9819 if (!nr_running && idle_cpu_without(i, p)) 9820 sgs->idle_cpus++; 9821 9822 /* Check if task fits in the CPU */ 9823 if (sd->flags & SD_ASYM_CPUCAPACITY && 9824 sgs->group_misfit_task_load && 9825 task_fits_cpu(p, i)) 9826 sgs->group_misfit_task_load = 0; 9827 9828 } 9829 9830 sgs->group_capacity = group->sgc->capacity; 9831 9832 sgs->group_weight = group->group_weight; 9833 9834 sgs->group_type = group_classify(sd->imbalance_pct, group, sgs); 9835 9836 /* 9837 * Computing avg_load makes sense only when group is fully busy or 9838 * overloaded 9839 */ 9840 if (sgs->group_type == group_fully_busy || 9841 sgs->group_type == group_overloaded) 9842 sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) / 9843 sgs->group_capacity; 9844 } 9845 9846 static bool update_pick_idlest(struct sched_group *idlest, 9847 struct sg_lb_stats *idlest_sgs, 9848 struct sched_group *group, 9849 struct sg_lb_stats *sgs) 9850 { 9851 if (sgs->group_type < idlest_sgs->group_type) 9852 return true; 9853 9854 if (sgs->group_type > idlest_sgs->group_type) 9855 return false; 9856 9857 /* 9858 * The candidate and the current idlest group are the same type of 9859 * group. Let check which one is the idlest according to the type. 9860 */ 9861 9862 switch (sgs->group_type) { 9863 case group_overloaded: 9864 case group_fully_busy: 9865 /* Select the group with lowest avg_load. */ 9866 if (idlest_sgs->avg_load <= sgs->avg_load) 9867 return false; 9868 break; 9869 9870 case group_imbalanced: 9871 case group_asym_packing: 9872 /* Those types are not used in the slow wakeup path */ 9873 return false; 9874 9875 case group_misfit_task: 9876 /* Select group with the highest max capacity */ 9877 if (idlest->sgc->max_capacity >= group->sgc->max_capacity) 9878 return false; 9879 break; 9880 9881 case group_has_spare: 9882 /* Select group with most idle CPUs */ 9883 if (idlest_sgs->idle_cpus > sgs->idle_cpus) 9884 return false; 9885 9886 /* Select group with lowest group_util */ 9887 if (idlest_sgs->idle_cpus == sgs->idle_cpus && 9888 idlest_sgs->group_util <= sgs->group_util) 9889 return false; 9890 9891 break; 9892 } 9893 9894 return true; 9895 } 9896 9897 /* 9898 * find_idlest_group() finds and returns the least busy CPU group within the 9899 * domain. 9900 * 9901 * Assumes p is allowed on at least one CPU in sd. 9902 */ 9903 static struct sched_group * 9904 find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu) 9905 { 9906 struct sched_group *idlest = NULL, *local = NULL, *group = sd->groups; 9907 struct sg_lb_stats local_sgs, tmp_sgs; 9908 struct sg_lb_stats *sgs; 9909 unsigned long imbalance; 9910 struct sg_lb_stats idlest_sgs = { 9911 .avg_load = UINT_MAX, 9912 .group_type = group_overloaded, 9913 }; 9914 9915 do { 9916 int local_group; 9917 9918 /* Skip over this group if it has no CPUs allowed */ 9919 if (!cpumask_intersects(sched_group_span(group), 9920 p->cpus_ptr)) 9921 continue; 9922 9923 /* Skip over this group if no cookie matched */ 9924 if (!sched_group_cookie_match(cpu_rq(this_cpu), p, group)) 9925 continue; 9926 9927 local_group = cpumask_test_cpu(this_cpu, 9928 sched_group_span(group)); 9929 9930 if (local_group) { 9931 sgs = &local_sgs; 9932 local = group; 9933 } else { 9934 sgs = &tmp_sgs; 9935 } 9936 9937 update_sg_wakeup_stats(sd, group, sgs, p); 9938 9939 if (!local_group && update_pick_idlest(idlest, &idlest_sgs, group, sgs)) { 9940 idlest = group; 9941 idlest_sgs = *sgs; 9942 } 9943 9944 } while (group = group->next, group != sd->groups); 9945 9946 9947 /* There is no idlest group to push tasks to */ 9948 if (!idlest) 9949 return NULL; 9950 9951 /* The local group has been skipped because of CPU affinity */ 9952 if (!local) 9953 return idlest; 9954 9955 /* 9956 * If the local group is idler than the selected idlest group 9957 * don't try and push the task. 9958 */ 9959 if (local_sgs.group_type < idlest_sgs.group_type) 9960 return NULL; 9961 9962 /* 9963 * If the local group is busier than the selected idlest group 9964 * try and push the task. 9965 */ 9966 if (local_sgs.group_type > idlest_sgs.group_type) 9967 return idlest; 9968 9969 switch (local_sgs.group_type) { 9970 case group_overloaded: 9971 case group_fully_busy: 9972 9973 /* Calculate allowed imbalance based on load */ 9974 imbalance = scale_load_down(NICE_0_LOAD) * 9975 (sd->imbalance_pct-100) / 100; 9976 9977 /* 9978 * When comparing groups across NUMA domains, it's possible for 9979 * the local domain to be very lightly loaded relative to the 9980 * remote domains but "imbalance" skews the comparison making 9981 * remote CPUs look much more favourable. When considering 9982 * cross-domain, add imbalance to the load on the remote node 9983 * and consider staying local. 9984 */ 9985 9986 if ((sd->flags & SD_NUMA) && 9987 ((idlest_sgs.avg_load + imbalance) >= local_sgs.avg_load)) 9988 return NULL; 9989 9990 /* 9991 * If the local group is less loaded than the selected 9992 * idlest group don't try and push any tasks. 9993 */ 9994 if (idlest_sgs.avg_load >= (local_sgs.avg_load + imbalance)) 9995 return NULL; 9996 9997 if (100 * local_sgs.avg_load <= sd->imbalance_pct * idlest_sgs.avg_load) 9998 return NULL; 9999 break; 10000 10001 case group_imbalanced: 10002 case group_asym_packing: 10003 /* Those type are not used in the slow wakeup path */ 10004 return NULL; 10005 10006 case group_misfit_task: 10007 /* Select group with the highest max capacity */ 10008 if (local->sgc->max_capacity >= idlest->sgc->max_capacity) 10009 return NULL; 10010 break; 10011 10012 case group_has_spare: 10013 #ifdef CONFIG_NUMA 10014 if (sd->flags & SD_NUMA) { 10015 int imb_numa_nr = sd->imb_numa_nr; 10016 #ifdef CONFIG_NUMA_BALANCING 10017 int idlest_cpu; 10018 /* 10019 * If there is spare capacity at NUMA, try to select 10020 * the preferred node 10021 */ 10022 if (cpu_to_node(this_cpu) == p->numa_preferred_nid) 10023 return NULL; 10024 10025 idlest_cpu = cpumask_first(sched_group_span(idlest)); 10026 if (cpu_to_node(idlest_cpu) == p->numa_preferred_nid) 10027 return idlest; 10028 #endif /* CONFIG_NUMA_BALANCING */ 10029 /* 10030 * Otherwise, keep the task close to the wakeup source 10031 * and improve locality if the number of running tasks 10032 * would remain below threshold where an imbalance is 10033 * allowed while accounting for the possibility the 10034 * task is pinned to a subset of CPUs. If there is a 10035 * real need of migration, periodic load balance will 10036 * take care of it. 10037 */ 10038 if (p->nr_cpus_allowed != NR_CPUS) { 10039 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 10040 10041 cpumask_and(cpus, sched_group_span(local), p->cpus_ptr); 10042 imb_numa_nr = min(cpumask_weight(cpus), sd->imb_numa_nr); 10043 } 10044 10045 imbalance = abs(local_sgs.idle_cpus - idlest_sgs.idle_cpus); 10046 if (!adjust_numa_imbalance(imbalance, 10047 local_sgs.sum_nr_running + 1, 10048 imb_numa_nr)) { 10049 return NULL; 10050 } 10051 } 10052 #endif /* CONFIG_NUMA */ 10053 10054 /* 10055 * Select group with highest number of idle CPUs. We could also 10056 * compare the utilization which is more stable but it can end 10057 * up that the group has less spare capacity but finally more 10058 * idle CPUs which means more opportunity to run task. 10059 */ 10060 if (local_sgs.idle_cpus >= idlest_sgs.idle_cpus) 10061 return NULL; 10062 break; 10063 } 10064 10065 return idlest; 10066 } 10067 10068 static void update_idle_cpu_scan(struct lb_env *env, 10069 unsigned long sum_util) 10070 { 10071 struct sched_domain_shared *sd_share; 10072 int llc_weight, pct; 10073 u64 x, y, tmp; 10074 /* 10075 * Update the number of CPUs to scan in LLC domain, which could 10076 * be used as a hint in select_idle_cpu(). The update of sd_share 10077 * could be expensive because it is within a shared cache line. 10078 * So the write of this hint only occurs during periodic load 10079 * balancing, rather than CPU_NEWLY_IDLE, because the latter 10080 * can fire way more frequently than the former. 10081 */ 10082 if (!sched_feat(SIS_UTIL) || env->idle == CPU_NEWLY_IDLE) 10083 return; 10084 10085 llc_weight = per_cpu(sd_llc_size, env->dst_cpu); 10086 if (env->sd->span_weight != llc_weight) 10087 return; 10088 10089 sd_share = rcu_dereference(per_cpu(sd_llc_shared, env->dst_cpu)); 10090 if (!sd_share) 10091 return; 10092 10093 /* 10094 * The number of CPUs to search drops as sum_util increases, when 10095 * sum_util hits 85% or above, the scan stops. 10096 * The reason to choose 85% as the threshold is because this is the 10097 * imbalance_pct(117) when a LLC sched group is overloaded. 10098 * 10099 * let y = SCHED_CAPACITY_SCALE - p * x^2 [1] 10100 * and y'= y / SCHED_CAPACITY_SCALE 10101 * 10102 * x is the ratio of sum_util compared to the CPU capacity: 10103 * x = sum_util / (llc_weight * SCHED_CAPACITY_SCALE) 10104 * y' is the ratio of CPUs to be scanned in the LLC domain, 10105 * and the number of CPUs to scan is calculated by: 10106 * 10107 * nr_scan = llc_weight * y' [2] 10108 * 10109 * When x hits the threshold of overloaded, AKA, when 10110 * x = 100 / pct, y drops to 0. According to [1], 10111 * p should be SCHED_CAPACITY_SCALE * pct^2 / 10000 10112 * 10113 * Scale x by SCHED_CAPACITY_SCALE: 10114 * x' = sum_util / llc_weight; [3] 10115 * 10116 * and finally [1] becomes: 10117 * y = SCHED_CAPACITY_SCALE - 10118 * x'^2 * pct^2 / (10000 * SCHED_CAPACITY_SCALE) [4] 10119 * 10120 */ 10121 /* equation [3] */ 10122 x = sum_util; 10123 do_div(x, llc_weight); 10124 10125 /* equation [4] */ 10126 pct = env->sd->imbalance_pct; 10127 tmp = x * x * pct * pct; 10128 do_div(tmp, 10000 * SCHED_CAPACITY_SCALE); 10129 tmp = min_t(long, tmp, SCHED_CAPACITY_SCALE); 10130 y = SCHED_CAPACITY_SCALE - tmp; 10131 10132 /* equation [2] */ 10133 y *= llc_weight; 10134 do_div(y, SCHED_CAPACITY_SCALE); 10135 if ((int)y != sd_share->nr_idle_scan) 10136 WRITE_ONCE(sd_share->nr_idle_scan, (int)y); 10137 } 10138 10139 /** 10140 * update_sd_lb_stats - Update sched_domain's statistics for load balancing. 10141 * @env: The load balancing environment. 10142 * @sds: variable to hold the statistics for this sched_domain. 10143 */ 10144 10145 static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sds) 10146 { 10147 struct sched_group *sg = env->sd->groups; 10148 struct sg_lb_stats *local = &sds->local_stat; 10149 struct sg_lb_stats tmp_sgs; 10150 unsigned long sum_util = 0; 10151 int sg_status = 0; 10152 10153 do { 10154 struct sg_lb_stats *sgs = &tmp_sgs; 10155 int local_group; 10156 10157 local_group = cpumask_test_cpu(env->dst_cpu, sched_group_span(sg)); 10158 if (local_group) { 10159 sds->local = sg; 10160 sgs = local; 10161 10162 if (env->idle != CPU_NEWLY_IDLE || 10163 time_after_eq(jiffies, sg->sgc->next_update)) 10164 update_group_capacity(env->sd, env->dst_cpu); 10165 } 10166 10167 update_sg_lb_stats(env, sds, sg, sgs, &sg_status); 10168 10169 if (local_group) 10170 goto next_group; 10171 10172 10173 if (update_sd_pick_busiest(env, sds, sg, sgs)) { 10174 sds->busiest = sg; 10175 sds->busiest_stat = *sgs; 10176 } 10177 10178 next_group: 10179 /* Now, start updating sd_lb_stats */ 10180 sds->total_load += sgs->group_load; 10181 sds->total_capacity += sgs->group_capacity; 10182 10183 sum_util += sgs->group_util; 10184 sg = sg->next; 10185 } while (sg != env->sd->groups); 10186 10187 /* 10188 * Indicate that the child domain of the busiest group prefers tasks 10189 * go to a child's sibling domains first. NB the flags of a sched group 10190 * are those of the child domain. 10191 */ 10192 if (sds->busiest) 10193 sds->prefer_sibling = !!(sds->busiest->flags & SD_PREFER_SIBLING); 10194 10195 10196 if (env->sd->flags & SD_NUMA) 10197 env->fbq_type = fbq_classify_group(&sds->busiest_stat); 10198 10199 if (!env->sd->parent) { 10200 struct root_domain *rd = env->dst_rq->rd; 10201 10202 /* update overload indicator if we are at root domain */ 10203 WRITE_ONCE(rd->overload, sg_status & SG_OVERLOAD); 10204 10205 /* Update over-utilization (tipping point, U >= 0) indicator */ 10206 WRITE_ONCE(rd->overutilized, sg_status & SG_OVERUTILIZED); 10207 trace_sched_overutilized_tp(rd, sg_status & SG_OVERUTILIZED); 10208 } else if (sg_status & SG_OVERUTILIZED) { 10209 struct root_domain *rd = env->dst_rq->rd; 10210 10211 WRITE_ONCE(rd->overutilized, SG_OVERUTILIZED); 10212 trace_sched_overutilized_tp(rd, SG_OVERUTILIZED); 10213 } 10214 10215 update_idle_cpu_scan(env, sum_util); 10216 } 10217 10218 /** 10219 * calculate_imbalance - Calculate the amount of imbalance present within the 10220 * groups of a given sched_domain during load balance. 10221 * @env: load balance environment 10222 * @sds: statistics of the sched_domain whose imbalance is to be calculated. 10223 */ 10224 static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds) 10225 { 10226 struct sg_lb_stats *local, *busiest; 10227 10228 local = &sds->local_stat; 10229 busiest = &sds->busiest_stat; 10230 10231 if (busiest->group_type == group_misfit_task) { 10232 if (env->sd->flags & SD_ASYM_CPUCAPACITY) { 10233 /* Set imbalance to allow misfit tasks to be balanced. */ 10234 env->migration_type = migrate_misfit; 10235 env->imbalance = 1; 10236 } else { 10237 /* 10238 * Set load imbalance to allow moving task from cpu 10239 * with reduced capacity. 10240 */ 10241 env->migration_type = migrate_load; 10242 env->imbalance = busiest->group_misfit_task_load; 10243 } 10244 return; 10245 } 10246 10247 if (busiest->group_type == group_asym_packing) { 10248 /* 10249 * In case of asym capacity, we will try to migrate all load to 10250 * the preferred CPU. 10251 */ 10252 env->migration_type = migrate_task; 10253 env->imbalance = busiest->sum_h_nr_running; 10254 return; 10255 } 10256 10257 if (busiest->group_type == group_imbalanced) { 10258 /* 10259 * In the group_imb case we cannot rely on group-wide averages 10260 * to ensure CPU-load equilibrium, try to move any task to fix 10261 * the imbalance. The next load balance will take care of 10262 * balancing back the system. 10263 */ 10264 env->migration_type = migrate_task; 10265 env->imbalance = 1; 10266 return; 10267 } 10268 10269 /* 10270 * Try to use spare capacity of local group without overloading it or 10271 * emptying busiest. 10272 */ 10273 if (local->group_type == group_has_spare) { 10274 if ((busiest->group_type > group_fully_busy) && 10275 !(env->sd->flags & SD_SHARE_PKG_RESOURCES)) { 10276 /* 10277 * If busiest is overloaded, try to fill spare 10278 * capacity. This might end up creating spare capacity 10279 * in busiest or busiest still being overloaded but 10280 * there is no simple way to directly compute the 10281 * amount of load to migrate in order to balance the 10282 * system. 10283 */ 10284 env->migration_type = migrate_util; 10285 env->imbalance = max(local->group_capacity, local->group_util) - 10286 local->group_util; 10287 10288 /* 10289 * In some cases, the group's utilization is max or even 10290 * higher than capacity because of migrations but the 10291 * local CPU is (newly) idle. There is at least one 10292 * waiting task in this overloaded busiest group. Let's 10293 * try to pull it. 10294 */ 10295 if (env->idle != CPU_NOT_IDLE && env->imbalance == 0) { 10296 env->migration_type = migrate_task; 10297 env->imbalance = 1; 10298 } 10299 10300 return; 10301 } 10302 10303 if (busiest->group_weight == 1 || sds->prefer_sibling) { 10304 unsigned int nr_diff = busiest->sum_nr_running; 10305 /* 10306 * When prefer sibling, evenly spread running tasks on 10307 * groups. 10308 */ 10309 env->migration_type = migrate_task; 10310 lsub_positive(&nr_diff, local->sum_nr_running); 10311 env->imbalance = nr_diff; 10312 } else { 10313 10314 /* 10315 * If there is no overload, we just want to even the number of 10316 * idle cpus. 10317 */ 10318 env->migration_type = migrate_task; 10319 env->imbalance = max_t(long, 0, 10320 (local->idle_cpus - busiest->idle_cpus)); 10321 } 10322 10323 #ifdef CONFIG_NUMA 10324 /* Consider allowing a small imbalance between NUMA groups */ 10325 if (env->sd->flags & SD_NUMA) { 10326 env->imbalance = adjust_numa_imbalance(env->imbalance, 10327 local->sum_nr_running + 1, 10328 env->sd->imb_numa_nr); 10329 } 10330 #endif 10331 10332 /* Number of tasks to move to restore balance */ 10333 env->imbalance >>= 1; 10334 10335 return; 10336 } 10337 10338 /* 10339 * Local is fully busy but has to take more load to relieve the 10340 * busiest group 10341 */ 10342 if (local->group_type < group_overloaded) { 10343 /* 10344 * Local will become overloaded so the avg_load metrics are 10345 * finally needed. 10346 */ 10347 10348 local->avg_load = (local->group_load * SCHED_CAPACITY_SCALE) / 10349 local->group_capacity; 10350 10351 /* 10352 * If the local group is more loaded than the selected 10353 * busiest group don't try to pull any tasks. 10354 */ 10355 if (local->avg_load >= busiest->avg_load) { 10356 env->imbalance = 0; 10357 return; 10358 } 10359 10360 sds->avg_load = (sds->total_load * SCHED_CAPACITY_SCALE) / 10361 sds->total_capacity; 10362 10363 /* 10364 * If the local group is more loaded than the average system 10365 * load, don't try to pull any tasks. 10366 */ 10367 if (local->avg_load >= sds->avg_load) { 10368 env->imbalance = 0; 10369 return; 10370 } 10371 10372 } 10373 10374 /* 10375 * Both group are or will become overloaded and we're trying to get all 10376 * the CPUs to the average_load, so we don't want to push ourselves 10377 * above the average load, nor do we wish to reduce the max loaded CPU 10378 * below the average load. At the same time, we also don't want to 10379 * reduce the group load below the group capacity. Thus we look for 10380 * the minimum possible imbalance. 10381 */ 10382 env->migration_type = migrate_load; 10383 env->imbalance = min( 10384 (busiest->avg_load - sds->avg_load) * busiest->group_capacity, 10385 (sds->avg_load - local->avg_load) * local->group_capacity 10386 ) / SCHED_CAPACITY_SCALE; 10387 } 10388 10389 /******* find_busiest_group() helpers end here *********************/ 10390 10391 /* 10392 * Decision matrix according to the local and busiest group type: 10393 * 10394 * busiest \ local has_spare fully_busy misfit asym imbalanced overloaded 10395 * has_spare nr_idle balanced N/A N/A balanced balanced 10396 * fully_busy nr_idle nr_idle N/A N/A balanced balanced 10397 * misfit_task force N/A N/A N/A N/A N/A 10398 * asym_packing force force N/A N/A force force 10399 * imbalanced force force N/A N/A force force 10400 * overloaded force force N/A N/A force avg_load 10401 * 10402 * N/A : Not Applicable because already filtered while updating 10403 * statistics. 10404 * balanced : The system is balanced for these 2 groups. 10405 * force : Calculate the imbalance as load migration is probably needed. 10406 * avg_load : Only if imbalance is significant enough. 10407 * nr_idle : dst_cpu is not busy and the number of idle CPUs is quite 10408 * different in groups. 10409 */ 10410 10411 /** 10412 * find_busiest_group - Returns the busiest group within the sched_domain 10413 * if there is an imbalance. 10414 * @env: The load balancing environment. 10415 * 10416 * Also calculates the amount of runnable load which should be moved 10417 * to restore balance. 10418 * 10419 * Return: - The busiest group if imbalance exists. 10420 */ 10421 static struct sched_group *find_busiest_group(struct lb_env *env) 10422 { 10423 struct sg_lb_stats *local, *busiest; 10424 struct sd_lb_stats sds; 10425 10426 init_sd_lb_stats(&sds); 10427 10428 /* 10429 * Compute the various statistics relevant for load balancing at 10430 * this level. 10431 */ 10432 update_sd_lb_stats(env, &sds); 10433 10434 /* There is no busy sibling group to pull tasks from */ 10435 if (!sds.busiest) 10436 goto out_balanced; 10437 10438 busiest = &sds.busiest_stat; 10439 10440 /* Misfit tasks should be dealt with regardless of the avg load */ 10441 if (busiest->group_type == group_misfit_task) 10442 goto force_balance; 10443 10444 if (sched_energy_enabled()) { 10445 struct root_domain *rd = env->dst_rq->rd; 10446 10447 if (rcu_dereference(rd->pd) && !READ_ONCE(rd->overutilized)) 10448 goto out_balanced; 10449 } 10450 10451 /* ASYM feature bypasses nice load balance check */ 10452 if (busiest->group_type == group_asym_packing) 10453 goto force_balance; 10454 10455 /* 10456 * If the busiest group is imbalanced the below checks don't 10457 * work because they assume all things are equal, which typically 10458 * isn't true due to cpus_ptr constraints and the like. 10459 */ 10460 if (busiest->group_type == group_imbalanced) 10461 goto force_balance; 10462 10463 local = &sds.local_stat; 10464 /* 10465 * If the local group is busier than the selected busiest group 10466 * don't try and pull any tasks. 10467 */ 10468 if (local->group_type > busiest->group_type) 10469 goto out_balanced; 10470 10471 /* 10472 * When groups are overloaded, use the avg_load to ensure fairness 10473 * between tasks. 10474 */ 10475 if (local->group_type == group_overloaded) { 10476 /* 10477 * If the local group is more loaded than the selected 10478 * busiest group don't try to pull any tasks. 10479 */ 10480 if (local->avg_load >= busiest->avg_load) 10481 goto out_balanced; 10482 10483 /* XXX broken for overlapping NUMA groups */ 10484 sds.avg_load = (sds.total_load * SCHED_CAPACITY_SCALE) / 10485 sds.total_capacity; 10486 10487 /* 10488 * Don't pull any tasks if this group is already above the 10489 * domain average load. 10490 */ 10491 if (local->avg_load >= sds.avg_load) 10492 goto out_balanced; 10493 10494 /* 10495 * If the busiest group is more loaded, use imbalance_pct to be 10496 * conservative. 10497 */ 10498 if (100 * busiest->avg_load <= 10499 env->sd->imbalance_pct * local->avg_load) 10500 goto out_balanced; 10501 } 10502 10503 /* 10504 * Try to move all excess tasks to a sibling domain of the busiest 10505 * group's child domain. 10506 */ 10507 if (sds.prefer_sibling && local->group_type == group_has_spare && 10508 busiest->sum_nr_running > local->sum_nr_running + 1) 10509 goto force_balance; 10510 10511 if (busiest->group_type != group_overloaded) { 10512 if (env->idle == CPU_NOT_IDLE) 10513 /* 10514 * If the busiest group is not overloaded (and as a 10515 * result the local one too) but this CPU is already 10516 * busy, let another idle CPU try to pull task. 10517 */ 10518 goto out_balanced; 10519 10520 if (busiest->group_weight > 1 && 10521 local->idle_cpus <= (busiest->idle_cpus + 1)) 10522 /* 10523 * If the busiest group is not overloaded 10524 * and there is no imbalance between this and busiest 10525 * group wrt idle CPUs, it is balanced. The imbalance 10526 * becomes significant if the diff is greater than 1 10527 * otherwise we might end up to just move the imbalance 10528 * on another group. Of course this applies only if 10529 * there is more than 1 CPU per group. 10530 */ 10531 goto out_balanced; 10532 10533 if (busiest->sum_h_nr_running == 1) 10534 /* 10535 * busiest doesn't have any tasks waiting to run 10536 */ 10537 goto out_balanced; 10538 } 10539 10540 force_balance: 10541 /* Looks like there is an imbalance. Compute it */ 10542 calculate_imbalance(env, &sds); 10543 return env->imbalance ? sds.busiest : NULL; 10544 10545 out_balanced: 10546 env->imbalance = 0; 10547 return NULL; 10548 } 10549 10550 /* 10551 * find_busiest_queue - find the busiest runqueue among the CPUs in the group. 10552 */ 10553 static struct rq *find_busiest_queue(struct lb_env *env, 10554 struct sched_group *group) 10555 { 10556 struct rq *busiest = NULL, *rq; 10557 unsigned long busiest_util = 0, busiest_load = 0, busiest_capacity = 1; 10558 unsigned int busiest_nr = 0; 10559 int i; 10560 10561 for_each_cpu_and(i, sched_group_span(group), env->cpus) { 10562 unsigned long capacity, load, util; 10563 unsigned int nr_running; 10564 enum fbq_type rt; 10565 10566 rq = cpu_rq(i); 10567 rt = fbq_classify_rq(rq); 10568 10569 /* 10570 * We classify groups/runqueues into three groups: 10571 * - regular: there are !numa tasks 10572 * - remote: there are numa tasks that run on the 'wrong' node 10573 * - all: there is no distinction 10574 * 10575 * In order to avoid migrating ideally placed numa tasks, 10576 * ignore those when there's better options. 10577 * 10578 * If we ignore the actual busiest queue to migrate another 10579 * task, the next balance pass can still reduce the busiest 10580 * queue by moving tasks around inside the node. 10581 * 10582 * If we cannot move enough load due to this classification 10583 * the next pass will adjust the group classification and 10584 * allow migration of more tasks. 10585 * 10586 * Both cases only affect the total convergence complexity. 10587 */ 10588 if (rt > env->fbq_type) 10589 continue; 10590 10591 nr_running = rq->cfs.h_nr_running; 10592 if (!nr_running) 10593 continue; 10594 10595 capacity = capacity_of(i); 10596 10597 /* 10598 * For ASYM_CPUCAPACITY domains, don't pick a CPU that could 10599 * eventually lead to active_balancing high->low capacity. 10600 * Higher per-CPU capacity is considered better than balancing 10601 * average load. 10602 */ 10603 if (env->sd->flags & SD_ASYM_CPUCAPACITY && 10604 !capacity_greater(capacity_of(env->dst_cpu), capacity) && 10605 nr_running == 1) 10606 continue; 10607 10608 /* 10609 * Make sure we only pull tasks from a CPU of lower priority 10610 * when balancing between SMT siblings. 10611 * 10612 * If balancing between cores, let lower priority CPUs help 10613 * SMT cores with more than one busy sibling. 10614 */ 10615 if ((env->sd->flags & SD_ASYM_PACKING) && 10616 sched_use_asym_prio(env->sd, i) && 10617 sched_asym_prefer(i, env->dst_cpu) && 10618 nr_running == 1) 10619 continue; 10620 10621 switch (env->migration_type) { 10622 case migrate_load: 10623 /* 10624 * When comparing with load imbalance, use cpu_load() 10625 * which is not scaled with the CPU capacity. 10626 */ 10627 load = cpu_load(rq); 10628 10629 if (nr_running == 1 && load > env->imbalance && 10630 !check_cpu_capacity(rq, env->sd)) 10631 break; 10632 10633 /* 10634 * For the load comparisons with the other CPUs, 10635 * consider the cpu_load() scaled with the CPU 10636 * capacity, so that the load can be moved away 10637 * from the CPU that is potentially running at a 10638 * lower capacity. 10639 * 10640 * Thus we're looking for max(load_i / capacity_i), 10641 * crosswise multiplication to rid ourselves of the 10642 * division works out to: 10643 * load_i * capacity_j > load_j * capacity_i; 10644 * where j is our previous maximum. 10645 */ 10646 if (load * busiest_capacity > busiest_load * capacity) { 10647 busiest_load = load; 10648 busiest_capacity = capacity; 10649 busiest = rq; 10650 } 10651 break; 10652 10653 case migrate_util: 10654 util = cpu_util_cfs_boost(i); 10655 10656 /* 10657 * Don't try to pull utilization from a CPU with one 10658 * running task. Whatever its utilization, we will fail 10659 * detach the task. 10660 */ 10661 if (nr_running <= 1) 10662 continue; 10663 10664 if (busiest_util < util) { 10665 busiest_util = util; 10666 busiest = rq; 10667 } 10668 break; 10669 10670 case migrate_task: 10671 if (busiest_nr < nr_running) { 10672 busiest_nr = nr_running; 10673 busiest = rq; 10674 } 10675 break; 10676 10677 case migrate_misfit: 10678 /* 10679 * For ASYM_CPUCAPACITY domains with misfit tasks we 10680 * simply seek the "biggest" misfit task. 10681 */ 10682 if (rq->misfit_task_load > busiest_load) { 10683 busiest_load = rq->misfit_task_load; 10684 busiest = rq; 10685 } 10686 10687 break; 10688 10689 } 10690 } 10691 10692 return busiest; 10693 } 10694 10695 /* 10696 * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but 10697 * so long as it is large enough. 10698 */ 10699 #define MAX_PINNED_INTERVAL 512 10700 10701 static inline bool 10702 asym_active_balance(struct lb_env *env) 10703 { 10704 /* 10705 * ASYM_PACKING needs to force migrate tasks from busy but lower 10706 * priority CPUs in order to pack all tasks in the highest priority 10707 * CPUs. When done between cores, do it only if the whole core if the 10708 * whole core is idle. 10709 * 10710 * If @env::src_cpu is an SMT core with busy siblings, let 10711 * the lower priority @env::dst_cpu help it. Do not follow 10712 * CPU priority. 10713 */ 10714 return env->idle != CPU_NOT_IDLE && (env->sd->flags & SD_ASYM_PACKING) && 10715 sched_use_asym_prio(env->sd, env->dst_cpu) && 10716 (sched_asym_prefer(env->dst_cpu, env->src_cpu) || 10717 !sched_use_asym_prio(env->sd, env->src_cpu)); 10718 } 10719 10720 static inline bool 10721 imbalanced_active_balance(struct lb_env *env) 10722 { 10723 struct sched_domain *sd = env->sd; 10724 10725 /* 10726 * The imbalanced case includes the case of pinned tasks preventing a fair 10727 * distribution of the load on the system but also the even distribution of the 10728 * threads on a system with spare capacity 10729 */ 10730 if ((env->migration_type == migrate_task) && 10731 (sd->nr_balance_failed > sd->cache_nice_tries+2)) 10732 return 1; 10733 10734 return 0; 10735 } 10736 10737 static int need_active_balance(struct lb_env *env) 10738 { 10739 struct sched_domain *sd = env->sd; 10740 10741 if (asym_active_balance(env)) 10742 return 1; 10743 10744 if (imbalanced_active_balance(env)) 10745 return 1; 10746 10747 /* 10748 * The dst_cpu is idle and the src_cpu CPU has only 1 CFS task. 10749 * It's worth migrating the task if the src_cpu's capacity is reduced 10750 * because of other sched_class or IRQs if more capacity stays 10751 * available on dst_cpu. 10752 */ 10753 if ((env->idle != CPU_NOT_IDLE) && 10754 (env->src_rq->cfs.h_nr_running == 1)) { 10755 if ((check_cpu_capacity(env->src_rq, sd)) && 10756 (capacity_of(env->src_cpu)*sd->imbalance_pct < capacity_of(env->dst_cpu)*100)) 10757 return 1; 10758 } 10759 10760 if (env->migration_type == migrate_misfit) 10761 return 1; 10762 10763 return 0; 10764 } 10765 10766 static int active_load_balance_cpu_stop(void *data); 10767 10768 static int should_we_balance(struct lb_env *env) 10769 { 10770 struct sched_group *sg = env->sd->groups; 10771 int cpu; 10772 10773 /* 10774 * Ensure the balancing environment is consistent; can happen 10775 * when the softirq triggers 'during' hotplug. 10776 */ 10777 if (!cpumask_test_cpu(env->dst_cpu, env->cpus)) 10778 return 0; 10779 10780 /* 10781 * In the newly idle case, we will allow all the CPUs 10782 * to do the newly idle load balance. 10783 * 10784 * However, we bail out if we already have tasks or a wakeup pending, 10785 * to optimize wakeup latency. 10786 */ 10787 if (env->idle == CPU_NEWLY_IDLE) { 10788 if (env->dst_rq->nr_running > 0 || env->dst_rq->ttwu_pending) 10789 return 0; 10790 return 1; 10791 } 10792 10793 /* Try to find first idle CPU */ 10794 for_each_cpu_and(cpu, group_balance_mask(sg), env->cpus) { 10795 if (!idle_cpu(cpu)) 10796 continue; 10797 10798 /* Are we the first idle CPU? */ 10799 return cpu == env->dst_cpu; 10800 } 10801 10802 /* Are we the first CPU of this group ? */ 10803 return group_balance_cpu(sg) == env->dst_cpu; 10804 } 10805 10806 /* 10807 * Check this_cpu to ensure it is balanced within domain. Attempt to move 10808 * tasks if there is an imbalance. 10809 */ 10810 static int load_balance(int this_cpu, struct rq *this_rq, 10811 struct sched_domain *sd, enum cpu_idle_type idle, 10812 int *continue_balancing) 10813 { 10814 int ld_moved, cur_ld_moved, active_balance = 0; 10815 struct sched_domain *sd_parent = sd->parent; 10816 struct sched_group *group; 10817 struct rq *busiest; 10818 struct rq_flags rf; 10819 struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask); 10820 struct lb_env env = { 10821 .sd = sd, 10822 .dst_cpu = this_cpu, 10823 .dst_rq = this_rq, 10824 .dst_grpmask = group_balance_mask(sd->groups), 10825 .idle = idle, 10826 .loop_break = SCHED_NR_MIGRATE_BREAK, 10827 .cpus = cpus, 10828 .fbq_type = all, 10829 .tasks = LIST_HEAD_INIT(env.tasks), 10830 }; 10831 10832 cpumask_and(cpus, sched_domain_span(sd), cpu_active_mask); 10833 10834 schedstat_inc(sd->lb_count[idle]); 10835 10836 redo: 10837 if (!should_we_balance(&env)) { 10838 *continue_balancing = 0; 10839 goto out_balanced; 10840 } 10841 10842 group = find_busiest_group(&env); 10843 if (!group) { 10844 schedstat_inc(sd->lb_nobusyg[idle]); 10845 goto out_balanced; 10846 } 10847 10848 busiest = find_busiest_queue(&env, group); 10849 if (!busiest) { 10850 schedstat_inc(sd->lb_nobusyq[idle]); 10851 goto out_balanced; 10852 } 10853 10854 WARN_ON_ONCE(busiest == env.dst_rq); 10855 10856 schedstat_add(sd->lb_imbalance[idle], env.imbalance); 10857 10858 env.src_cpu = busiest->cpu; 10859 env.src_rq = busiest; 10860 10861 ld_moved = 0; 10862 /* Clear this flag as soon as we find a pullable task */ 10863 env.flags |= LBF_ALL_PINNED; 10864 if (busiest->nr_running > 1) { 10865 /* 10866 * Attempt to move tasks. If find_busiest_group has found 10867 * an imbalance but busiest->nr_running <= 1, the group is 10868 * still unbalanced. ld_moved simply stays zero, so it is 10869 * correctly treated as an imbalance. 10870 */ 10871 env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running); 10872 10873 more_balance: 10874 rq_lock_irqsave(busiest, &rf); 10875 update_rq_clock(busiest); 10876 10877 /* 10878 * cur_ld_moved - load moved in current iteration 10879 * ld_moved - cumulative load moved across iterations 10880 */ 10881 cur_ld_moved = detach_tasks(&env); 10882 10883 /* 10884 * We've detached some tasks from busiest_rq. Every 10885 * task is masked "TASK_ON_RQ_MIGRATING", so we can safely 10886 * unlock busiest->lock, and we are able to be sure 10887 * that nobody can manipulate the tasks in parallel. 10888 * See task_rq_lock() family for the details. 10889 */ 10890 10891 rq_unlock(busiest, &rf); 10892 10893 if (cur_ld_moved) { 10894 attach_tasks(&env); 10895 ld_moved += cur_ld_moved; 10896 } 10897 10898 local_irq_restore(rf.flags); 10899 10900 if (env.flags & LBF_NEED_BREAK) { 10901 env.flags &= ~LBF_NEED_BREAK; 10902 /* Stop if we tried all running tasks */ 10903 if (env.loop < busiest->nr_running) 10904 goto more_balance; 10905 } 10906 10907 /* 10908 * Revisit (affine) tasks on src_cpu that couldn't be moved to 10909 * us and move them to an alternate dst_cpu in our sched_group 10910 * where they can run. The upper limit on how many times we 10911 * iterate on same src_cpu is dependent on number of CPUs in our 10912 * sched_group. 10913 * 10914 * This changes load balance semantics a bit on who can move 10915 * load to a given_cpu. In addition to the given_cpu itself 10916 * (or a ilb_cpu acting on its behalf where given_cpu is 10917 * nohz-idle), we now have balance_cpu in a position to move 10918 * load to given_cpu. In rare situations, this may cause 10919 * conflicts (balance_cpu and given_cpu/ilb_cpu deciding 10920 * _independently_ and at _same_ time to move some load to 10921 * given_cpu) causing excess load to be moved to given_cpu. 10922 * This however should not happen so much in practice and 10923 * moreover subsequent load balance cycles should correct the 10924 * excess load moved. 10925 */ 10926 if ((env.flags & LBF_DST_PINNED) && env.imbalance > 0) { 10927 10928 /* Prevent to re-select dst_cpu via env's CPUs */ 10929 __cpumask_clear_cpu(env.dst_cpu, env.cpus); 10930 10931 env.dst_rq = cpu_rq(env.new_dst_cpu); 10932 env.dst_cpu = env.new_dst_cpu; 10933 env.flags &= ~LBF_DST_PINNED; 10934 env.loop = 0; 10935 env.loop_break = SCHED_NR_MIGRATE_BREAK; 10936 10937 /* 10938 * Go back to "more_balance" rather than "redo" since we 10939 * need to continue with same src_cpu. 10940 */ 10941 goto more_balance; 10942 } 10943 10944 /* 10945 * We failed to reach balance because of affinity. 10946 */ 10947 if (sd_parent) { 10948 int *group_imbalance = &sd_parent->groups->sgc->imbalance; 10949 10950 if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0) 10951 *group_imbalance = 1; 10952 } 10953 10954 /* All tasks on this runqueue were pinned by CPU affinity */ 10955 if (unlikely(env.flags & LBF_ALL_PINNED)) { 10956 __cpumask_clear_cpu(cpu_of(busiest), cpus); 10957 /* 10958 * Attempting to continue load balancing at the current 10959 * sched_domain level only makes sense if there are 10960 * active CPUs remaining as possible busiest CPUs to 10961 * pull load from which are not contained within the 10962 * destination group that is receiving any migrated 10963 * load. 10964 */ 10965 if (!cpumask_subset(cpus, env.dst_grpmask)) { 10966 env.loop = 0; 10967 env.loop_break = SCHED_NR_MIGRATE_BREAK; 10968 goto redo; 10969 } 10970 goto out_all_pinned; 10971 } 10972 } 10973 10974 if (!ld_moved) { 10975 schedstat_inc(sd->lb_failed[idle]); 10976 /* 10977 * Increment the failure counter only on periodic balance. 10978 * We do not want newidle balance, which can be very 10979 * frequent, pollute the failure counter causing 10980 * excessive cache_hot migrations and active balances. 10981 */ 10982 if (idle != CPU_NEWLY_IDLE) 10983 sd->nr_balance_failed++; 10984 10985 if (need_active_balance(&env)) { 10986 unsigned long flags; 10987 10988 raw_spin_rq_lock_irqsave(busiest, flags); 10989 10990 /* 10991 * Don't kick the active_load_balance_cpu_stop, 10992 * if the curr task on busiest CPU can't be 10993 * moved to this_cpu: 10994 */ 10995 if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr)) { 10996 raw_spin_rq_unlock_irqrestore(busiest, flags); 10997 goto out_one_pinned; 10998 } 10999 11000 /* Record that we found at least one task that could run on this_cpu */ 11001 env.flags &= ~LBF_ALL_PINNED; 11002 11003 /* 11004 * ->active_balance synchronizes accesses to 11005 * ->active_balance_work. Once set, it's cleared 11006 * only after active load balance is finished. 11007 */ 11008 if (!busiest->active_balance) { 11009 busiest->active_balance = 1; 11010 busiest->push_cpu = this_cpu; 11011 active_balance = 1; 11012 } 11013 raw_spin_rq_unlock_irqrestore(busiest, flags); 11014 11015 if (active_balance) { 11016 stop_one_cpu_nowait(cpu_of(busiest), 11017 active_load_balance_cpu_stop, busiest, 11018 &busiest->active_balance_work); 11019 } 11020 } 11021 } else { 11022 sd->nr_balance_failed = 0; 11023 } 11024 11025 if (likely(!active_balance) || need_active_balance(&env)) { 11026 /* We were unbalanced, so reset the balancing interval */ 11027 sd->balance_interval = sd->min_interval; 11028 } 11029 11030 goto out; 11031 11032 out_balanced: 11033 /* 11034 * We reach balance although we may have faced some affinity 11035 * constraints. Clear the imbalance flag only if other tasks got 11036 * a chance to move and fix the imbalance. 11037 */ 11038 if (sd_parent && !(env.flags & LBF_ALL_PINNED)) { 11039 int *group_imbalance = &sd_parent->groups->sgc->imbalance; 11040 11041 if (*group_imbalance) 11042 *group_imbalance = 0; 11043 } 11044 11045 out_all_pinned: 11046 /* 11047 * We reach balance because all tasks are pinned at this level so 11048 * we can't migrate them. Let the imbalance flag set so parent level 11049 * can try to migrate them. 11050 */ 11051 schedstat_inc(sd->lb_balanced[idle]); 11052 11053 sd->nr_balance_failed = 0; 11054 11055 out_one_pinned: 11056 ld_moved = 0; 11057 11058 /* 11059 * newidle_balance() disregards balance intervals, so we could 11060 * repeatedly reach this code, which would lead to balance_interval 11061 * skyrocketing in a short amount of time. Skip the balance_interval 11062 * increase logic to avoid that. 11063 */ 11064 if (env.idle == CPU_NEWLY_IDLE) 11065 goto out; 11066 11067 /* tune up the balancing interval */ 11068 if ((env.flags & LBF_ALL_PINNED && 11069 sd->balance_interval < MAX_PINNED_INTERVAL) || 11070 sd->balance_interval < sd->max_interval) 11071 sd->balance_interval *= 2; 11072 out: 11073 return ld_moved; 11074 } 11075 11076 static inline unsigned long 11077 get_sd_balance_interval(struct sched_domain *sd, int cpu_busy) 11078 { 11079 unsigned long interval = sd->balance_interval; 11080 11081 if (cpu_busy) 11082 interval *= sd->busy_factor; 11083 11084 /* scale ms to jiffies */ 11085 interval = msecs_to_jiffies(interval); 11086 11087 /* 11088 * Reduce likelihood of busy balancing at higher domains racing with 11089 * balancing at lower domains by preventing their balancing periods 11090 * from being multiples of each other. 11091 */ 11092 if (cpu_busy) 11093 interval -= 1; 11094 11095 interval = clamp(interval, 1UL, max_load_balance_interval); 11096 11097 return interval; 11098 } 11099 11100 static inline void 11101 update_next_balance(struct sched_domain *sd, unsigned long *next_balance) 11102 { 11103 unsigned long interval, next; 11104 11105 /* used by idle balance, so cpu_busy = 0 */ 11106 interval = get_sd_balance_interval(sd, 0); 11107 next = sd->last_balance + interval; 11108 11109 if (time_after(*next_balance, next)) 11110 *next_balance = next; 11111 } 11112 11113 /* 11114 * active_load_balance_cpu_stop is run by the CPU stopper. It pushes 11115 * running tasks off the busiest CPU onto idle CPUs. It requires at 11116 * least 1 task to be running on each physical CPU where possible, and 11117 * avoids physical / logical imbalances. 11118 */ 11119 static int active_load_balance_cpu_stop(void *data) 11120 { 11121 struct rq *busiest_rq = data; 11122 int busiest_cpu = cpu_of(busiest_rq); 11123 int target_cpu = busiest_rq->push_cpu; 11124 struct rq *target_rq = cpu_rq(target_cpu); 11125 struct sched_domain *sd; 11126 struct task_struct *p = NULL; 11127 struct rq_flags rf; 11128 11129 rq_lock_irq(busiest_rq, &rf); 11130 /* 11131 * Between queueing the stop-work and running it is a hole in which 11132 * CPUs can become inactive. We should not move tasks from or to 11133 * inactive CPUs. 11134 */ 11135 if (!cpu_active(busiest_cpu) || !cpu_active(target_cpu)) 11136 goto out_unlock; 11137 11138 /* Make sure the requested CPU hasn't gone down in the meantime: */ 11139 if (unlikely(busiest_cpu != smp_processor_id() || 11140 !busiest_rq->active_balance)) 11141 goto out_unlock; 11142 11143 /* Is there any task to move? */ 11144 if (busiest_rq->nr_running <= 1) 11145 goto out_unlock; 11146 11147 /* 11148 * This condition is "impossible", if it occurs 11149 * we need to fix it. Originally reported by 11150 * Bjorn Helgaas on a 128-CPU setup. 11151 */ 11152 WARN_ON_ONCE(busiest_rq == target_rq); 11153 11154 /* Search for an sd spanning us and the target CPU. */ 11155 rcu_read_lock(); 11156 for_each_domain(target_cpu, sd) { 11157 if (cpumask_test_cpu(busiest_cpu, sched_domain_span(sd))) 11158 break; 11159 } 11160 11161 if (likely(sd)) { 11162 struct lb_env env = { 11163 .sd = sd, 11164 .dst_cpu = target_cpu, 11165 .dst_rq = target_rq, 11166 .src_cpu = busiest_rq->cpu, 11167 .src_rq = busiest_rq, 11168 .idle = CPU_IDLE, 11169 .flags = LBF_ACTIVE_LB, 11170 }; 11171 11172 schedstat_inc(sd->alb_count); 11173 update_rq_clock(busiest_rq); 11174 11175 p = detach_one_task(&env); 11176 if (p) { 11177 schedstat_inc(sd->alb_pushed); 11178 /* Active balancing done, reset the failure counter. */ 11179 sd->nr_balance_failed = 0; 11180 } else { 11181 schedstat_inc(sd->alb_failed); 11182 } 11183 } 11184 rcu_read_unlock(); 11185 out_unlock: 11186 busiest_rq->active_balance = 0; 11187 rq_unlock(busiest_rq, &rf); 11188 11189 if (p) 11190 attach_one_task(target_rq, p); 11191 11192 local_irq_enable(); 11193 11194 return 0; 11195 } 11196 11197 static DEFINE_SPINLOCK(balancing); 11198 11199 /* 11200 * Scale the max load_balance interval with the number of CPUs in the system. 11201 * This trades load-balance latency on larger machines for less cross talk. 11202 */ 11203 void update_max_interval(void) 11204 { 11205 max_load_balance_interval = HZ*num_online_cpus()/10; 11206 } 11207 11208 static inline bool update_newidle_cost(struct sched_domain *sd, u64 cost) 11209 { 11210 if (cost > sd->max_newidle_lb_cost) { 11211 /* 11212 * Track max cost of a domain to make sure to not delay the 11213 * next wakeup on the CPU. 11214 */ 11215 sd->max_newidle_lb_cost = cost; 11216 sd->last_decay_max_lb_cost = jiffies; 11217 } else if (time_after(jiffies, sd->last_decay_max_lb_cost + HZ)) { 11218 /* 11219 * Decay the newidle max times by ~1% per second to ensure that 11220 * it is not outdated and the current max cost is actually 11221 * shorter. 11222 */ 11223 sd->max_newidle_lb_cost = (sd->max_newidle_lb_cost * 253) / 256; 11224 sd->last_decay_max_lb_cost = jiffies; 11225 11226 return true; 11227 } 11228 11229 return false; 11230 } 11231 11232 /* 11233 * It checks each scheduling domain to see if it is due to be balanced, 11234 * and initiates a balancing operation if so. 11235 * 11236 * Balancing parameters are set up in init_sched_domains. 11237 */ 11238 static void rebalance_domains(struct rq *rq, enum cpu_idle_type idle) 11239 { 11240 int continue_balancing = 1; 11241 int cpu = rq->cpu; 11242 int busy = idle != CPU_IDLE && !sched_idle_cpu(cpu); 11243 unsigned long interval; 11244 struct sched_domain *sd; 11245 /* Earliest time when we have to do rebalance again */ 11246 unsigned long next_balance = jiffies + 60*HZ; 11247 int update_next_balance = 0; 11248 int need_serialize, need_decay = 0; 11249 u64 max_cost = 0; 11250 11251 rcu_read_lock(); 11252 for_each_domain(cpu, sd) { 11253 /* 11254 * Decay the newidle max times here because this is a regular 11255 * visit to all the domains. 11256 */ 11257 need_decay = update_newidle_cost(sd, 0); 11258 max_cost += sd->max_newidle_lb_cost; 11259 11260 /* 11261 * Stop the load balance at this level. There is another 11262 * CPU in our sched group which is doing load balancing more 11263 * actively. 11264 */ 11265 if (!continue_balancing) { 11266 if (need_decay) 11267 continue; 11268 break; 11269 } 11270 11271 interval = get_sd_balance_interval(sd, busy); 11272 11273 need_serialize = sd->flags & SD_SERIALIZE; 11274 if (need_serialize) { 11275 if (!spin_trylock(&balancing)) 11276 goto out; 11277 } 11278 11279 if (time_after_eq(jiffies, sd->last_balance + interval)) { 11280 if (load_balance(cpu, rq, sd, idle, &continue_balancing)) { 11281 /* 11282 * The LBF_DST_PINNED logic could have changed 11283 * env->dst_cpu, so we can't know our idle 11284 * state even if we migrated tasks. Update it. 11285 */ 11286 idle = idle_cpu(cpu) ? CPU_IDLE : CPU_NOT_IDLE; 11287 busy = idle != CPU_IDLE && !sched_idle_cpu(cpu); 11288 } 11289 sd->last_balance = jiffies; 11290 interval = get_sd_balance_interval(sd, busy); 11291 } 11292 if (need_serialize) 11293 spin_unlock(&balancing); 11294 out: 11295 if (time_after(next_balance, sd->last_balance + interval)) { 11296 next_balance = sd->last_balance + interval; 11297 update_next_balance = 1; 11298 } 11299 } 11300 if (need_decay) { 11301 /* 11302 * Ensure the rq-wide value also decays but keep it at a 11303 * reasonable floor to avoid funnies with rq->avg_idle. 11304 */ 11305 rq->max_idle_balance_cost = 11306 max((u64)sysctl_sched_migration_cost, max_cost); 11307 } 11308 rcu_read_unlock(); 11309 11310 /* 11311 * next_balance will be updated only when there is a need. 11312 * When the cpu is attached to null domain for ex, it will not be 11313 * updated. 11314 */ 11315 if (likely(update_next_balance)) 11316 rq->next_balance = next_balance; 11317 11318 } 11319 11320 static inline int on_null_domain(struct rq *rq) 11321 { 11322 return unlikely(!rcu_dereference_sched(rq->sd)); 11323 } 11324 11325 #ifdef CONFIG_NO_HZ_COMMON 11326 /* 11327 * idle load balancing details 11328 * - When one of the busy CPUs notice that there may be an idle rebalancing 11329 * needed, they will kick the idle load balancer, which then does idle 11330 * load balancing for all the idle CPUs. 11331 * - HK_TYPE_MISC CPUs are used for this task, because HK_TYPE_SCHED not set 11332 * anywhere yet. 11333 */ 11334 11335 static inline int find_new_ilb(void) 11336 { 11337 int ilb; 11338 const struct cpumask *hk_mask; 11339 11340 hk_mask = housekeeping_cpumask(HK_TYPE_MISC); 11341 11342 for_each_cpu_and(ilb, nohz.idle_cpus_mask, hk_mask) { 11343 11344 if (ilb == smp_processor_id()) 11345 continue; 11346 11347 if (idle_cpu(ilb)) 11348 return ilb; 11349 } 11350 11351 return nr_cpu_ids; 11352 } 11353 11354 /* 11355 * Kick a CPU to do the nohz balancing, if it is time for it. We pick any 11356 * idle CPU in the HK_TYPE_MISC housekeeping set (if there is one). 11357 */ 11358 static void kick_ilb(unsigned int flags) 11359 { 11360 int ilb_cpu; 11361 11362 /* 11363 * Increase nohz.next_balance only when if full ilb is triggered but 11364 * not if we only update stats. 11365 */ 11366 if (flags & NOHZ_BALANCE_KICK) 11367 nohz.next_balance = jiffies+1; 11368 11369 ilb_cpu = find_new_ilb(); 11370 11371 if (ilb_cpu >= nr_cpu_ids) 11372 return; 11373 11374 /* 11375 * Access to rq::nohz_csd is serialized by NOHZ_KICK_MASK; he who sets 11376 * the first flag owns it; cleared by nohz_csd_func(). 11377 */ 11378 flags = atomic_fetch_or(flags, nohz_flags(ilb_cpu)); 11379 if (flags & NOHZ_KICK_MASK) 11380 return; 11381 11382 /* 11383 * This way we generate an IPI on the target CPU which 11384 * is idle. And the softirq performing nohz idle load balance 11385 * will be run before returning from the IPI. 11386 */ 11387 smp_call_function_single_async(ilb_cpu, &cpu_rq(ilb_cpu)->nohz_csd); 11388 } 11389 11390 /* 11391 * Current decision point for kicking the idle load balancer in the presence 11392 * of idle CPUs in the system. 11393 */ 11394 static void nohz_balancer_kick(struct rq *rq) 11395 { 11396 unsigned long now = jiffies; 11397 struct sched_domain_shared *sds; 11398 struct sched_domain *sd; 11399 int nr_busy, i, cpu = rq->cpu; 11400 unsigned int flags = 0; 11401 11402 if (unlikely(rq->idle_balance)) 11403 return; 11404 11405 /* 11406 * We may be recently in ticked or tickless idle mode. At the first 11407 * busy tick after returning from idle, we will update the busy stats. 11408 */ 11409 nohz_balance_exit_idle(rq); 11410 11411 /* 11412 * None are in tickless mode and hence no need for NOHZ idle load 11413 * balancing. 11414 */ 11415 if (likely(!atomic_read(&nohz.nr_cpus))) 11416 return; 11417 11418 if (READ_ONCE(nohz.has_blocked) && 11419 time_after(now, READ_ONCE(nohz.next_blocked))) 11420 flags = NOHZ_STATS_KICK; 11421 11422 if (time_before(now, nohz.next_balance)) 11423 goto out; 11424 11425 if (rq->nr_running >= 2) { 11426 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 11427 goto out; 11428 } 11429 11430 rcu_read_lock(); 11431 11432 sd = rcu_dereference(rq->sd); 11433 if (sd) { 11434 /* 11435 * If there's a CFS task and the current CPU has reduced 11436 * capacity; kick the ILB to see if there's a better CPU to run 11437 * on. 11438 */ 11439 if (rq->cfs.h_nr_running >= 1 && check_cpu_capacity(rq, sd)) { 11440 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 11441 goto unlock; 11442 } 11443 } 11444 11445 sd = rcu_dereference(per_cpu(sd_asym_packing, cpu)); 11446 if (sd) { 11447 /* 11448 * When ASYM_PACKING; see if there's a more preferred CPU 11449 * currently idle; in which case, kick the ILB to move tasks 11450 * around. 11451 * 11452 * When balancing betwen cores, all the SMT siblings of the 11453 * preferred CPU must be idle. 11454 */ 11455 for_each_cpu_and(i, sched_domain_span(sd), nohz.idle_cpus_mask) { 11456 if (sched_use_asym_prio(sd, i) && 11457 sched_asym_prefer(i, cpu)) { 11458 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 11459 goto unlock; 11460 } 11461 } 11462 } 11463 11464 sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, cpu)); 11465 if (sd) { 11466 /* 11467 * When ASYM_CPUCAPACITY; see if there's a higher capacity CPU 11468 * to run the misfit task on. 11469 */ 11470 if (check_misfit_status(rq, sd)) { 11471 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 11472 goto unlock; 11473 } 11474 11475 /* 11476 * For asymmetric systems, we do not want to nicely balance 11477 * cache use, instead we want to embrace asymmetry and only 11478 * ensure tasks have enough CPU capacity. 11479 * 11480 * Skip the LLC logic because it's not relevant in that case. 11481 */ 11482 goto unlock; 11483 } 11484 11485 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); 11486 if (sds) { 11487 /* 11488 * If there is an imbalance between LLC domains (IOW we could 11489 * increase the overall cache use), we need some less-loaded LLC 11490 * domain to pull some load. Likewise, we may need to spread 11491 * load within the current LLC domain (e.g. packed SMT cores but 11492 * other CPUs are idle). We can't really know from here how busy 11493 * the others are - so just get a nohz balance going if it looks 11494 * like this LLC domain has tasks we could move. 11495 */ 11496 nr_busy = atomic_read(&sds->nr_busy_cpus); 11497 if (nr_busy > 1) { 11498 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 11499 goto unlock; 11500 } 11501 } 11502 unlock: 11503 rcu_read_unlock(); 11504 out: 11505 if (READ_ONCE(nohz.needs_update)) 11506 flags |= NOHZ_NEXT_KICK; 11507 11508 if (flags) 11509 kick_ilb(flags); 11510 } 11511 11512 static void set_cpu_sd_state_busy(int cpu) 11513 { 11514 struct sched_domain *sd; 11515 11516 rcu_read_lock(); 11517 sd = rcu_dereference(per_cpu(sd_llc, cpu)); 11518 11519 if (!sd || !sd->nohz_idle) 11520 goto unlock; 11521 sd->nohz_idle = 0; 11522 11523 atomic_inc(&sd->shared->nr_busy_cpus); 11524 unlock: 11525 rcu_read_unlock(); 11526 } 11527 11528 void nohz_balance_exit_idle(struct rq *rq) 11529 { 11530 SCHED_WARN_ON(rq != this_rq()); 11531 11532 if (likely(!rq->nohz_tick_stopped)) 11533 return; 11534 11535 rq->nohz_tick_stopped = 0; 11536 cpumask_clear_cpu(rq->cpu, nohz.idle_cpus_mask); 11537 atomic_dec(&nohz.nr_cpus); 11538 11539 set_cpu_sd_state_busy(rq->cpu); 11540 } 11541 11542 static void set_cpu_sd_state_idle(int cpu) 11543 { 11544 struct sched_domain *sd; 11545 11546 rcu_read_lock(); 11547 sd = rcu_dereference(per_cpu(sd_llc, cpu)); 11548 11549 if (!sd || sd->nohz_idle) 11550 goto unlock; 11551 sd->nohz_idle = 1; 11552 11553 atomic_dec(&sd->shared->nr_busy_cpus); 11554 unlock: 11555 rcu_read_unlock(); 11556 } 11557 11558 /* 11559 * This routine will record that the CPU is going idle with tick stopped. 11560 * This info will be used in performing idle load balancing in the future. 11561 */ 11562 void nohz_balance_enter_idle(int cpu) 11563 { 11564 struct rq *rq = cpu_rq(cpu); 11565 11566 SCHED_WARN_ON(cpu != smp_processor_id()); 11567 11568 /* If this CPU is going down, then nothing needs to be done: */ 11569 if (!cpu_active(cpu)) 11570 return; 11571 11572 /* Spare idle load balancing on CPUs that don't want to be disturbed: */ 11573 if (!housekeeping_cpu(cpu, HK_TYPE_SCHED)) 11574 return; 11575 11576 /* 11577 * Can be set safely without rq->lock held 11578 * If a clear happens, it will have evaluated last additions because 11579 * rq->lock is held during the check and the clear 11580 */ 11581 rq->has_blocked_load = 1; 11582 11583 /* 11584 * The tick is still stopped but load could have been added in the 11585 * meantime. We set the nohz.has_blocked flag to trig a check of the 11586 * *_avg. The CPU is already part of nohz.idle_cpus_mask so the clear 11587 * of nohz.has_blocked can only happen after checking the new load 11588 */ 11589 if (rq->nohz_tick_stopped) 11590 goto out; 11591 11592 /* If we're a completely isolated CPU, we don't play: */ 11593 if (on_null_domain(rq)) 11594 return; 11595 11596 rq->nohz_tick_stopped = 1; 11597 11598 cpumask_set_cpu(cpu, nohz.idle_cpus_mask); 11599 atomic_inc(&nohz.nr_cpus); 11600 11601 /* 11602 * Ensures that if nohz_idle_balance() fails to observe our 11603 * @idle_cpus_mask store, it must observe the @has_blocked 11604 * and @needs_update stores. 11605 */ 11606 smp_mb__after_atomic(); 11607 11608 set_cpu_sd_state_idle(cpu); 11609 11610 WRITE_ONCE(nohz.needs_update, 1); 11611 out: 11612 /* 11613 * Each time a cpu enter idle, we assume that it has blocked load and 11614 * enable the periodic update of the load of idle cpus 11615 */ 11616 WRITE_ONCE(nohz.has_blocked, 1); 11617 } 11618 11619 static bool update_nohz_stats(struct rq *rq) 11620 { 11621 unsigned int cpu = rq->cpu; 11622 11623 if (!rq->has_blocked_load) 11624 return false; 11625 11626 if (!cpumask_test_cpu(cpu, nohz.idle_cpus_mask)) 11627 return false; 11628 11629 if (!time_after(jiffies, READ_ONCE(rq->last_blocked_load_update_tick))) 11630 return true; 11631 11632 update_blocked_averages(cpu); 11633 11634 return rq->has_blocked_load; 11635 } 11636 11637 /* 11638 * Internal function that runs load balance for all idle cpus. The load balance 11639 * can be a simple update of blocked load or a complete load balance with 11640 * tasks movement depending of flags. 11641 */ 11642 static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags) 11643 { 11644 /* Earliest time when we have to do rebalance again */ 11645 unsigned long now = jiffies; 11646 unsigned long next_balance = now + 60*HZ; 11647 bool has_blocked_load = false; 11648 int update_next_balance = 0; 11649 int this_cpu = this_rq->cpu; 11650 int balance_cpu; 11651 struct rq *rq; 11652 11653 SCHED_WARN_ON((flags & NOHZ_KICK_MASK) == NOHZ_BALANCE_KICK); 11654 11655 /* 11656 * We assume there will be no idle load after this update and clear 11657 * the has_blocked flag. If a cpu enters idle in the mean time, it will 11658 * set the has_blocked flag and trigger another update of idle load. 11659 * Because a cpu that becomes idle, is added to idle_cpus_mask before 11660 * setting the flag, we are sure to not clear the state and not 11661 * check the load of an idle cpu. 11662 * 11663 * Same applies to idle_cpus_mask vs needs_update. 11664 */ 11665 if (flags & NOHZ_STATS_KICK) 11666 WRITE_ONCE(nohz.has_blocked, 0); 11667 if (flags & NOHZ_NEXT_KICK) 11668 WRITE_ONCE(nohz.needs_update, 0); 11669 11670 /* 11671 * Ensures that if we miss the CPU, we must see the has_blocked 11672 * store from nohz_balance_enter_idle(). 11673 */ 11674 smp_mb(); 11675 11676 /* 11677 * Start with the next CPU after this_cpu so we will end with this_cpu and let a 11678 * chance for other idle cpu to pull load. 11679 */ 11680 for_each_cpu_wrap(balance_cpu, nohz.idle_cpus_mask, this_cpu+1) { 11681 if (!idle_cpu(balance_cpu)) 11682 continue; 11683 11684 /* 11685 * If this CPU gets work to do, stop the load balancing 11686 * work being done for other CPUs. Next load 11687 * balancing owner will pick it up. 11688 */ 11689 if (need_resched()) { 11690 if (flags & NOHZ_STATS_KICK) 11691 has_blocked_load = true; 11692 if (flags & NOHZ_NEXT_KICK) 11693 WRITE_ONCE(nohz.needs_update, 1); 11694 goto abort; 11695 } 11696 11697 rq = cpu_rq(balance_cpu); 11698 11699 if (flags & NOHZ_STATS_KICK) 11700 has_blocked_load |= update_nohz_stats(rq); 11701 11702 /* 11703 * If time for next balance is due, 11704 * do the balance. 11705 */ 11706 if (time_after_eq(jiffies, rq->next_balance)) { 11707 struct rq_flags rf; 11708 11709 rq_lock_irqsave(rq, &rf); 11710 update_rq_clock(rq); 11711 rq_unlock_irqrestore(rq, &rf); 11712 11713 if (flags & NOHZ_BALANCE_KICK) 11714 rebalance_domains(rq, CPU_IDLE); 11715 } 11716 11717 if (time_after(next_balance, rq->next_balance)) { 11718 next_balance = rq->next_balance; 11719 update_next_balance = 1; 11720 } 11721 } 11722 11723 /* 11724 * next_balance will be updated only when there is a need. 11725 * When the CPU is attached to null domain for ex, it will not be 11726 * updated. 11727 */ 11728 if (likely(update_next_balance)) 11729 nohz.next_balance = next_balance; 11730 11731 if (flags & NOHZ_STATS_KICK) 11732 WRITE_ONCE(nohz.next_blocked, 11733 now + msecs_to_jiffies(LOAD_AVG_PERIOD)); 11734 11735 abort: 11736 /* There is still blocked load, enable periodic update */ 11737 if (has_blocked_load) 11738 WRITE_ONCE(nohz.has_blocked, 1); 11739 } 11740 11741 /* 11742 * In CONFIG_NO_HZ_COMMON case, the idle balance kickee will do the 11743 * rebalancing for all the cpus for whom scheduler ticks are stopped. 11744 */ 11745 static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) 11746 { 11747 unsigned int flags = this_rq->nohz_idle_balance; 11748 11749 if (!flags) 11750 return false; 11751 11752 this_rq->nohz_idle_balance = 0; 11753 11754 if (idle != CPU_IDLE) 11755 return false; 11756 11757 _nohz_idle_balance(this_rq, flags); 11758 11759 return true; 11760 } 11761 11762 /* 11763 * Check if we need to run the ILB for updating blocked load before entering 11764 * idle state. 11765 */ 11766 void nohz_run_idle_balance(int cpu) 11767 { 11768 unsigned int flags; 11769 11770 flags = atomic_fetch_andnot(NOHZ_NEWILB_KICK, nohz_flags(cpu)); 11771 11772 /* 11773 * Update the blocked load only if no SCHED_SOFTIRQ is about to happen 11774 * (ie NOHZ_STATS_KICK set) and will do the same. 11775 */ 11776 if ((flags == NOHZ_NEWILB_KICK) && !need_resched()) 11777 _nohz_idle_balance(cpu_rq(cpu), NOHZ_STATS_KICK); 11778 } 11779 11780 static void nohz_newidle_balance(struct rq *this_rq) 11781 { 11782 int this_cpu = this_rq->cpu; 11783 11784 /* 11785 * This CPU doesn't want to be disturbed by scheduler 11786 * housekeeping 11787 */ 11788 if (!housekeeping_cpu(this_cpu, HK_TYPE_SCHED)) 11789 return; 11790 11791 /* Will wake up very soon. No time for doing anything else*/ 11792 if (this_rq->avg_idle < sysctl_sched_migration_cost) 11793 return; 11794 11795 /* Don't need to update blocked load of idle CPUs*/ 11796 if (!READ_ONCE(nohz.has_blocked) || 11797 time_before(jiffies, READ_ONCE(nohz.next_blocked))) 11798 return; 11799 11800 /* 11801 * Set the need to trigger ILB in order to update blocked load 11802 * before entering idle state. 11803 */ 11804 atomic_or(NOHZ_NEWILB_KICK, nohz_flags(this_cpu)); 11805 } 11806 11807 #else /* !CONFIG_NO_HZ_COMMON */ 11808 static inline void nohz_balancer_kick(struct rq *rq) { } 11809 11810 static inline bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) 11811 { 11812 return false; 11813 } 11814 11815 static inline void nohz_newidle_balance(struct rq *this_rq) { } 11816 #endif /* CONFIG_NO_HZ_COMMON */ 11817 11818 /* 11819 * newidle_balance is called by schedule() if this_cpu is about to become 11820 * idle. Attempts to pull tasks from other CPUs. 11821 * 11822 * Returns: 11823 * < 0 - we released the lock and there are !fair tasks present 11824 * 0 - failed, no new tasks 11825 * > 0 - success, new (fair) tasks present 11826 */ 11827 static int newidle_balance(struct rq *this_rq, struct rq_flags *rf) 11828 { 11829 unsigned long next_balance = jiffies + HZ; 11830 int this_cpu = this_rq->cpu; 11831 u64 t0, t1, curr_cost = 0; 11832 struct sched_domain *sd; 11833 int pulled_task = 0; 11834 11835 update_misfit_status(NULL, this_rq); 11836 11837 /* 11838 * There is a task waiting to run. No need to search for one. 11839 * Return 0; the task will be enqueued when switching to idle. 11840 */ 11841 if (this_rq->ttwu_pending) 11842 return 0; 11843 11844 /* 11845 * We must set idle_stamp _before_ calling idle_balance(), such that we 11846 * measure the duration of idle_balance() as idle time. 11847 */ 11848 this_rq->idle_stamp = rq_clock(this_rq); 11849 11850 /* 11851 * Do not pull tasks towards !active CPUs... 11852 */ 11853 if (!cpu_active(this_cpu)) 11854 return 0; 11855 11856 /* 11857 * This is OK, because current is on_cpu, which avoids it being picked 11858 * for load-balance and preemption/IRQs are still disabled avoiding 11859 * further scheduler activity on it and we're being very careful to 11860 * re-start the picking loop. 11861 */ 11862 rq_unpin_lock(this_rq, rf); 11863 11864 rcu_read_lock(); 11865 sd = rcu_dereference_check_sched_domain(this_rq->sd); 11866 11867 if (!READ_ONCE(this_rq->rd->overload) || 11868 (sd && this_rq->avg_idle < sd->max_newidle_lb_cost)) { 11869 11870 if (sd) 11871 update_next_balance(sd, &next_balance); 11872 rcu_read_unlock(); 11873 11874 goto out; 11875 } 11876 rcu_read_unlock(); 11877 11878 raw_spin_rq_unlock(this_rq); 11879 11880 t0 = sched_clock_cpu(this_cpu); 11881 update_blocked_averages(this_cpu); 11882 11883 rcu_read_lock(); 11884 for_each_domain(this_cpu, sd) { 11885 int continue_balancing = 1; 11886 u64 domain_cost; 11887 11888 update_next_balance(sd, &next_balance); 11889 11890 if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost) 11891 break; 11892 11893 if (sd->flags & SD_BALANCE_NEWIDLE) { 11894 11895 pulled_task = load_balance(this_cpu, this_rq, 11896 sd, CPU_NEWLY_IDLE, 11897 &continue_balancing); 11898 11899 t1 = sched_clock_cpu(this_cpu); 11900 domain_cost = t1 - t0; 11901 update_newidle_cost(sd, domain_cost); 11902 11903 curr_cost += domain_cost; 11904 t0 = t1; 11905 } 11906 11907 /* 11908 * Stop searching for tasks to pull if there are 11909 * now runnable tasks on this rq. 11910 */ 11911 if (pulled_task || this_rq->nr_running > 0 || 11912 this_rq->ttwu_pending) 11913 break; 11914 } 11915 rcu_read_unlock(); 11916 11917 raw_spin_rq_lock(this_rq); 11918 11919 if (curr_cost > this_rq->max_idle_balance_cost) 11920 this_rq->max_idle_balance_cost = curr_cost; 11921 11922 /* 11923 * While browsing the domains, we released the rq lock, a task could 11924 * have been enqueued in the meantime. Since we're not going idle, 11925 * pretend we pulled a task. 11926 */ 11927 if (this_rq->cfs.h_nr_running && !pulled_task) 11928 pulled_task = 1; 11929 11930 /* Is there a task of a high priority class? */ 11931 if (this_rq->nr_running != this_rq->cfs.h_nr_running) 11932 pulled_task = -1; 11933 11934 out: 11935 /* Move the next balance forward */ 11936 if (time_after(this_rq->next_balance, next_balance)) 11937 this_rq->next_balance = next_balance; 11938 11939 if (pulled_task) 11940 this_rq->idle_stamp = 0; 11941 else 11942 nohz_newidle_balance(this_rq); 11943 11944 rq_repin_lock(this_rq, rf); 11945 11946 return pulled_task; 11947 } 11948 11949 /* 11950 * run_rebalance_domains is triggered when needed from the scheduler tick. 11951 * Also triggered for nohz idle balancing (with nohz_balancing_kick set). 11952 */ 11953 static __latent_entropy void run_rebalance_domains(struct softirq_action *h) 11954 { 11955 struct rq *this_rq = this_rq(); 11956 enum cpu_idle_type idle = this_rq->idle_balance ? 11957 CPU_IDLE : CPU_NOT_IDLE; 11958 11959 /* 11960 * If this CPU has a pending nohz_balance_kick, then do the 11961 * balancing on behalf of the other idle CPUs whose ticks are 11962 * stopped. Do nohz_idle_balance *before* rebalance_domains to 11963 * give the idle CPUs a chance to load balance. Else we may 11964 * load balance only within the local sched_domain hierarchy 11965 * and abort nohz_idle_balance altogether if we pull some load. 11966 */ 11967 if (nohz_idle_balance(this_rq, idle)) 11968 return; 11969 11970 /* normal load balance */ 11971 update_blocked_averages(this_rq->cpu); 11972 rebalance_domains(this_rq, idle); 11973 } 11974 11975 /* 11976 * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing. 11977 */ 11978 void trigger_load_balance(struct rq *rq) 11979 { 11980 /* 11981 * Don't need to rebalance while attached to NULL domain or 11982 * runqueue CPU is not active 11983 */ 11984 if (unlikely(on_null_domain(rq) || !cpu_active(cpu_of(rq)))) 11985 return; 11986 11987 if (time_after_eq(jiffies, rq->next_balance)) 11988 raise_softirq(SCHED_SOFTIRQ); 11989 11990 nohz_balancer_kick(rq); 11991 } 11992 11993 static void rq_online_fair(struct rq *rq) 11994 { 11995 update_sysctl(); 11996 11997 update_runtime_enabled(rq); 11998 } 11999 12000 static void rq_offline_fair(struct rq *rq) 12001 { 12002 update_sysctl(); 12003 12004 /* Ensure any throttled groups are reachable by pick_next_task */ 12005 unthrottle_offline_cfs_rqs(rq); 12006 } 12007 12008 #endif /* CONFIG_SMP */ 12009 12010 #ifdef CONFIG_SCHED_CORE 12011 static inline bool 12012 __entity_slice_used(struct sched_entity *se, int min_nr_tasks) 12013 { 12014 u64 slice = sched_slice(cfs_rq_of(se), se); 12015 u64 rtime = se->sum_exec_runtime - se->prev_sum_exec_runtime; 12016 12017 return (rtime * min_nr_tasks > slice); 12018 } 12019 12020 #define MIN_NR_TASKS_DURING_FORCEIDLE 2 12021 static inline void task_tick_core(struct rq *rq, struct task_struct *curr) 12022 { 12023 if (!sched_core_enabled(rq)) 12024 return; 12025 12026 /* 12027 * If runqueue has only one task which used up its slice and 12028 * if the sibling is forced idle, then trigger schedule to 12029 * give forced idle task a chance. 12030 * 12031 * sched_slice() considers only this active rq and it gets the 12032 * whole slice. But during force idle, we have siblings acting 12033 * like a single runqueue and hence we need to consider runnable 12034 * tasks on this CPU and the forced idle CPU. Ideally, we should 12035 * go through the forced idle rq, but that would be a perf hit. 12036 * We can assume that the forced idle CPU has at least 12037 * MIN_NR_TASKS_DURING_FORCEIDLE - 1 tasks and use that to check 12038 * if we need to give up the CPU. 12039 */ 12040 if (rq->core->core_forceidle_count && rq->cfs.nr_running == 1 && 12041 __entity_slice_used(&curr->se, MIN_NR_TASKS_DURING_FORCEIDLE)) 12042 resched_curr(rq); 12043 } 12044 12045 /* 12046 * se_fi_update - Update the cfs_rq->min_vruntime_fi in a CFS hierarchy if needed. 12047 */ 12048 static void se_fi_update(const struct sched_entity *se, unsigned int fi_seq, 12049 bool forceidle) 12050 { 12051 for_each_sched_entity(se) { 12052 struct cfs_rq *cfs_rq = cfs_rq_of(se); 12053 12054 if (forceidle) { 12055 if (cfs_rq->forceidle_seq == fi_seq) 12056 break; 12057 cfs_rq->forceidle_seq = fi_seq; 12058 } 12059 12060 cfs_rq->min_vruntime_fi = cfs_rq->min_vruntime; 12061 } 12062 } 12063 12064 void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi) 12065 { 12066 struct sched_entity *se = &p->se; 12067 12068 if (p->sched_class != &fair_sched_class) 12069 return; 12070 12071 se_fi_update(se, rq->core->core_forceidle_seq, in_fi); 12072 } 12073 12074 bool cfs_prio_less(const struct task_struct *a, const struct task_struct *b, 12075 bool in_fi) 12076 { 12077 struct rq *rq = task_rq(a); 12078 const struct sched_entity *sea = &a->se; 12079 const struct sched_entity *seb = &b->se; 12080 struct cfs_rq *cfs_rqa; 12081 struct cfs_rq *cfs_rqb; 12082 s64 delta; 12083 12084 SCHED_WARN_ON(task_rq(b)->core != rq->core); 12085 12086 #ifdef CONFIG_FAIR_GROUP_SCHED 12087 /* 12088 * Find an se in the hierarchy for tasks a and b, such that the se's 12089 * are immediate siblings. 12090 */ 12091 while (sea->cfs_rq->tg != seb->cfs_rq->tg) { 12092 int sea_depth = sea->depth; 12093 int seb_depth = seb->depth; 12094 12095 if (sea_depth >= seb_depth) 12096 sea = parent_entity(sea); 12097 if (sea_depth <= seb_depth) 12098 seb = parent_entity(seb); 12099 } 12100 12101 se_fi_update(sea, rq->core->core_forceidle_seq, in_fi); 12102 se_fi_update(seb, rq->core->core_forceidle_seq, in_fi); 12103 12104 cfs_rqa = sea->cfs_rq; 12105 cfs_rqb = seb->cfs_rq; 12106 #else 12107 cfs_rqa = &task_rq(a)->cfs; 12108 cfs_rqb = &task_rq(b)->cfs; 12109 #endif 12110 12111 /* 12112 * Find delta after normalizing se's vruntime with its cfs_rq's 12113 * min_vruntime_fi, which would have been updated in prior calls 12114 * to se_fi_update(). 12115 */ 12116 delta = (s64)(sea->vruntime - seb->vruntime) + 12117 (s64)(cfs_rqb->min_vruntime_fi - cfs_rqa->min_vruntime_fi); 12118 12119 return delta > 0; 12120 } 12121 12122 static int task_is_throttled_fair(struct task_struct *p, int cpu) 12123 { 12124 struct cfs_rq *cfs_rq; 12125 12126 #ifdef CONFIG_FAIR_GROUP_SCHED 12127 cfs_rq = task_group(p)->cfs_rq[cpu]; 12128 #else 12129 cfs_rq = &cpu_rq(cpu)->cfs; 12130 #endif 12131 return throttled_hierarchy(cfs_rq); 12132 } 12133 #else 12134 static inline void task_tick_core(struct rq *rq, struct task_struct *curr) {} 12135 #endif 12136 12137 /* 12138 * scheduler tick hitting a task of our scheduling class. 12139 * 12140 * NOTE: This function can be called remotely by the tick offload that 12141 * goes along full dynticks. Therefore no local assumption can be made 12142 * and everything must be accessed through the @rq and @curr passed in 12143 * parameters. 12144 */ 12145 static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) 12146 { 12147 struct cfs_rq *cfs_rq; 12148 struct sched_entity *se = &curr->se; 12149 12150 for_each_sched_entity(se) { 12151 cfs_rq = cfs_rq_of(se); 12152 entity_tick(cfs_rq, se, queued); 12153 } 12154 12155 if (static_branch_unlikely(&sched_numa_balancing)) 12156 task_tick_numa(rq, curr); 12157 12158 update_misfit_status(curr, rq); 12159 update_overutilized_status(task_rq(curr)); 12160 12161 task_tick_core(rq, curr); 12162 } 12163 12164 /* 12165 * called on fork with the child task as argument from the parent's context 12166 * - child not yet on the tasklist 12167 * - preemption disabled 12168 */ 12169 static void task_fork_fair(struct task_struct *p) 12170 { 12171 struct cfs_rq *cfs_rq; 12172 struct sched_entity *se = &p->se, *curr; 12173 struct rq *rq = this_rq(); 12174 struct rq_flags rf; 12175 12176 rq_lock(rq, &rf); 12177 update_rq_clock(rq); 12178 12179 cfs_rq = task_cfs_rq(current); 12180 curr = cfs_rq->curr; 12181 if (curr) { 12182 update_curr(cfs_rq); 12183 se->vruntime = curr->vruntime; 12184 } 12185 place_entity(cfs_rq, se, 1); 12186 12187 if (sysctl_sched_child_runs_first && curr && entity_before(curr, se)) { 12188 /* 12189 * Upon rescheduling, sched_class::put_prev_task() will place 12190 * 'current' within the tree based on its new key value. 12191 */ 12192 swap(curr->vruntime, se->vruntime); 12193 resched_curr(rq); 12194 } 12195 12196 se->vruntime -= cfs_rq->min_vruntime; 12197 rq_unlock(rq, &rf); 12198 } 12199 12200 /* 12201 * Priority of the task has changed. Check to see if we preempt 12202 * the current task. 12203 */ 12204 static void 12205 prio_changed_fair(struct rq *rq, struct task_struct *p, int oldprio) 12206 { 12207 if (!task_on_rq_queued(p)) 12208 return; 12209 12210 if (rq->cfs.nr_running == 1) 12211 return; 12212 12213 /* 12214 * Reschedule if we are currently running on this runqueue and 12215 * our priority decreased, or if we are not currently running on 12216 * this runqueue and our priority is higher than the current's 12217 */ 12218 if (task_current(rq, p)) { 12219 if (p->prio > oldprio) 12220 resched_curr(rq); 12221 } else 12222 check_preempt_curr(rq, p, 0); 12223 } 12224 12225 static inline bool vruntime_normalized(struct task_struct *p) 12226 { 12227 struct sched_entity *se = &p->se; 12228 12229 /* 12230 * In both the TASK_ON_RQ_QUEUED and TASK_ON_RQ_MIGRATING cases, 12231 * the dequeue_entity(.flags=0) will already have normalized the 12232 * vruntime. 12233 */ 12234 if (p->on_rq) 12235 return true; 12236 12237 /* 12238 * When !on_rq, vruntime of the task has usually NOT been normalized. 12239 * But there are some cases where it has already been normalized: 12240 * 12241 * - A forked child which is waiting for being woken up by 12242 * wake_up_new_task(). 12243 * - A task which has been woken up by try_to_wake_up() and 12244 * waiting for actually being woken up by sched_ttwu_pending(). 12245 */ 12246 if (!se->sum_exec_runtime || 12247 (READ_ONCE(p->__state) == TASK_WAKING && p->sched_remote_wakeup)) 12248 return true; 12249 12250 return false; 12251 } 12252 12253 #ifdef CONFIG_FAIR_GROUP_SCHED 12254 /* 12255 * Propagate the changes of the sched_entity across the tg tree to make it 12256 * visible to the root 12257 */ 12258 static void propagate_entity_cfs_rq(struct sched_entity *se) 12259 { 12260 struct cfs_rq *cfs_rq = cfs_rq_of(se); 12261 12262 if (cfs_rq_throttled(cfs_rq)) 12263 return; 12264 12265 if (!throttled_hierarchy(cfs_rq)) 12266 list_add_leaf_cfs_rq(cfs_rq); 12267 12268 /* Start to propagate at parent */ 12269 se = se->parent; 12270 12271 for_each_sched_entity(se) { 12272 cfs_rq = cfs_rq_of(se); 12273 12274 update_load_avg(cfs_rq, se, UPDATE_TG); 12275 12276 if (cfs_rq_throttled(cfs_rq)) 12277 break; 12278 12279 if (!throttled_hierarchy(cfs_rq)) 12280 list_add_leaf_cfs_rq(cfs_rq); 12281 } 12282 } 12283 #else 12284 static void propagate_entity_cfs_rq(struct sched_entity *se) { } 12285 #endif 12286 12287 static void detach_entity_cfs_rq(struct sched_entity *se) 12288 { 12289 struct cfs_rq *cfs_rq = cfs_rq_of(se); 12290 12291 #ifdef CONFIG_SMP 12292 /* 12293 * In case the task sched_avg hasn't been attached: 12294 * - A forked task which hasn't been woken up by wake_up_new_task(). 12295 * - A task which has been woken up by try_to_wake_up() but is 12296 * waiting for actually being woken up by sched_ttwu_pending(). 12297 */ 12298 if (!se->avg.last_update_time) 12299 return; 12300 #endif 12301 12302 /* Catch up with the cfs_rq and remove our load when we leave */ 12303 update_load_avg(cfs_rq, se, 0); 12304 detach_entity_load_avg(cfs_rq, se); 12305 update_tg_load_avg(cfs_rq); 12306 propagate_entity_cfs_rq(se); 12307 } 12308 12309 static void attach_entity_cfs_rq(struct sched_entity *se) 12310 { 12311 struct cfs_rq *cfs_rq = cfs_rq_of(se); 12312 12313 /* Synchronize entity with its cfs_rq */ 12314 update_load_avg(cfs_rq, se, sched_feat(ATTACH_AGE_LOAD) ? 0 : SKIP_AGE_LOAD); 12315 attach_entity_load_avg(cfs_rq, se); 12316 update_tg_load_avg(cfs_rq); 12317 propagate_entity_cfs_rq(se); 12318 } 12319 12320 static void detach_task_cfs_rq(struct task_struct *p) 12321 { 12322 struct sched_entity *se = &p->se; 12323 struct cfs_rq *cfs_rq = cfs_rq_of(se); 12324 12325 if (!vruntime_normalized(p)) { 12326 /* 12327 * Fix up our vruntime so that the current sleep doesn't 12328 * cause 'unlimited' sleep bonus. 12329 */ 12330 place_entity(cfs_rq, se, 0); 12331 se->vruntime -= cfs_rq->min_vruntime; 12332 } 12333 12334 detach_entity_cfs_rq(se); 12335 } 12336 12337 static void attach_task_cfs_rq(struct task_struct *p) 12338 { 12339 struct sched_entity *se = &p->se; 12340 struct cfs_rq *cfs_rq = cfs_rq_of(se); 12341 12342 attach_entity_cfs_rq(se); 12343 12344 if (!vruntime_normalized(p)) 12345 se->vruntime += cfs_rq->min_vruntime; 12346 } 12347 12348 static void switched_from_fair(struct rq *rq, struct task_struct *p) 12349 { 12350 detach_task_cfs_rq(p); 12351 } 12352 12353 static void switched_to_fair(struct rq *rq, struct task_struct *p) 12354 { 12355 attach_task_cfs_rq(p); 12356 12357 if (task_on_rq_queued(p)) { 12358 /* 12359 * We were most likely switched from sched_rt, so 12360 * kick off the schedule if running, otherwise just see 12361 * if we can still preempt the current task. 12362 */ 12363 if (task_current(rq, p)) 12364 resched_curr(rq); 12365 else 12366 check_preempt_curr(rq, p, 0); 12367 } 12368 } 12369 12370 /* Account for a task changing its policy or group. 12371 * 12372 * This routine is mostly called to set cfs_rq->curr field when a task 12373 * migrates between groups/classes. 12374 */ 12375 static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) 12376 { 12377 struct sched_entity *se = &p->se; 12378 12379 #ifdef CONFIG_SMP 12380 if (task_on_rq_queued(p)) { 12381 /* 12382 * Move the next running task to the front of the list, so our 12383 * cfs_tasks list becomes MRU one. 12384 */ 12385 list_move(&se->group_node, &rq->cfs_tasks); 12386 } 12387 #endif 12388 12389 for_each_sched_entity(se) { 12390 struct cfs_rq *cfs_rq = cfs_rq_of(se); 12391 12392 set_next_entity(cfs_rq, se); 12393 /* ensure bandwidth has been allocated on our new cfs_rq */ 12394 account_cfs_rq_runtime(cfs_rq, 0); 12395 } 12396 } 12397 12398 void init_cfs_rq(struct cfs_rq *cfs_rq) 12399 { 12400 cfs_rq->tasks_timeline = RB_ROOT_CACHED; 12401 u64_u32_store(cfs_rq->min_vruntime, (u64)(-(1LL << 20))); 12402 #ifdef CONFIG_SMP 12403 raw_spin_lock_init(&cfs_rq->removed.lock); 12404 #endif 12405 } 12406 12407 #ifdef CONFIG_FAIR_GROUP_SCHED 12408 static void task_change_group_fair(struct task_struct *p) 12409 { 12410 /* 12411 * We couldn't detach or attach a forked task which 12412 * hasn't been woken up by wake_up_new_task(). 12413 */ 12414 if (READ_ONCE(p->__state) == TASK_NEW) 12415 return; 12416 12417 detach_task_cfs_rq(p); 12418 12419 #ifdef CONFIG_SMP 12420 /* Tell se's cfs_rq has been changed -- migrated */ 12421 p->se.avg.last_update_time = 0; 12422 #endif 12423 set_task_rq(p, task_cpu(p)); 12424 attach_task_cfs_rq(p); 12425 } 12426 12427 void free_fair_sched_group(struct task_group *tg) 12428 { 12429 int i; 12430 12431 for_each_possible_cpu(i) { 12432 if (tg->cfs_rq) 12433 kfree(tg->cfs_rq[i]); 12434 if (tg->se) 12435 kfree(tg->se[i]); 12436 } 12437 12438 kfree(tg->cfs_rq); 12439 kfree(tg->se); 12440 } 12441 12442 int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) 12443 { 12444 struct sched_entity *se; 12445 struct cfs_rq *cfs_rq; 12446 int i; 12447 12448 tg->cfs_rq = kcalloc(nr_cpu_ids, sizeof(cfs_rq), GFP_KERNEL); 12449 if (!tg->cfs_rq) 12450 goto err; 12451 tg->se = kcalloc(nr_cpu_ids, sizeof(se), GFP_KERNEL); 12452 if (!tg->se) 12453 goto err; 12454 12455 tg->shares = NICE_0_LOAD; 12456 12457 init_cfs_bandwidth(tg_cfs_bandwidth(tg)); 12458 12459 for_each_possible_cpu(i) { 12460 cfs_rq = kzalloc_node(sizeof(struct cfs_rq), 12461 GFP_KERNEL, cpu_to_node(i)); 12462 if (!cfs_rq) 12463 goto err; 12464 12465 se = kzalloc_node(sizeof(struct sched_entity_stats), 12466 GFP_KERNEL, cpu_to_node(i)); 12467 if (!se) 12468 goto err_free_rq; 12469 12470 init_cfs_rq(cfs_rq); 12471 init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]); 12472 init_entity_runnable_average(se); 12473 } 12474 12475 return 1; 12476 12477 err_free_rq: 12478 kfree(cfs_rq); 12479 err: 12480 return 0; 12481 } 12482 12483 void online_fair_sched_group(struct task_group *tg) 12484 { 12485 struct sched_entity *se; 12486 struct rq_flags rf; 12487 struct rq *rq; 12488 int i; 12489 12490 for_each_possible_cpu(i) { 12491 rq = cpu_rq(i); 12492 se = tg->se[i]; 12493 rq_lock_irq(rq, &rf); 12494 update_rq_clock(rq); 12495 attach_entity_cfs_rq(se); 12496 sync_throttle(tg, i); 12497 rq_unlock_irq(rq, &rf); 12498 } 12499 } 12500 12501 void unregister_fair_sched_group(struct task_group *tg) 12502 { 12503 unsigned long flags; 12504 struct rq *rq; 12505 int cpu; 12506 12507 destroy_cfs_bandwidth(tg_cfs_bandwidth(tg)); 12508 12509 for_each_possible_cpu(cpu) { 12510 if (tg->se[cpu]) 12511 remove_entity_load_avg(tg->se[cpu]); 12512 12513 /* 12514 * Only empty task groups can be destroyed; so we can speculatively 12515 * check on_list without danger of it being re-added. 12516 */ 12517 if (!tg->cfs_rq[cpu]->on_list) 12518 continue; 12519 12520 rq = cpu_rq(cpu); 12521 12522 raw_spin_rq_lock_irqsave(rq, flags); 12523 list_del_leaf_cfs_rq(tg->cfs_rq[cpu]); 12524 raw_spin_rq_unlock_irqrestore(rq, flags); 12525 } 12526 } 12527 12528 void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq, 12529 struct sched_entity *se, int cpu, 12530 struct sched_entity *parent) 12531 { 12532 struct rq *rq = cpu_rq(cpu); 12533 12534 cfs_rq->tg = tg; 12535 cfs_rq->rq = rq; 12536 init_cfs_rq_runtime(cfs_rq); 12537 12538 tg->cfs_rq[cpu] = cfs_rq; 12539 tg->se[cpu] = se; 12540 12541 /* se could be NULL for root_task_group */ 12542 if (!se) 12543 return; 12544 12545 if (!parent) { 12546 se->cfs_rq = &rq->cfs; 12547 se->depth = 0; 12548 } else { 12549 se->cfs_rq = parent->my_q; 12550 se->depth = parent->depth + 1; 12551 } 12552 12553 se->my_q = cfs_rq; 12554 /* guarantee group entities always have weight */ 12555 update_load_set(&se->load, NICE_0_LOAD); 12556 se->parent = parent; 12557 } 12558 12559 static DEFINE_MUTEX(shares_mutex); 12560 12561 static int __sched_group_set_shares(struct task_group *tg, unsigned long shares) 12562 { 12563 int i; 12564 12565 lockdep_assert_held(&shares_mutex); 12566 12567 /* 12568 * We can't change the weight of the root cgroup. 12569 */ 12570 if (!tg->se[0]) 12571 return -EINVAL; 12572 12573 shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES)); 12574 12575 if (tg->shares == shares) 12576 return 0; 12577 12578 tg->shares = shares; 12579 for_each_possible_cpu(i) { 12580 struct rq *rq = cpu_rq(i); 12581 struct sched_entity *se = tg->se[i]; 12582 struct rq_flags rf; 12583 12584 /* Propagate contribution to hierarchy */ 12585 rq_lock_irqsave(rq, &rf); 12586 update_rq_clock(rq); 12587 for_each_sched_entity(se) { 12588 update_load_avg(cfs_rq_of(se), se, UPDATE_TG); 12589 update_cfs_group(se); 12590 } 12591 rq_unlock_irqrestore(rq, &rf); 12592 } 12593 12594 return 0; 12595 } 12596 12597 int sched_group_set_shares(struct task_group *tg, unsigned long shares) 12598 { 12599 int ret; 12600 12601 mutex_lock(&shares_mutex); 12602 if (tg_is_idle(tg)) 12603 ret = -EINVAL; 12604 else 12605 ret = __sched_group_set_shares(tg, shares); 12606 mutex_unlock(&shares_mutex); 12607 12608 return ret; 12609 } 12610 12611 int sched_group_set_idle(struct task_group *tg, long idle) 12612 { 12613 int i; 12614 12615 if (tg == &root_task_group) 12616 return -EINVAL; 12617 12618 if (idle < 0 || idle > 1) 12619 return -EINVAL; 12620 12621 mutex_lock(&shares_mutex); 12622 12623 if (tg->idle == idle) { 12624 mutex_unlock(&shares_mutex); 12625 return 0; 12626 } 12627 12628 tg->idle = idle; 12629 12630 for_each_possible_cpu(i) { 12631 struct rq *rq = cpu_rq(i); 12632 struct sched_entity *se = tg->se[i]; 12633 struct cfs_rq *parent_cfs_rq, *grp_cfs_rq = tg->cfs_rq[i]; 12634 bool was_idle = cfs_rq_is_idle(grp_cfs_rq); 12635 long idle_task_delta; 12636 struct rq_flags rf; 12637 12638 rq_lock_irqsave(rq, &rf); 12639 12640 grp_cfs_rq->idle = idle; 12641 if (WARN_ON_ONCE(was_idle == cfs_rq_is_idle(grp_cfs_rq))) 12642 goto next_cpu; 12643 12644 if (se->on_rq) { 12645 parent_cfs_rq = cfs_rq_of(se); 12646 if (cfs_rq_is_idle(grp_cfs_rq)) 12647 parent_cfs_rq->idle_nr_running++; 12648 else 12649 parent_cfs_rq->idle_nr_running--; 12650 } 12651 12652 idle_task_delta = grp_cfs_rq->h_nr_running - 12653 grp_cfs_rq->idle_h_nr_running; 12654 if (!cfs_rq_is_idle(grp_cfs_rq)) 12655 idle_task_delta *= -1; 12656 12657 for_each_sched_entity(se) { 12658 struct cfs_rq *cfs_rq = cfs_rq_of(se); 12659 12660 if (!se->on_rq) 12661 break; 12662 12663 cfs_rq->idle_h_nr_running += idle_task_delta; 12664 12665 /* Already accounted at parent level and above. */ 12666 if (cfs_rq_is_idle(cfs_rq)) 12667 break; 12668 } 12669 12670 next_cpu: 12671 rq_unlock_irqrestore(rq, &rf); 12672 } 12673 12674 /* Idle groups have minimum weight. */ 12675 if (tg_is_idle(tg)) 12676 __sched_group_set_shares(tg, scale_load(WEIGHT_IDLEPRIO)); 12677 else 12678 __sched_group_set_shares(tg, NICE_0_LOAD); 12679 12680 mutex_unlock(&shares_mutex); 12681 return 0; 12682 } 12683 12684 #else /* CONFIG_FAIR_GROUP_SCHED */ 12685 12686 void free_fair_sched_group(struct task_group *tg) { } 12687 12688 int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) 12689 { 12690 return 1; 12691 } 12692 12693 void online_fair_sched_group(struct task_group *tg) { } 12694 12695 void unregister_fair_sched_group(struct task_group *tg) { } 12696 12697 #endif /* CONFIG_FAIR_GROUP_SCHED */ 12698 12699 12700 static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task) 12701 { 12702 struct sched_entity *se = &task->se; 12703 unsigned int rr_interval = 0; 12704 12705 /* 12706 * Time slice is 0 for SCHED_OTHER tasks that are on an otherwise 12707 * idle runqueue: 12708 */ 12709 if (rq->cfs.load.weight) 12710 rr_interval = NS_TO_JIFFIES(sched_slice(cfs_rq_of(se), se)); 12711 12712 return rr_interval; 12713 } 12714 12715 /* 12716 * All the scheduling class methods: 12717 */ 12718 DEFINE_SCHED_CLASS(fair) = { 12719 12720 .enqueue_task = enqueue_task_fair, 12721 .dequeue_task = dequeue_task_fair, 12722 .yield_task = yield_task_fair, 12723 .yield_to_task = yield_to_task_fair, 12724 12725 .check_preempt_curr = check_preempt_wakeup, 12726 12727 .pick_next_task = __pick_next_task_fair, 12728 .put_prev_task = put_prev_task_fair, 12729 .set_next_task = set_next_task_fair, 12730 12731 #ifdef CONFIG_SMP 12732 .balance = balance_fair, 12733 .pick_task = pick_task_fair, 12734 .select_task_rq = select_task_rq_fair, 12735 .migrate_task_rq = migrate_task_rq_fair, 12736 12737 .rq_online = rq_online_fair, 12738 .rq_offline = rq_offline_fair, 12739 12740 .task_dead = task_dead_fair, 12741 .set_cpus_allowed = set_cpus_allowed_common, 12742 #endif 12743 12744 .task_tick = task_tick_fair, 12745 .task_fork = task_fork_fair, 12746 12747 .prio_changed = prio_changed_fair, 12748 .switched_from = switched_from_fair, 12749 .switched_to = switched_to_fair, 12750 12751 .get_rr_interval = get_rr_interval_fair, 12752 12753 .update_curr = update_curr_fair, 12754 12755 #ifdef CONFIG_FAIR_GROUP_SCHED 12756 .task_change_group = task_change_group_fair, 12757 #endif 12758 12759 #ifdef CONFIG_SCHED_CORE 12760 .task_is_throttled = task_is_throttled_fair, 12761 #endif 12762 12763 #ifdef CONFIG_UCLAMP_TASK 12764 .uclamp_enabled = 1, 12765 #endif 12766 }; 12767 12768 #ifdef CONFIG_SCHED_DEBUG 12769 void print_cfs_stats(struct seq_file *m, int cpu) 12770 { 12771 struct cfs_rq *cfs_rq, *pos; 12772 12773 rcu_read_lock(); 12774 for_each_leaf_cfs_rq_safe(cpu_rq(cpu), cfs_rq, pos) 12775 print_cfs_rq(m, cpu, cfs_rq); 12776 rcu_read_unlock(); 12777 } 12778 12779 #ifdef CONFIG_NUMA_BALANCING 12780 void show_numa_stats(struct task_struct *p, struct seq_file *m) 12781 { 12782 int node; 12783 unsigned long tsf = 0, tpf = 0, gsf = 0, gpf = 0; 12784 struct numa_group *ng; 12785 12786 rcu_read_lock(); 12787 ng = rcu_dereference(p->numa_group); 12788 for_each_online_node(node) { 12789 if (p->numa_faults) { 12790 tsf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 0)]; 12791 tpf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 1)]; 12792 } 12793 if (ng) { 12794 gsf = ng->faults[task_faults_idx(NUMA_MEM, node, 0)], 12795 gpf = ng->faults[task_faults_idx(NUMA_MEM, node, 1)]; 12796 } 12797 print_numa_stats(m, node, tsf, tpf, gsf, gpf); 12798 } 12799 rcu_read_unlock(); 12800 } 12801 #endif /* CONFIG_NUMA_BALANCING */ 12802 #endif /* CONFIG_SCHED_DEBUG */ 12803 12804 __init void init_sched_fair_class(void) 12805 { 12806 #ifdef CONFIG_SMP 12807 int i; 12808 12809 for_each_possible_cpu(i) { 12810 zalloc_cpumask_var_node(&per_cpu(load_balance_mask, i), GFP_KERNEL, cpu_to_node(i)); 12811 zalloc_cpumask_var_node(&per_cpu(select_rq_mask, i), GFP_KERNEL, cpu_to_node(i)); 12812 12813 #ifdef CONFIG_CFS_BANDWIDTH 12814 INIT_CSD(&cpu_rq(i)->cfsb_csd, __cfsb_csd_unthrottle, cpu_rq(i)); 12815 INIT_LIST_HEAD(&cpu_rq(i)->cfsb_csd_list); 12816 #endif 12817 } 12818 12819 open_softirq(SCHED_SOFTIRQ, run_rebalance_domains); 12820 12821 #ifdef CONFIG_NO_HZ_COMMON 12822 nohz.next_balance = jiffies; 12823 nohz.next_blocked = jiffies; 12824 zalloc_cpumask_var(&nohz.idle_cpus_mask, GFP_NOWAIT); 12825 #endif 12826 #endif /* SMP */ 12827 12828 } 12829