1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Completely Fair Scheduling (CFS) Class (SCHED_NORMAL/SCHED_BATCH) 4 * 5 * Copyright (C) 2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> 6 * 7 * Interactivity improvements by Mike Galbraith 8 * (C) 2007 Mike Galbraith <efault@gmx.de> 9 * 10 * Various enhancements by Dmitry Adamushko. 11 * (C) 2007 Dmitry Adamushko <dmitry.adamushko@gmail.com> 12 * 13 * Group scheduling enhancements by Srivatsa Vaddagiri 14 * Copyright IBM Corporation, 2007 15 * Author: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com> 16 * 17 * Scaled math optimizations by Thomas Gleixner 18 * Copyright (C) 2007, Thomas Gleixner <tglx@linutronix.de> 19 * 20 * Adaptive scheduling granularity, math enhancements by Peter Zijlstra 21 * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra 22 */ 23 #include <linux/energy_model.h> 24 #include <linux/mmap_lock.h> 25 #include <linux/hugetlb_inline.h> 26 #include <linux/jiffies.h> 27 #include <linux/mm_api.h> 28 #include <linux/highmem.h> 29 #include <linux/spinlock_api.h> 30 #include <linux/cpumask_api.h> 31 #include <linux/lockdep_api.h> 32 #include <linux/softirq.h> 33 #include <linux/refcount_api.h> 34 #include <linux/topology.h> 35 #include <linux/sched/clock.h> 36 #include <linux/sched/cond_resched.h> 37 #include <linux/sched/cputime.h> 38 #include <linux/sched/isolation.h> 39 #include <linux/sched/nohz.h> 40 41 #include <linux/cpuidle.h> 42 #include <linux/interrupt.h> 43 #include <linux/memory-tiers.h> 44 #include <linux/mempolicy.h> 45 #include <linux/mutex_api.h> 46 #include <linux/profile.h> 47 #include <linux/psi.h> 48 #include <linux/ratelimit.h> 49 #include <linux/task_work.h> 50 #include <linux/rbtree_augmented.h> 51 52 #include <asm/switch_to.h> 53 54 #include <linux/sched/cond_resched.h> 55 56 #include "sched.h" 57 #include "stats.h" 58 #include "autogroup.h" 59 60 /* 61 * The initial- and re-scaling of tunables is configurable 62 * 63 * Options are: 64 * 65 * SCHED_TUNABLESCALING_NONE - unscaled, always *1 66 * SCHED_TUNABLESCALING_LOG - scaled logarithmical, *1+ilog(ncpus) 67 * SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus 68 * 69 * (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus)) 70 */ 71 unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG; 72 73 /* 74 * Minimal preemption granularity for CPU-bound tasks: 75 * 76 * (default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds) 77 */ 78 unsigned int sysctl_sched_base_slice = 750000ULL; 79 static unsigned int normalized_sysctl_sched_base_slice = 750000ULL; 80 81 /* 82 * After fork, child runs first. If set to 0 (default) then 83 * parent will (try to) run first. 84 */ 85 unsigned int sysctl_sched_child_runs_first __read_mostly; 86 87 const_debug unsigned int sysctl_sched_migration_cost = 500000UL; 88 89 int sched_thermal_decay_shift; 90 static int __init setup_sched_thermal_decay_shift(char *str) 91 { 92 int _shift = 0; 93 94 if (kstrtoint(str, 0, &_shift)) 95 pr_warn("Unable to set scheduler thermal pressure decay shift parameter\n"); 96 97 sched_thermal_decay_shift = clamp(_shift, 0, 10); 98 return 1; 99 } 100 __setup("sched_thermal_decay_shift=", setup_sched_thermal_decay_shift); 101 102 #ifdef CONFIG_SMP 103 /* 104 * For asym packing, by default the lower numbered CPU has higher priority. 105 */ 106 int __weak arch_asym_cpu_priority(int cpu) 107 { 108 return -cpu; 109 } 110 111 /* 112 * The margin used when comparing utilization with CPU capacity. 113 * 114 * (default: ~20%) 115 */ 116 #define fits_capacity(cap, max) ((cap) * 1280 < (max) * 1024) 117 118 /* 119 * The margin used when comparing CPU capacities. 120 * is 'cap1' noticeably greater than 'cap2' 121 * 122 * (default: ~5%) 123 */ 124 #define capacity_greater(cap1, cap2) ((cap1) * 1024 > (cap2) * 1078) 125 #endif 126 127 #ifdef CONFIG_CFS_BANDWIDTH 128 /* 129 * Amount of runtime to allocate from global (tg) to local (per-cfs_rq) pool 130 * each time a cfs_rq requests quota. 131 * 132 * Note: in the case that the slice exceeds the runtime remaining (either due 133 * to consumption or the quota being specified to be smaller than the slice) 134 * we will always only issue the remaining available time. 135 * 136 * (default: 5 msec, units: microseconds) 137 */ 138 static unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; 139 #endif 140 141 #ifdef CONFIG_NUMA_BALANCING 142 /* Restrict the NUMA promotion throughput (MB/s) for each target node. */ 143 static unsigned int sysctl_numa_balancing_promote_rate_limit = 65536; 144 #endif 145 146 #ifdef CONFIG_SYSCTL 147 static struct ctl_table sched_fair_sysctls[] = { 148 { 149 .procname = "sched_child_runs_first", 150 .data = &sysctl_sched_child_runs_first, 151 .maxlen = sizeof(unsigned int), 152 .mode = 0644, 153 .proc_handler = proc_dointvec, 154 }, 155 #ifdef CONFIG_CFS_BANDWIDTH 156 { 157 .procname = "sched_cfs_bandwidth_slice_us", 158 .data = &sysctl_sched_cfs_bandwidth_slice, 159 .maxlen = sizeof(unsigned int), 160 .mode = 0644, 161 .proc_handler = proc_dointvec_minmax, 162 .extra1 = SYSCTL_ONE, 163 }, 164 #endif 165 #ifdef CONFIG_NUMA_BALANCING 166 { 167 .procname = "numa_balancing_promote_rate_limit_MBps", 168 .data = &sysctl_numa_balancing_promote_rate_limit, 169 .maxlen = sizeof(unsigned int), 170 .mode = 0644, 171 .proc_handler = proc_dointvec_minmax, 172 .extra1 = SYSCTL_ZERO, 173 }, 174 #endif /* CONFIG_NUMA_BALANCING */ 175 {} 176 }; 177 178 static int __init sched_fair_sysctl_init(void) 179 { 180 register_sysctl_init("kernel", sched_fair_sysctls); 181 return 0; 182 } 183 late_initcall(sched_fair_sysctl_init); 184 #endif 185 186 static inline void update_load_add(struct load_weight *lw, unsigned long inc) 187 { 188 lw->weight += inc; 189 lw->inv_weight = 0; 190 } 191 192 static inline void update_load_sub(struct load_weight *lw, unsigned long dec) 193 { 194 lw->weight -= dec; 195 lw->inv_weight = 0; 196 } 197 198 static inline void update_load_set(struct load_weight *lw, unsigned long w) 199 { 200 lw->weight = w; 201 lw->inv_weight = 0; 202 } 203 204 /* 205 * Increase the granularity value when there are more CPUs, 206 * because with more CPUs the 'effective latency' as visible 207 * to users decreases. But the relationship is not linear, 208 * so pick a second-best guess by going with the log2 of the 209 * number of CPUs. 210 * 211 * This idea comes from the SD scheduler of Con Kolivas: 212 */ 213 static unsigned int get_update_sysctl_factor(void) 214 { 215 unsigned int cpus = min_t(unsigned int, num_online_cpus(), 8); 216 unsigned int factor; 217 218 switch (sysctl_sched_tunable_scaling) { 219 case SCHED_TUNABLESCALING_NONE: 220 factor = 1; 221 break; 222 case SCHED_TUNABLESCALING_LINEAR: 223 factor = cpus; 224 break; 225 case SCHED_TUNABLESCALING_LOG: 226 default: 227 factor = 1 + ilog2(cpus); 228 break; 229 } 230 231 return factor; 232 } 233 234 static void update_sysctl(void) 235 { 236 unsigned int factor = get_update_sysctl_factor(); 237 238 #define SET_SYSCTL(name) \ 239 (sysctl_##name = (factor) * normalized_sysctl_##name) 240 SET_SYSCTL(sched_base_slice); 241 #undef SET_SYSCTL 242 } 243 244 void __init sched_init_granularity(void) 245 { 246 update_sysctl(); 247 } 248 249 #define WMULT_CONST (~0U) 250 #define WMULT_SHIFT 32 251 252 static void __update_inv_weight(struct load_weight *lw) 253 { 254 unsigned long w; 255 256 if (likely(lw->inv_weight)) 257 return; 258 259 w = scale_load_down(lw->weight); 260 261 if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST)) 262 lw->inv_weight = 1; 263 else if (unlikely(!w)) 264 lw->inv_weight = WMULT_CONST; 265 else 266 lw->inv_weight = WMULT_CONST / w; 267 } 268 269 /* 270 * delta_exec * weight / lw.weight 271 * OR 272 * (delta_exec * (weight * lw->inv_weight)) >> WMULT_SHIFT 273 * 274 * Either weight := NICE_0_LOAD and lw \e sched_prio_to_wmult[], in which case 275 * we're guaranteed shift stays positive because inv_weight is guaranteed to 276 * fit 32 bits, and NICE_0_LOAD gives another 10 bits; therefore shift >= 22. 277 * 278 * Or, weight =< lw.weight (because lw.weight is the runqueue weight), thus 279 * weight/lw.weight <= 1, and therefore our shift will also be positive. 280 */ 281 static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw) 282 { 283 u64 fact = scale_load_down(weight); 284 u32 fact_hi = (u32)(fact >> 32); 285 int shift = WMULT_SHIFT; 286 int fs; 287 288 __update_inv_weight(lw); 289 290 if (unlikely(fact_hi)) { 291 fs = fls(fact_hi); 292 shift -= fs; 293 fact >>= fs; 294 } 295 296 fact = mul_u32_u32(fact, lw->inv_weight); 297 298 fact_hi = (u32)(fact >> 32); 299 if (fact_hi) { 300 fs = fls(fact_hi); 301 shift -= fs; 302 fact >>= fs; 303 } 304 305 return mul_u64_u32_shr(delta_exec, fact, shift); 306 } 307 308 /* 309 * delta /= w 310 */ 311 static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se) 312 { 313 if (unlikely(se->load.weight != NICE_0_LOAD)) 314 delta = __calc_delta(delta, NICE_0_LOAD, &se->load); 315 316 return delta; 317 } 318 319 const struct sched_class fair_sched_class; 320 321 /************************************************************** 322 * CFS operations on generic schedulable entities: 323 */ 324 325 #ifdef CONFIG_FAIR_GROUP_SCHED 326 327 /* Walk up scheduling entities hierarchy */ 328 #define for_each_sched_entity(se) \ 329 for (; se; se = se->parent) 330 331 static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) 332 { 333 struct rq *rq = rq_of(cfs_rq); 334 int cpu = cpu_of(rq); 335 336 if (cfs_rq->on_list) 337 return rq->tmp_alone_branch == &rq->leaf_cfs_rq_list; 338 339 cfs_rq->on_list = 1; 340 341 /* 342 * Ensure we either appear before our parent (if already 343 * enqueued) or force our parent to appear after us when it is 344 * enqueued. The fact that we always enqueue bottom-up 345 * reduces this to two cases and a special case for the root 346 * cfs_rq. Furthermore, it also means that we will always reset 347 * tmp_alone_branch either when the branch is connected 348 * to a tree or when we reach the top of the tree 349 */ 350 if (cfs_rq->tg->parent && 351 cfs_rq->tg->parent->cfs_rq[cpu]->on_list) { 352 /* 353 * If parent is already on the list, we add the child 354 * just before. Thanks to circular linked property of 355 * the list, this means to put the child at the tail 356 * of the list that starts by parent. 357 */ 358 list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, 359 &(cfs_rq->tg->parent->cfs_rq[cpu]->leaf_cfs_rq_list)); 360 /* 361 * The branch is now connected to its tree so we can 362 * reset tmp_alone_branch to the beginning of the 363 * list. 364 */ 365 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 366 return true; 367 } 368 369 if (!cfs_rq->tg->parent) { 370 /* 371 * cfs rq without parent should be put 372 * at the tail of the list. 373 */ 374 list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, 375 &rq->leaf_cfs_rq_list); 376 /* 377 * We have reach the top of a tree so we can reset 378 * tmp_alone_branch to the beginning of the list. 379 */ 380 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 381 return true; 382 } 383 384 /* 385 * The parent has not already been added so we want to 386 * make sure that it will be put after us. 387 * tmp_alone_branch points to the begin of the branch 388 * where we will add parent. 389 */ 390 list_add_rcu(&cfs_rq->leaf_cfs_rq_list, rq->tmp_alone_branch); 391 /* 392 * update tmp_alone_branch to points to the new begin 393 * of the branch 394 */ 395 rq->tmp_alone_branch = &cfs_rq->leaf_cfs_rq_list; 396 return false; 397 } 398 399 static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) 400 { 401 if (cfs_rq->on_list) { 402 struct rq *rq = rq_of(cfs_rq); 403 404 /* 405 * With cfs_rq being unthrottled/throttled during an enqueue, 406 * it can happen the tmp_alone_branch points the a leaf that 407 * we finally want to del. In this case, tmp_alone_branch moves 408 * to the prev element but it will point to rq->leaf_cfs_rq_list 409 * at the end of the enqueue. 410 */ 411 if (rq->tmp_alone_branch == &cfs_rq->leaf_cfs_rq_list) 412 rq->tmp_alone_branch = cfs_rq->leaf_cfs_rq_list.prev; 413 414 list_del_rcu(&cfs_rq->leaf_cfs_rq_list); 415 cfs_rq->on_list = 0; 416 } 417 } 418 419 static inline void assert_list_leaf_cfs_rq(struct rq *rq) 420 { 421 SCHED_WARN_ON(rq->tmp_alone_branch != &rq->leaf_cfs_rq_list); 422 } 423 424 /* Iterate thr' all leaf cfs_rq's on a runqueue */ 425 #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \ 426 list_for_each_entry_safe(cfs_rq, pos, &rq->leaf_cfs_rq_list, \ 427 leaf_cfs_rq_list) 428 429 /* Do the two (enqueued) entities belong to the same group ? */ 430 static inline struct cfs_rq * 431 is_same_group(struct sched_entity *se, struct sched_entity *pse) 432 { 433 if (se->cfs_rq == pse->cfs_rq) 434 return se->cfs_rq; 435 436 return NULL; 437 } 438 439 static inline struct sched_entity *parent_entity(const struct sched_entity *se) 440 { 441 return se->parent; 442 } 443 444 static void 445 find_matching_se(struct sched_entity **se, struct sched_entity **pse) 446 { 447 int se_depth, pse_depth; 448 449 /* 450 * preemption test can be made between sibling entities who are in the 451 * same cfs_rq i.e who have a common parent. Walk up the hierarchy of 452 * both tasks until we find their ancestors who are siblings of common 453 * parent. 454 */ 455 456 /* First walk up until both entities are at same depth */ 457 se_depth = (*se)->depth; 458 pse_depth = (*pse)->depth; 459 460 while (se_depth > pse_depth) { 461 se_depth--; 462 *se = parent_entity(*se); 463 } 464 465 while (pse_depth > se_depth) { 466 pse_depth--; 467 *pse = parent_entity(*pse); 468 } 469 470 while (!is_same_group(*se, *pse)) { 471 *se = parent_entity(*se); 472 *pse = parent_entity(*pse); 473 } 474 } 475 476 static int tg_is_idle(struct task_group *tg) 477 { 478 return tg->idle > 0; 479 } 480 481 static int cfs_rq_is_idle(struct cfs_rq *cfs_rq) 482 { 483 return cfs_rq->idle > 0; 484 } 485 486 static int se_is_idle(struct sched_entity *se) 487 { 488 if (entity_is_task(se)) 489 return task_has_idle_policy(task_of(se)); 490 return cfs_rq_is_idle(group_cfs_rq(se)); 491 } 492 493 #else /* !CONFIG_FAIR_GROUP_SCHED */ 494 495 #define for_each_sched_entity(se) \ 496 for (; se; se = NULL) 497 498 static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) 499 { 500 return true; 501 } 502 503 static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) 504 { 505 } 506 507 static inline void assert_list_leaf_cfs_rq(struct rq *rq) 508 { 509 } 510 511 #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \ 512 for (cfs_rq = &rq->cfs, pos = NULL; cfs_rq; cfs_rq = pos) 513 514 static inline struct sched_entity *parent_entity(struct sched_entity *se) 515 { 516 return NULL; 517 } 518 519 static inline void 520 find_matching_se(struct sched_entity **se, struct sched_entity **pse) 521 { 522 } 523 524 static inline int tg_is_idle(struct task_group *tg) 525 { 526 return 0; 527 } 528 529 static int cfs_rq_is_idle(struct cfs_rq *cfs_rq) 530 { 531 return 0; 532 } 533 534 static int se_is_idle(struct sched_entity *se) 535 { 536 return task_has_idle_policy(task_of(se)); 537 } 538 539 #endif /* CONFIG_FAIR_GROUP_SCHED */ 540 541 static __always_inline 542 void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec); 543 544 /************************************************************** 545 * Scheduling class tree data structure manipulation methods: 546 */ 547 548 static inline u64 max_vruntime(u64 max_vruntime, u64 vruntime) 549 { 550 s64 delta = (s64)(vruntime - max_vruntime); 551 if (delta > 0) 552 max_vruntime = vruntime; 553 554 return max_vruntime; 555 } 556 557 static inline u64 min_vruntime(u64 min_vruntime, u64 vruntime) 558 { 559 s64 delta = (s64)(vruntime - min_vruntime); 560 if (delta < 0) 561 min_vruntime = vruntime; 562 563 return min_vruntime; 564 } 565 566 static inline bool entity_before(const struct sched_entity *a, 567 const struct sched_entity *b) 568 { 569 return (s64)(a->vruntime - b->vruntime) < 0; 570 } 571 572 static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se) 573 { 574 return (s64)(se->vruntime - cfs_rq->min_vruntime); 575 } 576 577 #define __node_2_se(node) \ 578 rb_entry((node), struct sched_entity, run_node) 579 580 /* 581 * Compute virtual time from the per-task service numbers: 582 * 583 * Fair schedulers conserve lag: 584 * 585 * \Sum lag_i = 0 586 * 587 * Where lag_i is given by: 588 * 589 * lag_i = S - s_i = w_i * (V - v_i) 590 * 591 * Where S is the ideal service time and V is it's virtual time counterpart. 592 * Therefore: 593 * 594 * \Sum lag_i = 0 595 * \Sum w_i * (V - v_i) = 0 596 * \Sum w_i * V - w_i * v_i = 0 597 * 598 * From which we can solve an expression for V in v_i (which we have in 599 * se->vruntime): 600 * 601 * \Sum v_i * w_i \Sum v_i * w_i 602 * V = -------------- = -------------- 603 * \Sum w_i W 604 * 605 * Specifically, this is the weighted average of all entity virtual runtimes. 606 * 607 * [[ NOTE: this is only equal to the ideal scheduler under the condition 608 * that join/leave operations happen at lag_i = 0, otherwise the 609 * virtual time has non-continguous motion equivalent to: 610 * 611 * V +-= lag_i / W 612 * 613 * Also see the comment in place_entity() that deals with this. ]] 614 * 615 * However, since v_i is u64, and the multiplcation could easily overflow 616 * transform it into a relative form that uses smaller quantities: 617 * 618 * Substitute: v_i == (v_i - v0) + v0 619 * 620 * \Sum ((v_i - v0) + v0) * w_i \Sum (v_i - v0) * w_i 621 * V = ---------------------------- = --------------------- + v0 622 * W W 623 * 624 * Which we track using: 625 * 626 * v0 := cfs_rq->min_vruntime 627 * \Sum (v_i - v0) * w_i := cfs_rq->avg_vruntime 628 * \Sum w_i := cfs_rq->avg_load 629 * 630 * Since min_vruntime is a monotonic increasing variable that closely tracks 631 * the per-task service, these deltas: (v_i - v), will be in the order of the 632 * maximal (virtual) lag induced in the system due to quantisation. 633 * 634 * Also, we use scale_load_down() to reduce the size. 635 * 636 * As measured, the max (key * weight) value was ~44 bits for a kernel build. 637 */ 638 static void 639 avg_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se) 640 { 641 unsigned long weight = scale_load_down(se->load.weight); 642 s64 key = entity_key(cfs_rq, se); 643 644 cfs_rq->avg_vruntime += key * weight; 645 cfs_rq->avg_load += weight; 646 } 647 648 static void 649 avg_vruntime_sub(struct cfs_rq *cfs_rq, struct sched_entity *se) 650 { 651 unsigned long weight = scale_load_down(se->load.weight); 652 s64 key = entity_key(cfs_rq, se); 653 654 cfs_rq->avg_vruntime -= key * weight; 655 cfs_rq->avg_load -= weight; 656 } 657 658 static inline 659 void avg_vruntime_update(struct cfs_rq *cfs_rq, s64 delta) 660 { 661 /* 662 * v' = v + d ==> avg_vruntime' = avg_runtime - d*avg_load 663 */ 664 cfs_rq->avg_vruntime -= cfs_rq->avg_load * delta; 665 } 666 667 /* 668 * Specifically: avg_runtime() + 0 must result in entity_eligible() := true 669 * For this to be so, the result of this function must have a left bias. 670 */ 671 u64 avg_vruntime(struct cfs_rq *cfs_rq) 672 { 673 struct sched_entity *curr = cfs_rq->curr; 674 s64 avg = cfs_rq->avg_vruntime; 675 long load = cfs_rq->avg_load; 676 677 if (curr && curr->on_rq) { 678 unsigned long weight = scale_load_down(curr->load.weight); 679 680 avg += entity_key(cfs_rq, curr) * weight; 681 load += weight; 682 } 683 684 if (load) { 685 /* sign flips effective floor / ceil */ 686 if (avg < 0) 687 avg -= (load - 1); 688 avg = div_s64(avg, load); 689 } 690 691 return cfs_rq->min_vruntime + avg; 692 } 693 694 /* 695 * lag_i = S - s_i = w_i * (V - v_i) 696 * 697 * However, since V is approximated by the weighted average of all entities it 698 * is possible -- by addition/removal/reweight to the tree -- to move V around 699 * and end up with a larger lag than we started with. 700 * 701 * Limit this to either double the slice length with a minimum of TICK_NSEC 702 * since that is the timing granularity. 703 * 704 * EEVDF gives the following limit for a steady state system: 705 * 706 * -r_max < lag < max(r_max, q) 707 * 708 * XXX could add max_slice to the augmented data to track this. 709 */ 710 static s64 entity_lag(u64 avruntime, struct sched_entity *se) 711 { 712 s64 vlag, limit; 713 714 vlag = avruntime - se->vruntime; 715 limit = calc_delta_fair(max_t(u64, 2*se->slice, TICK_NSEC), se); 716 717 return clamp(vlag, -limit, limit); 718 } 719 720 static void update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se) 721 { 722 SCHED_WARN_ON(!se->on_rq); 723 724 se->vlag = entity_lag(avg_vruntime(cfs_rq), se); 725 } 726 727 /* 728 * Entity is eligible once it received less service than it ought to have, 729 * eg. lag >= 0. 730 * 731 * lag_i = S - s_i = w_i*(V - v_i) 732 * 733 * lag_i >= 0 -> V >= v_i 734 * 735 * \Sum (v_i - v)*w_i 736 * V = ------------------ + v 737 * \Sum w_i 738 * 739 * lag_i >= 0 -> \Sum (v_i - v)*w_i >= (v_i - v)*(\Sum w_i) 740 * 741 * Note: using 'avg_vruntime() > se->vruntime' is inacurate due 742 * to the loss in precision caused by the division. 743 */ 744 int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se) 745 { 746 struct sched_entity *curr = cfs_rq->curr; 747 s64 avg = cfs_rq->avg_vruntime; 748 long load = cfs_rq->avg_load; 749 750 if (curr && curr->on_rq) { 751 unsigned long weight = scale_load_down(curr->load.weight); 752 753 avg += entity_key(cfs_rq, curr) * weight; 754 load += weight; 755 } 756 757 return avg >= entity_key(cfs_rq, se) * load; 758 } 759 760 static u64 __update_min_vruntime(struct cfs_rq *cfs_rq, u64 vruntime) 761 { 762 u64 min_vruntime = cfs_rq->min_vruntime; 763 /* 764 * open coded max_vruntime() to allow updating avg_vruntime 765 */ 766 s64 delta = (s64)(vruntime - min_vruntime); 767 if (delta > 0) { 768 avg_vruntime_update(cfs_rq, delta); 769 min_vruntime = vruntime; 770 } 771 return min_vruntime; 772 } 773 774 static void update_min_vruntime(struct cfs_rq *cfs_rq) 775 { 776 struct sched_entity *se = __pick_first_entity(cfs_rq); 777 struct sched_entity *curr = cfs_rq->curr; 778 779 u64 vruntime = cfs_rq->min_vruntime; 780 781 if (curr) { 782 if (curr->on_rq) 783 vruntime = curr->vruntime; 784 else 785 curr = NULL; 786 } 787 788 if (se) { 789 if (!curr) 790 vruntime = se->vruntime; 791 else 792 vruntime = min_vruntime(vruntime, se->vruntime); 793 } 794 795 /* ensure we never gain time by being placed backwards. */ 796 u64_u32_store(cfs_rq->min_vruntime, 797 __update_min_vruntime(cfs_rq, vruntime)); 798 } 799 800 static inline bool __entity_less(struct rb_node *a, const struct rb_node *b) 801 { 802 return entity_before(__node_2_se(a), __node_2_se(b)); 803 } 804 805 #define deadline_gt(field, lse, rse) ({ (s64)((lse)->field - (rse)->field) > 0; }) 806 807 static inline void __update_min_deadline(struct sched_entity *se, struct rb_node *node) 808 { 809 if (node) { 810 struct sched_entity *rse = __node_2_se(node); 811 if (deadline_gt(min_deadline, se, rse)) 812 se->min_deadline = rse->min_deadline; 813 } 814 } 815 816 /* 817 * se->min_deadline = min(se->deadline, left->min_deadline, right->min_deadline) 818 */ 819 static inline bool min_deadline_update(struct sched_entity *se, bool exit) 820 { 821 u64 old_min_deadline = se->min_deadline; 822 struct rb_node *node = &se->run_node; 823 824 se->min_deadline = se->deadline; 825 __update_min_deadline(se, node->rb_right); 826 __update_min_deadline(se, node->rb_left); 827 828 return se->min_deadline == old_min_deadline; 829 } 830 831 RB_DECLARE_CALLBACKS(static, min_deadline_cb, struct sched_entity, 832 run_node, min_deadline, min_deadline_update); 833 834 /* 835 * Enqueue an entity into the rb-tree: 836 */ 837 static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 838 { 839 avg_vruntime_add(cfs_rq, se); 840 se->min_deadline = se->deadline; 841 rb_add_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, 842 __entity_less, &min_deadline_cb); 843 } 844 845 static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 846 { 847 rb_erase_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, 848 &min_deadline_cb); 849 avg_vruntime_sub(cfs_rq, se); 850 } 851 852 struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq) 853 { 854 struct rb_node *left = rb_first_cached(&cfs_rq->tasks_timeline); 855 856 if (!left) 857 return NULL; 858 859 return __node_2_se(left); 860 } 861 862 /* 863 * Earliest Eligible Virtual Deadline First 864 * 865 * In order to provide latency guarantees for different request sizes 866 * EEVDF selects the best runnable task from two criteria: 867 * 868 * 1) the task must be eligible (must be owed service) 869 * 870 * 2) from those tasks that meet 1), we select the one 871 * with the earliest virtual deadline. 872 * 873 * We can do this in O(log n) time due to an augmented RB-tree. The 874 * tree keeps the entries sorted on service, but also functions as a 875 * heap based on the deadline by keeping: 876 * 877 * se->min_deadline = min(se->deadline, se->{left,right}->min_deadline) 878 * 879 * Which allows an EDF like search on (sub)trees. 880 */ 881 static struct sched_entity *__pick_eevdf(struct cfs_rq *cfs_rq) 882 { 883 struct rb_node *node = cfs_rq->tasks_timeline.rb_root.rb_node; 884 struct sched_entity *curr = cfs_rq->curr; 885 struct sched_entity *best = NULL; 886 struct sched_entity *best_left = NULL; 887 888 if (curr && (!curr->on_rq || !entity_eligible(cfs_rq, curr))) 889 curr = NULL; 890 best = curr; 891 892 /* 893 * Once selected, run a task until it either becomes non-eligible or 894 * until it gets a new slice. See the HACK in set_next_entity(). 895 */ 896 if (sched_feat(RUN_TO_PARITY) && curr && curr->vlag == curr->deadline) 897 return curr; 898 899 while (node) { 900 struct sched_entity *se = __node_2_se(node); 901 902 /* 903 * If this entity is not eligible, try the left subtree. 904 */ 905 if (!entity_eligible(cfs_rq, se)) { 906 node = node->rb_left; 907 continue; 908 } 909 910 /* 911 * Now we heap search eligible trees for the best (min_)deadline 912 */ 913 if (!best || deadline_gt(deadline, best, se)) 914 best = se; 915 916 /* 917 * Every se in a left branch is eligible, keep track of the 918 * branch with the best min_deadline 919 */ 920 if (node->rb_left) { 921 struct sched_entity *left = __node_2_se(node->rb_left); 922 923 if (!best_left || deadline_gt(min_deadline, best_left, left)) 924 best_left = left; 925 926 /* 927 * min_deadline is in the left branch. rb_left and all 928 * descendants are eligible, so immediately switch to the second 929 * loop. 930 */ 931 if (left->min_deadline == se->min_deadline) 932 break; 933 } 934 935 /* min_deadline is at this node, no need to look right */ 936 if (se->deadline == se->min_deadline) 937 break; 938 939 /* else min_deadline is in the right branch. */ 940 node = node->rb_right; 941 } 942 943 /* 944 * We ran into an eligible node which is itself the best. 945 * (Or nr_running == 0 and both are NULL) 946 */ 947 if (!best_left || (s64)(best_left->min_deadline - best->deadline) > 0) 948 return best; 949 950 /* 951 * Now best_left and all of its children are eligible, and we are just 952 * looking for deadline == min_deadline 953 */ 954 node = &best_left->run_node; 955 while (node) { 956 struct sched_entity *se = __node_2_se(node); 957 958 /* min_deadline is the current node */ 959 if (se->deadline == se->min_deadline) 960 return se; 961 962 /* min_deadline is in the left branch */ 963 if (node->rb_left && 964 __node_2_se(node->rb_left)->min_deadline == se->min_deadline) { 965 node = node->rb_left; 966 continue; 967 } 968 969 /* else min_deadline is in the right branch */ 970 node = node->rb_right; 971 } 972 return NULL; 973 } 974 975 static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq) 976 { 977 struct sched_entity *se = __pick_eevdf(cfs_rq); 978 979 if (!se) { 980 struct sched_entity *left = __pick_first_entity(cfs_rq); 981 if (left) { 982 pr_err("EEVDF scheduling fail, picking leftmost\n"); 983 return left; 984 } 985 } 986 987 return se; 988 } 989 990 #ifdef CONFIG_SCHED_DEBUG 991 struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq) 992 { 993 struct rb_node *last = rb_last(&cfs_rq->tasks_timeline.rb_root); 994 995 if (!last) 996 return NULL; 997 998 return __node_2_se(last); 999 } 1000 1001 /************************************************************** 1002 * Scheduling class statistics methods: 1003 */ 1004 #ifdef CONFIG_SMP 1005 int sched_update_scaling(void) 1006 { 1007 unsigned int factor = get_update_sysctl_factor(); 1008 1009 #define WRT_SYSCTL(name) \ 1010 (normalized_sysctl_##name = sysctl_##name / (factor)) 1011 WRT_SYSCTL(sched_base_slice); 1012 #undef WRT_SYSCTL 1013 1014 return 0; 1015 } 1016 #endif 1017 #endif 1018 1019 static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se); 1020 1021 /* 1022 * XXX: strictly: vd_i += N*r_i/w_i such that: vd_i > ve_i 1023 * this is probably good enough. 1024 */ 1025 static void update_deadline(struct cfs_rq *cfs_rq, struct sched_entity *se) 1026 { 1027 if ((s64)(se->vruntime - se->deadline) < 0) 1028 return; 1029 1030 /* 1031 * For EEVDF the virtual time slope is determined by w_i (iow. 1032 * nice) while the request time r_i is determined by 1033 * sysctl_sched_base_slice. 1034 */ 1035 se->slice = sysctl_sched_base_slice; 1036 1037 /* 1038 * EEVDF: vd_i = ve_i + r_i / w_i 1039 */ 1040 se->deadline = se->vruntime + calc_delta_fair(se->slice, se); 1041 1042 /* 1043 * The task has consumed its request, reschedule. 1044 */ 1045 if (cfs_rq->nr_running > 1) { 1046 resched_curr(rq_of(cfs_rq)); 1047 clear_buddies(cfs_rq, se); 1048 } 1049 } 1050 1051 #include "pelt.h" 1052 #ifdef CONFIG_SMP 1053 1054 static int select_idle_sibling(struct task_struct *p, int prev_cpu, int cpu); 1055 static unsigned long task_h_load(struct task_struct *p); 1056 static unsigned long capacity_of(int cpu); 1057 1058 /* Give new sched_entity start runnable values to heavy its load in infant time */ 1059 void init_entity_runnable_average(struct sched_entity *se) 1060 { 1061 struct sched_avg *sa = &se->avg; 1062 1063 memset(sa, 0, sizeof(*sa)); 1064 1065 /* 1066 * Tasks are initialized with full load to be seen as heavy tasks until 1067 * they get a chance to stabilize to their real load level. 1068 * Group entities are initialized with zero load to reflect the fact that 1069 * nothing has been attached to the task group yet. 1070 */ 1071 if (entity_is_task(se)) 1072 sa->load_avg = scale_load_down(se->load.weight); 1073 1074 /* when this task enqueue'ed, it will contribute to its cfs_rq's load_avg */ 1075 } 1076 1077 /* 1078 * With new tasks being created, their initial util_avgs are extrapolated 1079 * based on the cfs_rq's current util_avg: 1080 * 1081 * util_avg = cfs_rq->util_avg / (cfs_rq->load_avg + 1) * se.load.weight 1082 * 1083 * However, in many cases, the above util_avg does not give a desired 1084 * value. Moreover, the sum of the util_avgs may be divergent, such 1085 * as when the series is a harmonic series. 1086 * 1087 * To solve this problem, we also cap the util_avg of successive tasks to 1088 * only 1/2 of the left utilization budget: 1089 * 1090 * util_avg_cap = (cpu_scale - cfs_rq->avg.util_avg) / 2^n 1091 * 1092 * where n denotes the nth task and cpu_scale the CPU capacity. 1093 * 1094 * For example, for a CPU with 1024 of capacity, a simplest series from 1095 * the beginning would be like: 1096 * 1097 * task util_avg: 512, 256, 128, 64, 32, 16, 8, ... 1098 * cfs_rq util_avg: 512, 768, 896, 960, 992, 1008, 1016, ... 1099 * 1100 * Finally, that extrapolated util_avg is clamped to the cap (util_avg_cap) 1101 * if util_avg > util_avg_cap. 1102 */ 1103 void post_init_entity_util_avg(struct task_struct *p) 1104 { 1105 struct sched_entity *se = &p->se; 1106 struct cfs_rq *cfs_rq = cfs_rq_of(se); 1107 struct sched_avg *sa = &se->avg; 1108 long cpu_scale = arch_scale_cpu_capacity(cpu_of(rq_of(cfs_rq))); 1109 long cap = (long)(cpu_scale - cfs_rq->avg.util_avg) / 2; 1110 1111 if (p->sched_class != &fair_sched_class) { 1112 /* 1113 * For !fair tasks do: 1114 * 1115 update_cfs_rq_load_avg(now, cfs_rq); 1116 attach_entity_load_avg(cfs_rq, se); 1117 switched_from_fair(rq, p); 1118 * 1119 * such that the next switched_to_fair() has the 1120 * expected state. 1121 */ 1122 se->avg.last_update_time = cfs_rq_clock_pelt(cfs_rq); 1123 return; 1124 } 1125 1126 if (cap > 0) { 1127 if (cfs_rq->avg.util_avg != 0) { 1128 sa->util_avg = cfs_rq->avg.util_avg * se->load.weight; 1129 sa->util_avg /= (cfs_rq->avg.load_avg + 1); 1130 1131 if (sa->util_avg > cap) 1132 sa->util_avg = cap; 1133 } else { 1134 sa->util_avg = cap; 1135 } 1136 } 1137 1138 sa->runnable_avg = sa->util_avg; 1139 } 1140 1141 #else /* !CONFIG_SMP */ 1142 void init_entity_runnable_average(struct sched_entity *se) 1143 { 1144 } 1145 void post_init_entity_util_avg(struct task_struct *p) 1146 { 1147 } 1148 static void update_tg_load_avg(struct cfs_rq *cfs_rq) 1149 { 1150 } 1151 #endif /* CONFIG_SMP */ 1152 1153 static s64 update_curr_se(struct rq *rq, struct sched_entity *curr) 1154 { 1155 u64 now = rq_clock_task(rq); 1156 s64 delta_exec; 1157 1158 delta_exec = now - curr->exec_start; 1159 if (unlikely(delta_exec <= 0)) 1160 return delta_exec; 1161 1162 curr->exec_start = now; 1163 curr->sum_exec_runtime += delta_exec; 1164 1165 if (schedstat_enabled()) { 1166 struct sched_statistics *stats; 1167 1168 stats = __schedstats_from_se(curr); 1169 __schedstat_set(stats->exec_max, 1170 max(delta_exec, stats->exec_max)); 1171 } 1172 1173 return delta_exec; 1174 } 1175 1176 static inline void update_curr_task(struct task_struct *p, s64 delta_exec) 1177 { 1178 trace_sched_stat_runtime(p, delta_exec); 1179 account_group_exec_runtime(p, delta_exec); 1180 cgroup_account_cputime(p, delta_exec); 1181 } 1182 1183 /* 1184 * Used by other classes to account runtime. 1185 */ 1186 s64 update_curr_common(struct rq *rq) 1187 { 1188 struct task_struct *curr = rq->curr; 1189 s64 delta_exec; 1190 1191 delta_exec = update_curr_se(rq, &curr->se); 1192 if (likely(delta_exec > 0)) 1193 update_curr_task(curr, delta_exec); 1194 1195 return delta_exec; 1196 } 1197 1198 /* 1199 * Update the current task's runtime statistics. 1200 */ 1201 static void update_curr(struct cfs_rq *cfs_rq) 1202 { 1203 struct sched_entity *curr = cfs_rq->curr; 1204 s64 delta_exec; 1205 1206 if (unlikely(!curr)) 1207 return; 1208 1209 delta_exec = update_curr_se(rq_of(cfs_rq), curr); 1210 if (unlikely(delta_exec <= 0)) 1211 return; 1212 1213 curr->vruntime += calc_delta_fair(delta_exec, curr); 1214 update_deadline(cfs_rq, curr); 1215 update_min_vruntime(cfs_rq); 1216 1217 if (entity_is_task(curr)) 1218 update_curr_task(task_of(curr), delta_exec); 1219 1220 account_cfs_rq_runtime(cfs_rq, delta_exec); 1221 } 1222 1223 static void update_curr_fair(struct rq *rq) 1224 { 1225 update_curr(cfs_rq_of(&rq->curr->se)); 1226 } 1227 1228 static inline void 1229 update_stats_wait_start_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) 1230 { 1231 struct sched_statistics *stats; 1232 struct task_struct *p = NULL; 1233 1234 if (!schedstat_enabled()) 1235 return; 1236 1237 stats = __schedstats_from_se(se); 1238 1239 if (entity_is_task(se)) 1240 p = task_of(se); 1241 1242 __update_stats_wait_start(rq_of(cfs_rq), p, stats); 1243 } 1244 1245 static inline void 1246 update_stats_wait_end_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) 1247 { 1248 struct sched_statistics *stats; 1249 struct task_struct *p = NULL; 1250 1251 if (!schedstat_enabled()) 1252 return; 1253 1254 stats = __schedstats_from_se(se); 1255 1256 /* 1257 * When the sched_schedstat changes from 0 to 1, some sched se 1258 * maybe already in the runqueue, the se->statistics.wait_start 1259 * will be 0.So it will let the delta wrong. We need to avoid this 1260 * scenario. 1261 */ 1262 if (unlikely(!schedstat_val(stats->wait_start))) 1263 return; 1264 1265 if (entity_is_task(se)) 1266 p = task_of(se); 1267 1268 __update_stats_wait_end(rq_of(cfs_rq), p, stats); 1269 } 1270 1271 static inline void 1272 update_stats_enqueue_sleeper_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) 1273 { 1274 struct sched_statistics *stats; 1275 struct task_struct *tsk = NULL; 1276 1277 if (!schedstat_enabled()) 1278 return; 1279 1280 stats = __schedstats_from_se(se); 1281 1282 if (entity_is_task(se)) 1283 tsk = task_of(se); 1284 1285 __update_stats_enqueue_sleeper(rq_of(cfs_rq), tsk, stats); 1286 } 1287 1288 /* 1289 * Task is being enqueued - update stats: 1290 */ 1291 static inline void 1292 update_stats_enqueue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 1293 { 1294 if (!schedstat_enabled()) 1295 return; 1296 1297 /* 1298 * Are we enqueueing a waiting task? (for current tasks 1299 * a dequeue/enqueue event is a NOP) 1300 */ 1301 if (se != cfs_rq->curr) 1302 update_stats_wait_start_fair(cfs_rq, se); 1303 1304 if (flags & ENQUEUE_WAKEUP) 1305 update_stats_enqueue_sleeper_fair(cfs_rq, se); 1306 } 1307 1308 static inline void 1309 update_stats_dequeue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 1310 { 1311 1312 if (!schedstat_enabled()) 1313 return; 1314 1315 /* 1316 * Mark the end of the wait period if dequeueing a 1317 * waiting task: 1318 */ 1319 if (se != cfs_rq->curr) 1320 update_stats_wait_end_fair(cfs_rq, se); 1321 1322 if ((flags & DEQUEUE_SLEEP) && entity_is_task(se)) { 1323 struct task_struct *tsk = task_of(se); 1324 unsigned int state; 1325 1326 /* XXX racy against TTWU */ 1327 state = READ_ONCE(tsk->__state); 1328 if (state & TASK_INTERRUPTIBLE) 1329 __schedstat_set(tsk->stats.sleep_start, 1330 rq_clock(rq_of(cfs_rq))); 1331 if (state & TASK_UNINTERRUPTIBLE) 1332 __schedstat_set(tsk->stats.block_start, 1333 rq_clock(rq_of(cfs_rq))); 1334 } 1335 } 1336 1337 /* 1338 * We are picking a new current task - update its stats: 1339 */ 1340 static inline void 1341 update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se) 1342 { 1343 /* 1344 * We are starting a new run period: 1345 */ 1346 se->exec_start = rq_clock_task(rq_of(cfs_rq)); 1347 } 1348 1349 /************************************************** 1350 * Scheduling class queueing methods: 1351 */ 1352 1353 static inline bool is_core_idle(int cpu) 1354 { 1355 #ifdef CONFIG_SCHED_SMT 1356 int sibling; 1357 1358 for_each_cpu(sibling, cpu_smt_mask(cpu)) { 1359 if (cpu == sibling) 1360 continue; 1361 1362 if (!idle_cpu(sibling)) 1363 return false; 1364 } 1365 #endif 1366 1367 return true; 1368 } 1369 1370 #ifdef CONFIG_NUMA 1371 #define NUMA_IMBALANCE_MIN 2 1372 1373 static inline long 1374 adjust_numa_imbalance(int imbalance, int dst_running, int imb_numa_nr) 1375 { 1376 /* 1377 * Allow a NUMA imbalance if busy CPUs is less than the maximum 1378 * threshold. Above this threshold, individual tasks may be contending 1379 * for both memory bandwidth and any shared HT resources. This is an 1380 * approximation as the number of running tasks may not be related to 1381 * the number of busy CPUs due to sched_setaffinity. 1382 */ 1383 if (dst_running > imb_numa_nr) 1384 return imbalance; 1385 1386 /* 1387 * Allow a small imbalance based on a simple pair of communicating 1388 * tasks that remain local when the destination is lightly loaded. 1389 */ 1390 if (imbalance <= NUMA_IMBALANCE_MIN) 1391 return 0; 1392 1393 return imbalance; 1394 } 1395 #endif /* CONFIG_NUMA */ 1396 1397 #ifdef CONFIG_NUMA_BALANCING 1398 /* 1399 * Approximate time to scan a full NUMA task in ms. The task scan period is 1400 * calculated based on the tasks virtual memory size and 1401 * numa_balancing_scan_size. 1402 */ 1403 unsigned int sysctl_numa_balancing_scan_period_min = 1000; 1404 unsigned int sysctl_numa_balancing_scan_period_max = 60000; 1405 1406 /* Portion of address space to scan in MB */ 1407 unsigned int sysctl_numa_balancing_scan_size = 256; 1408 1409 /* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */ 1410 unsigned int sysctl_numa_balancing_scan_delay = 1000; 1411 1412 /* The page with hint page fault latency < threshold in ms is considered hot */ 1413 unsigned int sysctl_numa_balancing_hot_threshold = MSEC_PER_SEC; 1414 1415 struct numa_group { 1416 refcount_t refcount; 1417 1418 spinlock_t lock; /* nr_tasks, tasks */ 1419 int nr_tasks; 1420 pid_t gid; 1421 int active_nodes; 1422 1423 struct rcu_head rcu; 1424 unsigned long total_faults; 1425 unsigned long max_faults_cpu; 1426 /* 1427 * faults[] array is split into two regions: faults_mem and faults_cpu. 1428 * 1429 * Faults_cpu is used to decide whether memory should move 1430 * towards the CPU. As a consequence, these stats are weighted 1431 * more by CPU use than by memory faults. 1432 */ 1433 unsigned long faults[]; 1434 }; 1435 1436 /* 1437 * For functions that can be called in multiple contexts that permit reading 1438 * ->numa_group (see struct task_struct for locking rules). 1439 */ 1440 static struct numa_group *deref_task_numa_group(struct task_struct *p) 1441 { 1442 return rcu_dereference_check(p->numa_group, p == current || 1443 (lockdep_is_held(__rq_lockp(task_rq(p))) && !READ_ONCE(p->on_cpu))); 1444 } 1445 1446 static struct numa_group *deref_curr_numa_group(struct task_struct *p) 1447 { 1448 return rcu_dereference_protected(p->numa_group, p == current); 1449 } 1450 1451 static inline unsigned long group_faults_priv(struct numa_group *ng); 1452 static inline unsigned long group_faults_shared(struct numa_group *ng); 1453 1454 static unsigned int task_nr_scan_windows(struct task_struct *p) 1455 { 1456 unsigned long rss = 0; 1457 unsigned long nr_scan_pages; 1458 1459 /* 1460 * Calculations based on RSS as non-present and empty pages are skipped 1461 * by the PTE scanner and NUMA hinting faults should be trapped based 1462 * on resident pages 1463 */ 1464 nr_scan_pages = sysctl_numa_balancing_scan_size << (20 - PAGE_SHIFT); 1465 rss = get_mm_rss(p->mm); 1466 if (!rss) 1467 rss = nr_scan_pages; 1468 1469 rss = round_up(rss, nr_scan_pages); 1470 return rss / nr_scan_pages; 1471 } 1472 1473 /* For sanity's sake, never scan more PTEs than MAX_SCAN_WINDOW MB/sec. */ 1474 #define MAX_SCAN_WINDOW 2560 1475 1476 static unsigned int task_scan_min(struct task_struct *p) 1477 { 1478 unsigned int scan_size = READ_ONCE(sysctl_numa_balancing_scan_size); 1479 unsigned int scan, floor; 1480 unsigned int windows = 1; 1481 1482 if (scan_size < MAX_SCAN_WINDOW) 1483 windows = MAX_SCAN_WINDOW / scan_size; 1484 floor = 1000 / windows; 1485 1486 scan = sysctl_numa_balancing_scan_period_min / task_nr_scan_windows(p); 1487 return max_t(unsigned int, floor, scan); 1488 } 1489 1490 static unsigned int task_scan_start(struct task_struct *p) 1491 { 1492 unsigned long smin = task_scan_min(p); 1493 unsigned long period = smin; 1494 struct numa_group *ng; 1495 1496 /* Scale the maximum scan period with the amount of shared memory. */ 1497 rcu_read_lock(); 1498 ng = rcu_dereference(p->numa_group); 1499 if (ng) { 1500 unsigned long shared = group_faults_shared(ng); 1501 unsigned long private = group_faults_priv(ng); 1502 1503 period *= refcount_read(&ng->refcount); 1504 period *= shared + 1; 1505 period /= private + shared + 1; 1506 } 1507 rcu_read_unlock(); 1508 1509 return max(smin, period); 1510 } 1511 1512 static unsigned int task_scan_max(struct task_struct *p) 1513 { 1514 unsigned long smin = task_scan_min(p); 1515 unsigned long smax; 1516 struct numa_group *ng; 1517 1518 /* Watch for min being lower than max due to floor calculations */ 1519 smax = sysctl_numa_balancing_scan_period_max / task_nr_scan_windows(p); 1520 1521 /* Scale the maximum scan period with the amount of shared memory. */ 1522 ng = deref_curr_numa_group(p); 1523 if (ng) { 1524 unsigned long shared = group_faults_shared(ng); 1525 unsigned long private = group_faults_priv(ng); 1526 unsigned long period = smax; 1527 1528 period *= refcount_read(&ng->refcount); 1529 period *= shared + 1; 1530 period /= private + shared + 1; 1531 1532 smax = max(smax, period); 1533 } 1534 1535 return max(smin, smax); 1536 } 1537 1538 static void account_numa_enqueue(struct rq *rq, struct task_struct *p) 1539 { 1540 rq->nr_numa_running += (p->numa_preferred_nid != NUMA_NO_NODE); 1541 rq->nr_preferred_running += (p->numa_preferred_nid == task_node(p)); 1542 } 1543 1544 static void account_numa_dequeue(struct rq *rq, struct task_struct *p) 1545 { 1546 rq->nr_numa_running -= (p->numa_preferred_nid != NUMA_NO_NODE); 1547 rq->nr_preferred_running -= (p->numa_preferred_nid == task_node(p)); 1548 } 1549 1550 /* Shared or private faults. */ 1551 #define NR_NUMA_HINT_FAULT_TYPES 2 1552 1553 /* Memory and CPU locality */ 1554 #define NR_NUMA_HINT_FAULT_STATS (NR_NUMA_HINT_FAULT_TYPES * 2) 1555 1556 /* Averaged statistics, and temporary buffers. */ 1557 #define NR_NUMA_HINT_FAULT_BUCKETS (NR_NUMA_HINT_FAULT_STATS * 2) 1558 1559 pid_t task_numa_group_id(struct task_struct *p) 1560 { 1561 struct numa_group *ng; 1562 pid_t gid = 0; 1563 1564 rcu_read_lock(); 1565 ng = rcu_dereference(p->numa_group); 1566 if (ng) 1567 gid = ng->gid; 1568 rcu_read_unlock(); 1569 1570 return gid; 1571 } 1572 1573 /* 1574 * The averaged statistics, shared & private, memory & CPU, 1575 * occupy the first half of the array. The second half of the 1576 * array is for current counters, which are averaged into the 1577 * first set by task_numa_placement. 1578 */ 1579 static inline int task_faults_idx(enum numa_faults_stats s, int nid, int priv) 1580 { 1581 return NR_NUMA_HINT_FAULT_TYPES * (s * nr_node_ids + nid) + priv; 1582 } 1583 1584 static inline unsigned long task_faults(struct task_struct *p, int nid) 1585 { 1586 if (!p->numa_faults) 1587 return 0; 1588 1589 return p->numa_faults[task_faults_idx(NUMA_MEM, nid, 0)] + 1590 p->numa_faults[task_faults_idx(NUMA_MEM, nid, 1)]; 1591 } 1592 1593 static inline unsigned long group_faults(struct task_struct *p, int nid) 1594 { 1595 struct numa_group *ng = deref_task_numa_group(p); 1596 1597 if (!ng) 1598 return 0; 1599 1600 return ng->faults[task_faults_idx(NUMA_MEM, nid, 0)] + 1601 ng->faults[task_faults_idx(NUMA_MEM, nid, 1)]; 1602 } 1603 1604 static inline unsigned long group_faults_cpu(struct numa_group *group, int nid) 1605 { 1606 return group->faults[task_faults_idx(NUMA_CPU, nid, 0)] + 1607 group->faults[task_faults_idx(NUMA_CPU, nid, 1)]; 1608 } 1609 1610 static inline unsigned long group_faults_priv(struct numa_group *ng) 1611 { 1612 unsigned long faults = 0; 1613 int node; 1614 1615 for_each_online_node(node) { 1616 faults += ng->faults[task_faults_idx(NUMA_MEM, node, 1)]; 1617 } 1618 1619 return faults; 1620 } 1621 1622 static inline unsigned long group_faults_shared(struct numa_group *ng) 1623 { 1624 unsigned long faults = 0; 1625 int node; 1626 1627 for_each_online_node(node) { 1628 faults += ng->faults[task_faults_idx(NUMA_MEM, node, 0)]; 1629 } 1630 1631 return faults; 1632 } 1633 1634 /* 1635 * A node triggering more than 1/3 as many NUMA faults as the maximum is 1636 * considered part of a numa group's pseudo-interleaving set. Migrations 1637 * between these nodes are slowed down, to allow things to settle down. 1638 */ 1639 #define ACTIVE_NODE_FRACTION 3 1640 1641 static bool numa_is_active_node(int nid, struct numa_group *ng) 1642 { 1643 return group_faults_cpu(ng, nid) * ACTIVE_NODE_FRACTION > ng->max_faults_cpu; 1644 } 1645 1646 /* Handle placement on systems where not all nodes are directly connected. */ 1647 static unsigned long score_nearby_nodes(struct task_struct *p, int nid, 1648 int lim_dist, bool task) 1649 { 1650 unsigned long score = 0; 1651 int node, max_dist; 1652 1653 /* 1654 * All nodes are directly connected, and the same distance 1655 * from each other. No need for fancy placement algorithms. 1656 */ 1657 if (sched_numa_topology_type == NUMA_DIRECT) 1658 return 0; 1659 1660 /* sched_max_numa_distance may be changed in parallel. */ 1661 max_dist = READ_ONCE(sched_max_numa_distance); 1662 /* 1663 * This code is called for each node, introducing N^2 complexity, 1664 * which should be ok given the number of nodes rarely exceeds 8. 1665 */ 1666 for_each_online_node(node) { 1667 unsigned long faults; 1668 int dist = node_distance(nid, node); 1669 1670 /* 1671 * The furthest away nodes in the system are not interesting 1672 * for placement; nid was already counted. 1673 */ 1674 if (dist >= max_dist || node == nid) 1675 continue; 1676 1677 /* 1678 * On systems with a backplane NUMA topology, compare groups 1679 * of nodes, and move tasks towards the group with the most 1680 * memory accesses. When comparing two nodes at distance 1681 * "hoplimit", only nodes closer by than "hoplimit" are part 1682 * of each group. Skip other nodes. 1683 */ 1684 if (sched_numa_topology_type == NUMA_BACKPLANE && dist >= lim_dist) 1685 continue; 1686 1687 /* Add up the faults from nearby nodes. */ 1688 if (task) 1689 faults = task_faults(p, node); 1690 else 1691 faults = group_faults(p, node); 1692 1693 /* 1694 * On systems with a glueless mesh NUMA topology, there are 1695 * no fixed "groups of nodes". Instead, nodes that are not 1696 * directly connected bounce traffic through intermediate 1697 * nodes; a numa_group can occupy any set of nodes. 1698 * The further away a node is, the less the faults count. 1699 * This seems to result in good task placement. 1700 */ 1701 if (sched_numa_topology_type == NUMA_GLUELESS_MESH) { 1702 faults *= (max_dist - dist); 1703 faults /= (max_dist - LOCAL_DISTANCE); 1704 } 1705 1706 score += faults; 1707 } 1708 1709 return score; 1710 } 1711 1712 /* 1713 * These return the fraction of accesses done by a particular task, or 1714 * task group, on a particular numa node. The group weight is given a 1715 * larger multiplier, in order to group tasks together that are almost 1716 * evenly spread out between numa nodes. 1717 */ 1718 static inline unsigned long task_weight(struct task_struct *p, int nid, 1719 int dist) 1720 { 1721 unsigned long faults, total_faults; 1722 1723 if (!p->numa_faults) 1724 return 0; 1725 1726 total_faults = p->total_numa_faults; 1727 1728 if (!total_faults) 1729 return 0; 1730 1731 faults = task_faults(p, nid); 1732 faults += score_nearby_nodes(p, nid, dist, true); 1733 1734 return 1000 * faults / total_faults; 1735 } 1736 1737 static inline unsigned long group_weight(struct task_struct *p, int nid, 1738 int dist) 1739 { 1740 struct numa_group *ng = deref_task_numa_group(p); 1741 unsigned long faults, total_faults; 1742 1743 if (!ng) 1744 return 0; 1745 1746 total_faults = ng->total_faults; 1747 1748 if (!total_faults) 1749 return 0; 1750 1751 faults = group_faults(p, nid); 1752 faults += score_nearby_nodes(p, nid, dist, false); 1753 1754 return 1000 * faults / total_faults; 1755 } 1756 1757 /* 1758 * If memory tiering mode is enabled, cpupid of slow memory page is 1759 * used to record scan time instead of CPU and PID. When tiering mode 1760 * is disabled at run time, the scan time (in cpupid) will be 1761 * interpreted as CPU and PID. So CPU needs to be checked to avoid to 1762 * access out of array bound. 1763 */ 1764 static inline bool cpupid_valid(int cpupid) 1765 { 1766 return cpupid_to_cpu(cpupid) < nr_cpu_ids; 1767 } 1768 1769 /* 1770 * For memory tiering mode, if there are enough free pages (more than 1771 * enough watermark defined here) in fast memory node, to take full 1772 * advantage of fast memory capacity, all recently accessed slow 1773 * memory pages will be migrated to fast memory node without 1774 * considering hot threshold. 1775 */ 1776 static bool pgdat_free_space_enough(struct pglist_data *pgdat) 1777 { 1778 int z; 1779 unsigned long enough_wmark; 1780 1781 enough_wmark = max(1UL * 1024 * 1024 * 1024 >> PAGE_SHIFT, 1782 pgdat->node_present_pages >> 4); 1783 for (z = pgdat->nr_zones - 1; z >= 0; z--) { 1784 struct zone *zone = pgdat->node_zones + z; 1785 1786 if (!populated_zone(zone)) 1787 continue; 1788 1789 if (zone_watermark_ok(zone, 0, 1790 wmark_pages(zone, WMARK_PROMO) + enough_wmark, 1791 ZONE_MOVABLE, 0)) 1792 return true; 1793 } 1794 return false; 1795 } 1796 1797 /* 1798 * For memory tiering mode, when page tables are scanned, the scan 1799 * time will be recorded in struct page in addition to make page 1800 * PROT_NONE for slow memory page. So when the page is accessed, in 1801 * hint page fault handler, the hint page fault latency is calculated 1802 * via, 1803 * 1804 * hint page fault latency = hint page fault time - scan time 1805 * 1806 * The smaller the hint page fault latency, the higher the possibility 1807 * for the page to be hot. 1808 */ 1809 static int numa_hint_fault_latency(struct page *page) 1810 { 1811 int last_time, time; 1812 1813 time = jiffies_to_msecs(jiffies); 1814 last_time = xchg_page_access_time(page, time); 1815 1816 return (time - last_time) & PAGE_ACCESS_TIME_MASK; 1817 } 1818 1819 /* 1820 * For memory tiering mode, too high promotion/demotion throughput may 1821 * hurt application latency. So we provide a mechanism to rate limit 1822 * the number of pages that are tried to be promoted. 1823 */ 1824 static bool numa_promotion_rate_limit(struct pglist_data *pgdat, 1825 unsigned long rate_limit, int nr) 1826 { 1827 unsigned long nr_cand; 1828 unsigned int now, start; 1829 1830 now = jiffies_to_msecs(jiffies); 1831 mod_node_page_state(pgdat, PGPROMOTE_CANDIDATE, nr); 1832 nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE); 1833 start = pgdat->nbp_rl_start; 1834 if (now - start > MSEC_PER_SEC && 1835 cmpxchg(&pgdat->nbp_rl_start, start, now) == start) 1836 pgdat->nbp_rl_nr_cand = nr_cand; 1837 if (nr_cand - pgdat->nbp_rl_nr_cand >= rate_limit) 1838 return true; 1839 return false; 1840 } 1841 1842 #define NUMA_MIGRATION_ADJUST_STEPS 16 1843 1844 static void numa_promotion_adjust_threshold(struct pglist_data *pgdat, 1845 unsigned long rate_limit, 1846 unsigned int ref_th) 1847 { 1848 unsigned int now, start, th_period, unit_th, th; 1849 unsigned long nr_cand, ref_cand, diff_cand; 1850 1851 now = jiffies_to_msecs(jiffies); 1852 th_period = sysctl_numa_balancing_scan_period_max; 1853 start = pgdat->nbp_th_start; 1854 if (now - start > th_period && 1855 cmpxchg(&pgdat->nbp_th_start, start, now) == start) { 1856 ref_cand = rate_limit * 1857 sysctl_numa_balancing_scan_period_max / MSEC_PER_SEC; 1858 nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE); 1859 diff_cand = nr_cand - pgdat->nbp_th_nr_cand; 1860 unit_th = ref_th * 2 / NUMA_MIGRATION_ADJUST_STEPS; 1861 th = pgdat->nbp_threshold ? : ref_th; 1862 if (diff_cand > ref_cand * 11 / 10) 1863 th = max(th - unit_th, unit_th); 1864 else if (diff_cand < ref_cand * 9 / 10) 1865 th = min(th + unit_th, ref_th * 2); 1866 pgdat->nbp_th_nr_cand = nr_cand; 1867 pgdat->nbp_threshold = th; 1868 } 1869 } 1870 1871 bool should_numa_migrate_memory(struct task_struct *p, struct page * page, 1872 int src_nid, int dst_cpu) 1873 { 1874 struct numa_group *ng = deref_curr_numa_group(p); 1875 int dst_nid = cpu_to_node(dst_cpu); 1876 int last_cpupid, this_cpupid; 1877 1878 /* 1879 * The pages in slow memory node should be migrated according 1880 * to hot/cold instead of private/shared. 1881 */ 1882 if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING && 1883 !node_is_toptier(src_nid)) { 1884 struct pglist_data *pgdat; 1885 unsigned long rate_limit; 1886 unsigned int latency, th, def_th; 1887 1888 pgdat = NODE_DATA(dst_nid); 1889 if (pgdat_free_space_enough(pgdat)) { 1890 /* workload changed, reset hot threshold */ 1891 pgdat->nbp_threshold = 0; 1892 return true; 1893 } 1894 1895 def_th = sysctl_numa_balancing_hot_threshold; 1896 rate_limit = sysctl_numa_balancing_promote_rate_limit << \ 1897 (20 - PAGE_SHIFT); 1898 numa_promotion_adjust_threshold(pgdat, rate_limit, def_th); 1899 1900 th = pgdat->nbp_threshold ? : def_th; 1901 latency = numa_hint_fault_latency(page); 1902 if (latency >= th) 1903 return false; 1904 1905 return !numa_promotion_rate_limit(pgdat, rate_limit, 1906 thp_nr_pages(page)); 1907 } 1908 1909 this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid); 1910 last_cpupid = page_cpupid_xchg_last(page, this_cpupid); 1911 1912 if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) && 1913 !node_is_toptier(src_nid) && !cpupid_valid(last_cpupid)) 1914 return false; 1915 1916 /* 1917 * Allow first faults or private faults to migrate immediately early in 1918 * the lifetime of a task. The magic number 4 is based on waiting for 1919 * two full passes of the "multi-stage node selection" test that is 1920 * executed below. 1921 */ 1922 if ((p->numa_preferred_nid == NUMA_NO_NODE || p->numa_scan_seq <= 4) && 1923 (cpupid_pid_unset(last_cpupid) || cpupid_match_pid(p, last_cpupid))) 1924 return true; 1925 1926 /* 1927 * Multi-stage node selection is used in conjunction with a periodic 1928 * migration fault to build a temporal task<->page relation. By using 1929 * a two-stage filter we remove short/unlikely relations. 1930 * 1931 * Using P(p) ~ n_p / n_t as per frequentist probability, we can equate 1932 * a task's usage of a particular page (n_p) per total usage of this 1933 * page (n_t) (in a given time-span) to a probability. 1934 * 1935 * Our periodic faults will sample this probability and getting the 1936 * same result twice in a row, given these samples are fully 1937 * independent, is then given by P(n)^2, provided our sample period 1938 * is sufficiently short compared to the usage pattern. 1939 * 1940 * This quadric squishes small probabilities, making it less likely we 1941 * act on an unlikely task<->page relation. 1942 */ 1943 if (!cpupid_pid_unset(last_cpupid) && 1944 cpupid_to_nid(last_cpupid) != dst_nid) 1945 return false; 1946 1947 /* Always allow migrate on private faults */ 1948 if (cpupid_match_pid(p, last_cpupid)) 1949 return true; 1950 1951 /* A shared fault, but p->numa_group has not been set up yet. */ 1952 if (!ng) 1953 return true; 1954 1955 /* 1956 * Destination node is much more heavily used than the source 1957 * node? Allow migration. 1958 */ 1959 if (group_faults_cpu(ng, dst_nid) > group_faults_cpu(ng, src_nid) * 1960 ACTIVE_NODE_FRACTION) 1961 return true; 1962 1963 /* 1964 * Distribute memory according to CPU & memory use on each node, 1965 * with 3/4 hysteresis to avoid unnecessary memory migrations: 1966 * 1967 * faults_cpu(dst) 3 faults_cpu(src) 1968 * --------------- * - > --------------- 1969 * faults_mem(dst) 4 faults_mem(src) 1970 */ 1971 return group_faults_cpu(ng, dst_nid) * group_faults(p, src_nid) * 3 > 1972 group_faults_cpu(ng, src_nid) * group_faults(p, dst_nid) * 4; 1973 } 1974 1975 /* 1976 * 'numa_type' describes the node at the moment of load balancing. 1977 */ 1978 enum numa_type { 1979 /* The node has spare capacity that can be used to run more tasks. */ 1980 node_has_spare = 0, 1981 /* 1982 * The node is fully used and the tasks don't compete for more CPU 1983 * cycles. Nevertheless, some tasks might wait before running. 1984 */ 1985 node_fully_busy, 1986 /* 1987 * The node is overloaded and can't provide expected CPU cycles to all 1988 * tasks. 1989 */ 1990 node_overloaded 1991 }; 1992 1993 /* Cached statistics for all CPUs within a node */ 1994 struct numa_stats { 1995 unsigned long load; 1996 unsigned long runnable; 1997 unsigned long util; 1998 /* Total compute capacity of CPUs on a node */ 1999 unsigned long compute_capacity; 2000 unsigned int nr_running; 2001 unsigned int weight; 2002 enum numa_type node_type; 2003 int idle_cpu; 2004 }; 2005 2006 struct task_numa_env { 2007 struct task_struct *p; 2008 2009 int src_cpu, src_nid; 2010 int dst_cpu, dst_nid; 2011 int imb_numa_nr; 2012 2013 struct numa_stats src_stats, dst_stats; 2014 2015 int imbalance_pct; 2016 int dist; 2017 2018 struct task_struct *best_task; 2019 long best_imp; 2020 int best_cpu; 2021 }; 2022 2023 static unsigned long cpu_load(struct rq *rq); 2024 static unsigned long cpu_runnable(struct rq *rq); 2025 2026 static inline enum 2027 numa_type numa_classify(unsigned int imbalance_pct, 2028 struct numa_stats *ns) 2029 { 2030 if ((ns->nr_running > ns->weight) && 2031 (((ns->compute_capacity * 100) < (ns->util * imbalance_pct)) || 2032 ((ns->compute_capacity * imbalance_pct) < (ns->runnable * 100)))) 2033 return node_overloaded; 2034 2035 if ((ns->nr_running < ns->weight) || 2036 (((ns->compute_capacity * 100) > (ns->util * imbalance_pct)) && 2037 ((ns->compute_capacity * imbalance_pct) > (ns->runnable * 100)))) 2038 return node_has_spare; 2039 2040 return node_fully_busy; 2041 } 2042 2043 #ifdef CONFIG_SCHED_SMT 2044 /* Forward declarations of select_idle_sibling helpers */ 2045 static inline bool test_idle_cores(int cpu); 2046 static inline int numa_idle_core(int idle_core, int cpu) 2047 { 2048 if (!static_branch_likely(&sched_smt_present) || 2049 idle_core >= 0 || !test_idle_cores(cpu)) 2050 return idle_core; 2051 2052 /* 2053 * Prefer cores instead of packing HT siblings 2054 * and triggering future load balancing. 2055 */ 2056 if (is_core_idle(cpu)) 2057 idle_core = cpu; 2058 2059 return idle_core; 2060 } 2061 #else 2062 static inline int numa_idle_core(int idle_core, int cpu) 2063 { 2064 return idle_core; 2065 } 2066 #endif 2067 2068 /* 2069 * Gather all necessary information to make NUMA balancing placement 2070 * decisions that are compatible with standard load balancer. This 2071 * borrows code and logic from update_sg_lb_stats but sharing a 2072 * common implementation is impractical. 2073 */ 2074 static void update_numa_stats(struct task_numa_env *env, 2075 struct numa_stats *ns, int nid, 2076 bool find_idle) 2077 { 2078 int cpu, idle_core = -1; 2079 2080 memset(ns, 0, sizeof(*ns)); 2081 ns->idle_cpu = -1; 2082 2083 rcu_read_lock(); 2084 for_each_cpu(cpu, cpumask_of_node(nid)) { 2085 struct rq *rq = cpu_rq(cpu); 2086 2087 ns->load += cpu_load(rq); 2088 ns->runnable += cpu_runnable(rq); 2089 ns->util += cpu_util_cfs(cpu); 2090 ns->nr_running += rq->cfs.h_nr_running; 2091 ns->compute_capacity += capacity_of(cpu); 2092 2093 if (find_idle && idle_core < 0 && !rq->nr_running && idle_cpu(cpu)) { 2094 if (READ_ONCE(rq->numa_migrate_on) || 2095 !cpumask_test_cpu(cpu, env->p->cpus_ptr)) 2096 continue; 2097 2098 if (ns->idle_cpu == -1) 2099 ns->idle_cpu = cpu; 2100 2101 idle_core = numa_idle_core(idle_core, cpu); 2102 } 2103 } 2104 rcu_read_unlock(); 2105 2106 ns->weight = cpumask_weight(cpumask_of_node(nid)); 2107 2108 ns->node_type = numa_classify(env->imbalance_pct, ns); 2109 2110 if (idle_core >= 0) 2111 ns->idle_cpu = idle_core; 2112 } 2113 2114 static void task_numa_assign(struct task_numa_env *env, 2115 struct task_struct *p, long imp) 2116 { 2117 struct rq *rq = cpu_rq(env->dst_cpu); 2118 2119 /* Check if run-queue part of active NUMA balance. */ 2120 if (env->best_cpu != env->dst_cpu && xchg(&rq->numa_migrate_on, 1)) { 2121 int cpu; 2122 int start = env->dst_cpu; 2123 2124 /* Find alternative idle CPU. */ 2125 for_each_cpu_wrap(cpu, cpumask_of_node(env->dst_nid), start + 1) { 2126 if (cpu == env->best_cpu || !idle_cpu(cpu) || 2127 !cpumask_test_cpu(cpu, env->p->cpus_ptr)) { 2128 continue; 2129 } 2130 2131 env->dst_cpu = cpu; 2132 rq = cpu_rq(env->dst_cpu); 2133 if (!xchg(&rq->numa_migrate_on, 1)) 2134 goto assign; 2135 } 2136 2137 /* Failed to find an alternative idle CPU */ 2138 return; 2139 } 2140 2141 assign: 2142 /* 2143 * Clear previous best_cpu/rq numa-migrate flag, since task now 2144 * found a better CPU to move/swap. 2145 */ 2146 if (env->best_cpu != -1 && env->best_cpu != env->dst_cpu) { 2147 rq = cpu_rq(env->best_cpu); 2148 WRITE_ONCE(rq->numa_migrate_on, 0); 2149 } 2150 2151 if (env->best_task) 2152 put_task_struct(env->best_task); 2153 if (p) 2154 get_task_struct(p); 2155 2156 env->best_task = p; 2157 env->best_imp = imp; 2158 env->best_cpu = env->dst_cpu; 2159 } 2160 2161 static bool load_too_imbalanced(long src_load, long dst_load, 2162 struct task_numa_env *env) 2163 { 2164 long imb, old_imb; 2165 long orig_src_load, orig_dst_load; 2166 long src_capacity, dst_capacity; 2167 2168 /* 2169 * The load is corrected for the CPU capacity available on each node. 2170 * 2171 * src_load dst_load 2172 * ------------ vs --------- 2173 * src_capacity dst_capacity 2174 */ 2175 src_capacity = env->src_stats.compute_capacity; 2176 dst_capacity = env->dst_stats.compute_capacity; 2177 2178 imb = abs(dst_load * src_capacity - src_load * dst_capacity); 2179 2180 orig_src_load = env->src_stats.load; 2181 orig_dst_load = env->dst_stats.load; 2182 2183 old_imb = abs(orig_dst_load * src_capacity - orig_src_load * dst_capacity); 2184 2185 /* Would this change make things worse? */ 2186 return (imb > old_imb); 2187 } 2188 2189 /* 2190 * Maximum NUMA importance can be 1998 (2*999); 2191 * SMALLIMP @ 30 would be close to 1998/64. 2192 * Used to deter task migration. 2193 */ 2194 #define SMALLIMP 30 2195 2196 /* 2197 * This checks if the overall compute and NUMA accesses of the system would 2198 * be improved if the source tasks was migrated to the target dst_cpu taking 2199 * into account that it might be best if task running on the dst_cpu should 2200 * be exchanged with the source task 2201 */ 2202 static bool task_numa_compare(struct task_numa_env *env, 2203 long taskimp, long groupimp, bool maymove) 2204 { 2205 struct numa_group *cur_ng, *p_ng = deref_curr_numa_group(env->p); 2206 struct rq *dst_rq = cpu_rq(env->dst_cpu); 2207 long imp = p_ng ? groupimp : taskimp; 2208 struct task_struct *cur; 2209 long src_load, dst_load; 2210 int dist = env->dist; 2211 long moveimp = imp; 2212 long load; 2213 bool stopsearch = false; 2214 2215 if (READ_ONCE(dst_rq->numa_migrate_on)) 2216 return false; 2217 2218 rcu_read_lock(); 2219 cur = rcu_dereference(dst_rq->curr); 2220 if (cur && ((cur->flags & PF_EXITING) || is_idle_task(cur))) 2221 cur = NULL; 2222 2223 /* 2224 * Because we have preemption enabled we can get migrated around and 2225 * end try selecting ourselves (current == env->p) as a swap candidate. 2226 */ 2227 if (cur == env->p) { 2228 stopsearch = true; 2229 goto unlock; 2230 } 2231 2232 if (!cur) { 2233 if (maymove && moveimp >= env->best_imp) 2234 goto assign; 2235 else 2236 goto unlock; 2237 } 2238 2239 /* Skip this swap candidate if cannot move to the source cpu. */ 2240 if (!cpumask_test_cpu(env->src_cpu, cur->cpus_ptr)) 2241 goto unlock; 2242 2243 /* 2244 * Skip this swap candidate if it is not moving to its preferred 2245 * node and the best task is. 2246 */ 2247 if (env->best_task && 2248 env->best_task->numa_preferred_nid == env->src_nid && 2249 cur->numa_preferred_nid != env->src_nid) { 2250 goto unlock; 2251 } 2252 2253 /* 2254 * "imp" is the fault differential for the source task between the 2255 * source and destination node. Calculate the total differential for 2256 * the source task and potential destination task. The more negative 2257 * the value is, the more remote accesses that would be expected to 2258 * be incurred if the tasks were swapped. 2259 * 2260 * If dst and source tasks are in the same NUMA group, or not 2261 * in any group then look only at task weights. 2262 */ 2263 cur_ng = rcu_dereference(cur->numa_group); 2264 if (cur_ng == p_ng) { 2265 /* 2266 * Do not swap within a group or between tasks that have 2267 * no group if there is spare capacity. Swapping does 2268 * not address the load imbalance and helps one task at 2269 * the cost of punishing another. 2270 */ 2271 if (env->dst_stats.node_type == node_has_spare) 2272 goto unlock; 2273 2274 imp = taskimp + task_weight(cur, env->src_nid, dist) - 2275 task_weight(cur, env->dst_nid, dist); 2276 /* 2277 * Add some hysteresis to prevent swapping the 2278 * tasks within a group over tiny differences. 2279 */ 2280 if (cur_ng) 2281 imp -= imp / 16; 2282 } else { 2283 /* 2284 * Compare the group weights. If a task is all by itself 2285 * (not part of a group), use the task weight instead. 2286 */ 2287 if (cur_ng && p_ng) 2288 imp += group_weight(cur, env->src_nid, dist) - 2289 group_weight(cur, env->dst_nid, dist); 2290 else 2291 imp += task_weight(cur, env->src_nid, dist) - 2292 task_weight(cur, env->dst_nid, dist); 2293 } 2294 2295 /* Discourage picking a task already on its preferred node */ 2296 if (cur->numa_preferred_nid == env->dst_nid) 2297 imp -= imp / 16; 2298 2299 /* 2300 * Encourage picking a task that moves to its preferred node. 2301 * This potentially makes imp larger than it's maximum of 2302 * 1998 (see SMALLIMP and task_weight for why) but in this 2303 * case, it does not matter. 2304 */ 2305 if (cur->numa_preferred_nid == env->src_nid) 2306 imp += imp / 8; 2307 2308 if (maymove && moveimp > imp && moveimp > env->best_imp) { 2309 imp = moveimp; 2310 cur = NULL; 2311 goto assign; 2312 } 2313 2314 /* 2315 * Prefer swapping with a task moving to its preferred node over a 2316 * task that is not. 2317 */ 2318 if (env->best_task && cur->numa_preferred_nid == env->src_nid && 2319 env->best_task->numa_preferred_nid != env->src_nid) { 2320 goto assign; 2321 } 2322 2323 /* 2324 * If the NUMA importance is less than SMALLIMP, 2325 * task migration might only result in ping pong 2326 * of tasks and also hurt performance due to cache 2327 * misses. 2328 */ 2329 if (imp < SMALLIMP || imp <= env->best_imp + SMALLIMP / 2) 2330 goto unlock; 2331 2332 /* 2333 * In the overloaded case, try and keep the load balanced. 2334 */ 2335 load = task_h_load(env->p) - task_h_load(cur); 2336 if (!load) 2337 goto assign; 2338 2339 dst_load = env->dst_stats.load + load; 2340 src_load = env->src_stats.load - load; 2341 2342 if (load_too_imbalanced(src_load, dst_load, env)) 2343 goto unlock; 2344 2345 assign: 2346 /* Evaluate an idle CPU for a task numa move. */ 2347 if (!cur) { 2348 int cpu = env->dst_stats.idle_cpu; 2349 2350 /* Nothing cached so current CPU went idle since the search. */ 2351 if (cpu < 0) 2352 cpu = env->dst_cpu; 2353 2354 /* 2355 * If the CPU is no longer truly idle and the previous best CPU 2356 * is, keep using it. 2357 */ 2358 if (!idle_cpu(cpu) && env->best_cpu >= 0 && 2359 idle_cpu(env->best_cpu)) { 2360 cpu = env->best_cpu; 2361 } 2362 2363 env->dst_cpu = cpu; 2364 } 2365 2366 task_numa_assign(env, cur, imp); 2367 2368 /* 2369 * If a move to idle is allowed because there is capacity or load 2370 * balance improves then stop the search. While a better swap 2371 * candidate may exist, a search is not free. 2372 */ 2373 if (maymove && !cur && env->best_cpu >= 0 && idle_cpu(env->best_cpu)) 2374 stopsearch = true; 2375 2376 /* 2377 * If a swap candidate must be identified and the current best task 2378 * moves its preferred node then stop the search. 2379 */ 2380 if (!maymove && env->best_task && 2381 env->best_task->numa_preferred_nid == env->src_nid) { 2382 stopsearch = true; 2383 } 2384 unlock: 2385 rcu_read_unlock(); 2386 2387 return stopsearch; 2388 } 2389 2390 static void task_numa_find_cpu(struct task_numa_env *env, 2391 long taskimp, long groupimp) 2392 { 2393 bool maymove = false; 2394 int cpu; 2395 2396 /* 2397 * If dst node has spare capacity, then check if there is an 2398 * imbalance that would be overruled by the load balancer. 2399 */ 2400 if (env->dst_stats.node_type == node_has_spare) { 2401 unsigned int imbalance; 2402 int src_running, dst_running; 2403 2404 /* 2405 * Would movement cause an imbalance? Note that if src has 2406 * more running tasks that the imbalance is ignored as the 2407 * move improves the imbalance from the perspective of the 2408 * CPU load balancer. 2409 * */ 2410 src_running = env->src_stats.nr_running - 1; 2411 dst_running = env->dst_stats.nr_running + 1; 2412 imbalance = max(0, dst_running - src_running); 2413 imbalance = adjust_numa_imbalance(imbalance, dst_running, 2414 env->imb_numa_nr); 2415 2416 /* Use idle CPU if there is no imbalance */ 2417 if (!imbalance) { 2418 maymove = true; 2419 if (env->dst_stats.idle_cpu >= 0) { 2420 env->dst_cpu = env->dst_stats.idle_cpu; 2421 task_numa_assign(env, NULL, 0); 2422 return; 2423 } 2424 } 2425 } else { 2426 long src_load, dst_load, load; 2427 /* 2428 * If the improvement from just moving env->p direction is better 2429 * than swapping tasks around, check if a move is possible. 2430 */ 2431 load = task_h_load(env->p); 2432 dst_load = env->dst_stats.load + load; 2433 src_load = env->src_stats.load - load; 2434 maymove = !load_too_imbalanced(src_load, dst_load, env); 2435 } 2436 2437 for_each_cpu(cpu, cpumask_of_node(env->dst_nid)) { 2438 /* Skip this CPU if the source task cannot migrate */ 2439 if (!cpumask_test_cpu(cpu, env->p->cpus_ptr)) 2440 continue; 2441 2442 env->dst_cpu = cpu; 2443 if (task_numa_compare(env, taskimp, groupimp, maymove)) 2444 break; 2445 } 2446 } 2447 2448 static int task_numa_migrate(struct task_struct *p) 2449 { 2450 struct task_numa_env env = { 2451 .p = p, 2452 2453 .src_cpu = task_cpu(p), 2454 .src_nid = task_node(p), 2455 2456 .imbalance_pct = 112, 2457 2458 .best_task = NULL, 2459 .best_imp = 0, 2460 .best_cpu = -1, 2461 }; 2462 unsigned long taskweight, groupweight; 2463 struct sched_domain *sd; 2464 long taskimp, groupimp; 2465 struct numa_group *ng; 2466 struct rq *best_rq; 2467 int nid, ret, dist; 2468 2469 /* 2470 * Pick the lowest SD_NUMA domain, as that would have the smallest 2471 * imbalance and would be the first to start moving tasks about. 2472 * 2473 * And we want to avoid any moving of tasks about, as that would create 2474 * random movement of tasks -- counter the numa conditions we're trying 2475 * to satisfy here. 2476 */ 2477 rcu_read_lock(); 2478 sd = rcu_dereference(per_cpu(sd_numa, env.src_cpu)); 2479 if (sd) { 2480 env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2; 2481 env.imb_numa_nr = sd->imb_numa_nr; 2482 } 2483 rcu_read_unlock(); 2484 2485 /* 2486 * Cpusets can break the scheduler domain tree into smaller 2487 * balance domains, some of which do not cross NUMA boundaries. 2488 * Tasks that are "trapped" in such domains cannot be migrated 2489 * elsewhere, so there is no point in (re)trying. 2490 */ 2491 if (unlikely(!sd)) { 2492 sched_setnuma(p, task_node(p)); 2493 return -EINVAL; 2494 } 2495 2496 env.dst_nid = p->numa_preferred_nid; 2497 dist = env.dist = node_distance(env.src_nid, env.dst_nid); 2498 taskweight = task_weight(p, env.src_nid, dist); 2499 groupweight = group_weight(p, env.src_nid, dist); 2500 update_numa_stats(&env, &env.src_stats, env.src_nid, false); 2501 taskimp = task_weight(p, env.dst_nid, dist) - taskweight; 2502 groupimp = group_weight(p, env.dst_nid, dist) - groupweight; 2503 update_numa_stats(&env, &env.dst_stats, env.dst_nid, true); 2504 2505 /* Try to find a spot on the preferred nid. */ 2506 task_numa_find_cpu(&env, taskimp, groupimp); 2507 2508 /* 2509 * Look at other nodes in these cases: 2510 * - there is no space available on the preferred_nid 2511 * - the task is part of a numa_group that is interleaved across 2512 * multiple NUMA nodes; in order to better consolidate the group, 2513 * we need to check other locations. 2514 */ 2515 ng = deref_curr_numa_group(p); 2516 if (env.best_cpu == -1 || (ng && ng->active_nodes > 1)) { 2517 for_each_node_state(nid, N_CPU) { 2518 if (nid == env.src_nid || nid == p->numa_preferred_nid) 2519 continue; 2520 2521 dist = node_distance(env.src_nid, env.dst_nid); 2522 if (sched_numa_topology_type == NUMA_BACKPLANE && 2523 dist != env.dist) { 2524 taskweight = task_weight(p, env.src_nid, dist); 2525 groupweight = group_weight(p, env.src_nid, dist); 2526 } 2527 2528 /* Only consider nodes where both task and groups benefit */ 2529 taskimp = task_weight(p, nid, dist) - taskweight; 2530 groupimp = group_weight(p, nid, dist) - groupweight; 2531 if (taskimp < 0 && groupimp < 0) 2532 continue; 2533 2534 env.dist = dist; 2535 env.dst_nid = nid; 2536 update_numa_stats(&env, &env.dst_stats, env.dst_nid, true); 2537 task_numa_find_cpu(&env, taskimp, groupimp); 2538 } 2539 } 2540 2541 /* 2542 * If the task is part of a workload that spans multiple NUMA nodes, 2543 * and is migrating into one of the workload's active nodes, remember 2544 * this node as the task's preferred numa node, so the workload can 2545 * settle down. 2546 * A task that migrated to a second choice node will be better off 2547 * trying for a better one later. Do not set the preferred node here. 2548 */ 2549 if (ng) { 2550 if (env.best_cpu == -1) 2551 nid = env.src_nid; 2552 else 2553 nid = cpu_to_node(env.best_cpu); 2554 2555 if (nid != p->numa_preferred_nid) 2556 sched_setnuma(p, nid); 2557 } 2558 2559 /* No better CPU than the current one was found. */ 2560 if (env.best_cpu == -1) { 2561 trace_sched_stick_numa(p, env.src_cpu, NULL, -1); 2562 return -EAGAIN; 2563 } 2564 2565 best_rq = cpu_rq(env.best_cpu); 2566 if (env.best_task == NULL) { 2567 ret = migrate_task_to(p, env.best_cpu); 2568 WRITE_ONCE(best_rq->numa_migrate_on, 0); 2569 if (ret != 0) 2570 trace_sched_stick_numa(p, env.src_cpu, NULL, env.best_cpu); 2571 return ret; 2572 } 2573 2574 ret = migrate_swap(p, env.best_task, env.best_cpu, env.src_cpu); 2575 WRITE_ONCE(best_rq->numa_migrate_on, 0); 2576 2577 if (ret != 0) 2578 trace_sched_stick_numa(p, env.src_cpu, env.best_task, env.best_cpu); 2579 put_task_struct(env.best_task); 2580 return ret; 2581 } 2582 2583 /* Attempt to migrate a task to a CPU on the preferred node. */ 2584 static void numa_migrate_preferred(struct task_struct *p) 2585 { 2586 unsigned long interval = HZ; 2587 2588 /* This task has no NUMA fault statistics yet */ 2589 if (unlikely(p->numa_preferred_nid == NUMA_NO_NODE || !p->numa_faults)) 2590 return; 2591 2592 /* Periodically retry migrating the task to the preferred node */ 2593 interval = min(interval, msecs_to_jiffies(p->numa_scan_period) / 16); 2594 p->numa_migrate_retry = jiffies + interval; 2595 2596 /* Success if task is already running on preferred CPU */ 2597 if (task_node(p) == p->numa_preferred_nid) 2598 return; 2599 2600 /* Otherwise, try migrate to a CPU on the preferred node */ 2601 task_numa_migrate(p); 2602 } 2603 2604 /* 2605 * Find out how many nodes the workload is actively running on. Do this by 2606 * tracking the nodes from which NUMA hinting faults are triggered. This can 2607 * be different from the set of nodes where the workload's memory is currently 2608 * located. 2609 */ 2610 static void numa_group_count_active_nodes(struct numa_group *numa_group) 2611 { 2612 unsigned long faults, max_faults = 0; 2613 int nid, active_nodes = 0; 2614 2615 for_each_node_state(nid, N_CPU) { 2616 faults = group_faults_cpu(numa_group, nid); 2617 if (faults > max_faults) 2618 max_faults = faults; 2619 } 2620 2621 for_each_node_state(nid, N_CPU) { 2622 faults = group_faults_cpu(numa_group, nid); 2623 if (faults * ACTIVE_NODE_FRACTION > max_faults) 2624 active_nodes++; 2625 } 2626 2627 numa_group->max_faults_cpu = max_faults; 2628 numa_group->active_nodes = active_nodes; 2629 } 2630 2631 /* 2632 * When adapting the scan rate, the period is divided into NUMA_PERIOD_SLOTS 2633 * increments. The more local the fault statistics are, the higher the scan 2634 * period will be for the next scan window. If local/(local+remote) ratio is 2635 * below NUMA_PERIOD_THRESHOLD (where range of ratio is 1..NUMA_PERIOD_SLOTS) 2636 * the scan period will decrease. Aim for 70% local accesses. 2637 */ 2638 #define NUMA_PERIOD_SLOTS 10 2639 #define NUMA_PERIOD_THRESHOLD 7 2640 2641 /* 2642 * Increase the scan period (slow down scanning) if the majority of 2643 * our memory is already on our local node, or if the majority of 2644 * the page accesses are shared with other processes. 2645 * Otherwise, decrease the scan period. 2646 */ 2647 static void update_task_scan_period(struct task_struct *p, 2648 unsigned long shared, unsigned long private) 2649 { 2650 unsigned int period_slot; 2651 int lr_ratio, ps_ratio; 2652 int diff; 2653 2654 unsigned long remote = p->numa_faults_locality[0]; 2655 unsigned long local = p->numa_faults_locality[1]; 2656 2657 /* 2658 * If there were no record hinting faults then either the task is 2659 * completely idle or all activity is in areas that are not of interest 2660 * to automatic numa balancing. Related to that, if there were failed 2661 * migration then it implies we are migrating too quickly or the local 2662 * node is overloaded. In either case, scan slower 2663 */ 2664 if (local + shared == 0 || p->numa_faults_locality[2]) { 2665 p->numa_scan_period = min(p->numa_scan_period_max, 2666 p->numa_scan_period << 1); 2667 2668 p->mm->numa_next_scan = jiffies + 2669 msecs_to_jiffies(p->numa_scan_period); 2670 2671 return; 2672 } 2673 2674 /* 2675 * Prepare to scale scan period relative to the current period. 2676 * == NUMA_PERIOD_THRESHOLD scan period stays the same 2677 * < NUMA_PERIOD_THRESHOLD scan period decreases (scan faster) 2678 * >= NUMA_PERIOD_THRESHOLD scan period increases (scan slower) 2679 */ 2680 period_slot = DIV_ROUND_UP(p->numa_scan_period, NUMA_PERIOD_SLOTS); 2681 lr_ratio = (local * NUMA_PERIOD_SLOTS) / (local + remote); 2682 ps_ratio = (private * NUMA_PERIOD_SLOTS) / (private + shared); 2683 2684 if (ps_ratio >= NUMA_PERIOD_THRESHOLD) { 2685 /* 2686 * Most memory accesses are local. There is no need to 2687 * do fast NUMA scanning, since memory is already local. 2688 */ 2689 int slot = ps_ratio - NUMA_PERIOD_THRESHOLD; 2690 if (!slot) 2691 slot = 1; 2692 diff = slot * period_slot; 2693 } else if (lr_ratio >= NUMA_PERIOD_THRESHOLD) { 2694 /* 2695 * Most memory accesses are shared with other tasks. 2696 * There is no point in continuing fast NUMA scanning, 2697 * since other tasks may just move the memory elsewhere. 2698 */ 2699 int slot = lr_ratio - NUMA_PERIOD_THRESHOLD; 2700 if (!slot) 2701 slot = 1; 2702 diff = slot * period_slot; 2703 } else { 2704 /* 2705 * Private memory faults exceed (SLOTS-THRESHOLD)/SLOTS, 2706 * yet they are not on the local NUMA node. Speed up 2707 * NUMA scanning to get the memory moved over. 2708 */ 2709 int ratio = max(lr_ratio, ps_ratio); 2710 diff = -(NUMA_PERIOD_THRESHOLD - ratio) * period_slot; 2711 } 2712 2713 p->numa_scan_period = clamp(p->numa_scan_period + diff, 2714 task_scan_min(p), task_scan_max(p)); 2715 memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); 2716 } 2717 2718 /* 2719 * Get the fraction of time the task has been running since the last 2720 * NUMA placement cycle. The scheduler keeps similar statistics, but 2721 * decays those on a 32ms period, which is orders of magnitude off 2722 * from the dozens-of-seconds NUMA balancing period. Use the scheduler 2723 * stats only if the task is so new there are no NUMA statistics yet. 2724 */ 2725 static u64 numa_get_avg_runtime(struct task_struct *p, u64 *period) 2726 { 2727 u64 runtime, delta, now; 2728 /* Use the start of this time slice to avoid calculations. */ 2729 now = p->se.exec_start; 2730 runtime = p->se.sum_exec_runtime; 2731 2732 if (p->last_task_numa_placement) { 2733 delta = runtime - p->last_sum_exec_runtime; 2734 *period = now - p->last_task_numa_placement; 2735 2736 /* Avoid time going backwards, prevent potential divide error: */ 2737 if (unlikely((s64)*period < 0)) 2738 *period = 0; 2739 } else { 2740 delta = p->se.avg.load_sum; 2741 *period = LOAD_AVG_MAX; 2742 } 2743 2744 p->last_sum_exec_runtime = runtime; 2745 p->last_task_numa_placement = now; 2746 2747 return delta; 2748 } 2749 2750 /* 2751 * Determine the preferred nid for a task in a numa_group. This needs to 2752 * be done in a way that produces consistent results with group_weight, 2753 * otherwise workloads might not converge. 2754 */ 2755 static int preferred_group_nid(struct task_struct *p, int nid) 2756 { 2757 nodemask_t nodes; 2758 int dist; 2759 2760 /* Direct connections between all NUMA nodes. */ 2761 if (sched_numa_topology_type == NUMA_DIRECT) 2762 return nid; 2763 2764 /* 2765 * On a system with glueless mesh NUMA topology, group_weight 2766 * scores nodes according to the number of NUMA hinting faults on 2767 * both the node itself, and on nearby nodes. 2768 */ 2769 if (sched_numa_topology_type == NUMA_GLUELESS_MESH) { 2770 unsigned long score, max_score = 0; 2771 int node, max_node = nid; 2772 2773 dist = sched_max_numa_distance; 2774 2775 for_each_node_state(node, N_CPU) { 2776 score = group_weight(p, node, dist); 2777 if (score > max_score) { 2778 max_score = score; 2779 max_node = node; 2780 } 2781 } 2782 return max_node; 2783 } 2784 2785 /* 2786 * Finding the preferred nid in a system with NUMA backplane 2787 * interconnect topology is more involved. The goal is to locate 2788 * tasks from numa_groups near each other in the system, and 2789 * untangle workloads from different sides of the system. This requires 2790 * searching down the hierarchy of node groups, recursively searching 2791 * inside the highest scoring group of nodes. The nodemask tricks 2792 * keep the complexity of the search down. 2793 */ 2794 nodes = node_states[N_CPU]; 2795 for (dist = sched_max_numa_distance; dist > LOCAL_DISTANCE; dist--) { 2796 unsigned long max_faults = 0; 2797 nodemask_t max_group = NODE_MASK_NONE; 2798 int a, b; 2799 2800 /* Are there nodes at this distance from each other? */ 2801 if (!find_numa_distance(dist)) 2802 continue; 2803 2804 for_each_node_mask(a, nodes) { 2805 unsigned long faults = 0; 2806 nodemask_t this_group; 2807 nodes_clear(this_group); 2808 2809 /* Sum group's NUMA faults; includes a==b case. */ 2810 for_each_node_mask(b, nodes) { 2811 if (node_distance(a, b) < dist) { 2812 faults += group_faults(p, b); 2813 node_set(b, this_group); 2814 node_clear(b, nodes); 2815 } 2816 } 2817 2818 /* Remember the top group. */ 2819 if (faults > max_faults) { 2820 max_faults = faults; 2821 max_group = this_group; 2822 /* 2823 * subtle: at the smallest distance there is 2824 * just one node left in each "group", the 2825 * winner is the preferred nid. 2826 */ 2827 nid = a; 2828 } 2829 } 2830 /* Next round, evaluate the nodes within max_group. */ 2831 if (!max_faults) 2832 break; 2833 nodes = max_group; 2834 } 2835 return nid; 2836 } 2837 2838 static void task_numa_placement(struct task_struct *p) 2839 { 2840 int seq, nid, max_nid = NUMA_NO_NODE; 2841 unsigned long max_faults = 0; 2842 unsigned long fault_types[2] = { 0, 0 }; 2843 unsigned long total_faults; 2844 u64 runtime, period; 2845 spinlock_t *group_lock = NULL; 2846 struct numa_group *ng; 2847 2848 /* 2849 * The p->mm->numa_scan_seq field gets updated without 2850 * exclusive access. Use READ_ONCE() here to ensure 2851 * that the field is read in a single access: 2852 */ 2853 seq = READ_ONCE(p->mm->numa_scan_seq); 2854 if (p->numa_scan_seq == seq) 2855 return; 2856 p->numa_scan_seq = seq; 2857 p->numa_scan_period_max = task_scan_max(p); 2858 2859 total_faults = p->numa_faults_locality[0] + 2860 p->numa_faults_locality[1]; 2861 runtime = numa_get_avg_runtime(p, &period); 2862 2863 /* If the task is part of a group prevent parallel updates to group stats */ 2864 ng = deref_curr_numa_group(p); 2865 if (ng) { 2866 group_lock = &ng->lock; 2867 spin_lock_irq(group_lock); 2868 } 2869 2870 /* Find the node with the highest number of faults */ 2871 for_each_online_node(nid) { 2872 /* Keep track of the offsets in numa_faults array */ 2873 int mem_idx, membuf_idx, cpu_idx, cpubuf_idx; 2874 unsigned long faults = 0, group_faults = 0; 2875 int priv; 2876 2877 for (priv = 0; priv < NR_NUMA_HINT_FAULT_TYPES; priv++) { 2878 long diff, f_diff, f_weight; 2879 2880 mem_idx = task_faults_idx(NUMA_MEM, nid, priv); 2881 membuf_idx = task_faults_idx(NUMA_MEMBUF, nid, priv); 2882 cpu_idx = task_faults_idx(NUMA_CPU, nid, priv); 2883 cpubuf_idx = task_faults_idx(NUMA_CPUBUF, nid, priv); 2884 2885 /* Decay existing window, copy faults since last scan */ 2886 diff = p->numa_faults[membuf_idx] - p->numa_faults[mem_idx] / 2; 2887 fault_types[priv] += p->numa_faults[membuf_idx]; 2888 p->numa_faults[membuf_idx] = 0; 2889 2890 /* 2891 * Normalize the faults_from, so all tasks in a group 2892 * count according to CPU use, instead of by the raw 2893 * number of faults. Tasks with little runtime have 2894 * little over-all impact on throughput, and thus their 2895 * faults are less important. 2896 */ 2897 f_weight = div64_u64(runtime << 16, period + 1); 2898 f_weight = (f_weight * p->numa_faults[cpubuf_idx]) / 2899 (total_faults + 1); 2900 f_diff = f_weight - p->numa_faults[cpu_idx] / 2; 2901 p->numa_faults[cpubuf_idx] = 0; 2902 2903 p->numa_faults[mem_idx] += diff; 2904 p->numa_faults[cpu_idx] += f_diff; 2905 faults += p->numa_faults[mem_idx]; 2906 p->total_numa_faults += diff; 2907 if (ng) { 2908 /* 2909 * safe because we can only change our own group 2910 * 2911 * mem_idx represents the offset for a given 2912 * nid and priv in a specific region because it 2913 * is at the beginning of the numa_faults array. 2914 */ 2915 ng->faults[mem_idx] += diff; 2916 ng->faults[cpu_idx] += f_diff; 2917 ng->total_faults += diff; 2918 group_faults += ng->faults[mem_idx]; 2919 } 2920 } 2921 2922 if (!ng) { 2923 if (faults > max_faults) { 2924 max_faults = faults; 2925 max_nid = nid; 2926 } 2927 } else if (group_faults > max_faults) { 2928 max_faults = group_faults; 2929 max_nid = nid; 2930 } 2931 } 2932 2933 /* Cannot migrate task to CPU-less node */ 2934 if (max_nid != NUMA_NO_NODE && !node_state(max_nid, N_CPU)) { 2935 int near_nid = max_nid; 2936 int distance, near_distance = INT_MAX; 2937 2938 for_each_node_state(nid, N_CPU) { 2939 distance = node_distance(max_nid, nid); 2940 if (distance < near_distance) { 2941 near_nid = nid; 2942 near_distance = distance; 2943 } 2944 } 2945 max_nid = near_nid; 2946 } 2947 2948 if (ng) { 2949 numa_group_count_active_nodes(ng); 2950 spin_unlock_irq(group_lock); 2951 max_nid = preferred_group_nid(p, max_nid); 2952 } 2953 2954 if (max_faults) { 2955 /* Set the new preferred node */ 2956 if (max_nid != p->numa_preferred_nid) 2957 sched_setnuma(p, max_nid); 2958 } 2959 2960 update_task_scan_period(p, fault_types[0], fault_types[1]); 2961 } 2962 2963 static inline int get_numa_group(struct numa_group *grp) 2964 { 2965 return refcount_inc_not_zero(&grp->refcount); 2966 } 2967 2968 static inline void put_numa_group(struct numa_group *grp) 2969 { 2970 if (refcount_dec_and_test(&grp->refcount)) 2971 kfree_rcu(grp, rcu); 2972 } 2973 2974 static void task_numa_group(struct task_struct *p, int cpupid, int flags, 2975 int *priv) 2976 { 2977 struct numa_group *grp, *my_grp; 2978 struct task_struct *tsk; 2979 bool join = false; 2980 int cpu = cpupid_to_cpu(cpupid); 2981 int i; 2982 2983 if (unlikely(!deref_curr_numa_group(p))) { 2984 unsigned int size = sizeof(struct numa_group) + 2985 NR_NUMA_HINT_FAULT_STATS * 2986 nr_node_ids * sizeof(unsigned long); 2987 2988 grp = kzalloc(size, GFP_KERNEL | __GFP_NOWARN); 2989 if (!grp) 2990 return; 2991 2992 refcount_set(&grp->refcount, 1); 2993 grp->active_nodes = 1; 2994 grp->max_faults_cpu = 0; 2995 spin_lock_init(&grp->lock); 2996 grp->gid = p->pid; 2997 2998 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 2999 grp->faults[i] = p->numa_faults[i]; 3000 3001 grp->total_faults = p->total_numa_faults; 3002 3003 grp->nr_tasks++; 3004 rcu_assign_pointer(p->numa_group, grp); 3005 } 3006 3007 rcu_read_lock(); 3008 tsk = READ_ONCE(cpu_rq(cpu)->curr); 3009 3010 if (!cpupid_match_pid(tsk, cpupid)) 3011 goto no_join; 3012 3013 grp = rcu_dereference(tsk->numa_group); 3014 if (!grp) 3015 goto no_join; 3016 3017 my_grp = deref_curr_numa_group(p); 3018 if (grp == my_grp) 3019 goto no_join; 3020 3021 /* 3022 * Only join the other group if its bigger; if we're the bigger group, 3023 * the other task will join us. 3024 */ 3025 if (my_grp->nr_tasks > grp->nr_tasks) 3026 goto no_join; 3027 3028 /* 3029 * Tie-break on the grp address. 3030 */ 3031 if (my_grp->nr_tasks == grp->nr_tasks && my_grp > grp) 3032 goto no_join; 3033 3034 /* Always join threads in the same process. */ 3035 if (tsk->mm == current->mm) 3036 join = true; 3037 3038 /* Simple filter to avoid false positives due to PID collisions */ 3039 if (flags & TNF_SHARED) 3040 join = true; 3041 3042 /* Update priv based on whether false sharing was detected */ 3043 *priv = !join; 3044 3045 if (join && !get_numa_group(grp)) 3046 goto no_join; 3047 3048 rcu_read_unlock(); 3049 3050 if (!join) 3051 return; 3052 3053 WARN_ON_ONCE(irqs_disabled()); 3054 double_lock_irq(&my_grp->lock, &grp->lock); 3055 3056 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) { 3057 my_grp->faults[i] -= p->numa_faults[i]; 3058 grp->faults[i] += p->numa_faults[i]; 3059 } 3060 my_grp->total_faults -= p->total_numa_faults; 3061 grp->total_faults += p->total_numa_faults; 3062 3063 my_grp->nr_tasks--; 3064 grp->nr_tasks++; 3065 3066 spin_unlock(&my_grp->lock); 3067 spin_unlock_irq(&grp->lock); 3068 3069 rcu_assign_pointer(p->numa_group, grp); 3070 3071 put_numa_group(my_grp); 3072 return; 3073 3074 no_join: 3075 rcu_read_unlock(); 3076 return; 3077 } 3078 3079 /* 3080 * Get rid of NUMA statistics associated with a task (either current or dead). 3081 * If @final is set, the task is dead and has reached refcount zero, so we can 3082 * safely free all relevant data structures. Otherwise, there might be 3083 * concurrent reads from places like load balancing and procfs, and we should 3084 * reset the data back to default state without freeing ->numa_faults. 3085 */ 3086 void task_numa_free(struct task_struct *p, bool final) 3087 { 3088 /* safe: p either is current or is being freed by current */ 3089 struct numa_group *grp = rcu_dereference_raw(p->numa_group); 3090 unsigned long *numa_faults = p->numa_faults; 3091 unsigned long flags; 3092 int i; 3093 3094 if (!numa_faults) 3095 return; 3096 3097 if (grp) { 3098 spin_lock_irqsave(&grp->lock, flags); 3099 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 3100 grp->faults[i] -= p->numa_faults[i]; 3101 grp->total_faults -= p->total_numa_faults; 3102 3103 grp->nr_tasks--; 3104 spin_unlock_irqrestore(&grp->lock, flags); 3105 RCU_INIT_POINTER(p->numa_group, NULL); 3106 put_numa_group(grp); 3107 } 3108 3109 if (final) { 3110 p->numa_faults = NULL; 3111 kfree(numa_faults); 3112 } else { 3113 p->total_numa_faults = 0; 3114 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 3115 numa_faults[i] = 0; 3116 } 3117 } 3118 3119 /* 3120 * Got a PROT_NONE fault for a page on @node. 3121 */ 3122 void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags) 3123 { 3124 struct task_struct *p = current; 3125 bool migrated = flags & TNF_MIGRATED; 3126 int cpu_node = task_node(current); 3127 int local = !!(flags & TNF_FAULT_LOCAL); 3128 struct numa_group *ng; 3129 int priv; 3130 3131 if (!static_branch_likely(&sched_numa_balancing)) 3132 return; 3133 3134 /* for example, ksmd faulting in a user's mm */ 3135 if (!p->mm) 3136 return; 3137 3138 /* 3139 * NUMA faults statistics are unnecessary for the slow memory 3140 * node for memory tiering mode. 3141 */ 3142 if (!node_is_toptier(mem_node) && 3143 (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING || 3144 !cpupid_valid(last_cpupid))) 3145 return; 3146 3147 /* Allocate buffer to track faults on a per-node basis */ 3148 if (unlikely(!p->numa_faults)) { 3149 int size = sizeof(*p->numa_faults) * 3150 NR_NUMA_HINT_FAULT_BUCKETS * nr_node_ids; 3151 3152 p->numa_faults = kzalloc(size, GFP_KERNEL|__GFP_NOWARN); 3153 if (!p->numa_faults) 3154 return; 3155 3156 p->total_numa_faults = 0; 3157 memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); 3158 } 3159 3160 /* 3161 * First accesses are treated as private, otherwise consider accesses 3162 * to be private if the accessing pid has not changed 3163 */ 3164 if (unlikely(last_cpupid == (-1 & LAST_CPUPID_MASK))) { 3165 priv = 1; 3166 } else { 3167 priv = cpupid_match_pid(p, last_cpupid); 3168 if (!priv && !(flags & TNF_NO_GROUP)) 3169 task_numa_group(p, last_cpupid, flags, &priv); 3170 } 3171 3172 /* 3173 * If a workload spans multiple NUMA nodes, a shared fault that 3174 * occurs wholly within the set of nodes that the workload is 3175 * actively using should be counted as local. This allows the 3176 * scan rate to slow down when a workload has settled down. 3177 */ 3178 ng = deref_curr_numa_group(p); 3179 if (!priv && !local && ng && ng->active_nodes > 1 && 3180 numa_is_active_node(cpu_node, ng) && 3181 numa_is_active_node(mem_node, ng)) 3182 local = 1; 3183 3184 /* 3185 * Retry to migrate task to preferred node periodically, in case it 3186 * previously failed, or the scheduler moved us. 3187 */ 3188 if (time_after(jiffies, p->numa_migrate_retry)) { 3189 task_numa_placement(p); 3190 numa_migrate_preferred(p); 3191 } 3192 3193 if (migrated) 3194 p->numa_pages_migrated += pages; 3195 if (flags & TNF_MIGRATE_FAIL) 3196 p->numa_faults_locality[2] += pages; 3197 3198 p->numa_faults[task_faults_idx(NUMA_MEMBUF, mem_node, priv)] += pages; 3199 p->numa_faults[task_faults_idx(NUMA_CPUBUF, cpu_node, priv)] += pages; 3200 p->numa_faults_locality[local] += pages; 3201 } 3202 3203 static void reset_ptenuma_scan(struct task_struct *p) 3204 { 3205 /* 3206 * We only did a read acquisition of the mmap sem, so 3207 * p->mm->numa_scan_seq is written to without exclusive access 3208 * and the update is not guaranteed to be atomic. That's not 3209 * much of an issue though, since this is just used for 3210 * statistical sampling. Use READ_ONCE/WRITE_ONCE, which are not 3211 * expensive, to avoid any form of compiler optimizations: 3212 */ 3213 WRITE_ONCE(p->mm->numa_scan_seq, READ_ONCE(p->mm->numa_scan_seq) + 1); 3214 p->mm->numa_scan_offset = 0; 3215 } 3216 3217 static bool vma_is_accessed(struct mm_struct *mm, struct vm_area_struct *vma) 3218 { 3219 unsigned long pids; 3220 /* 3221 * Allow unconditional access first two times, so that all the (pages) 3222 * of VMAs get prot_none fault introduced irrespective of accesses. 3223 * This is also done to avoid any side effect of task scanning 3224 * amplifying the unfairness of disjoint set of VMAs' access. 3225 */ 3226 if ((READ_ONCE(current->mm->numa_scan_seq) - vma->numab_state->start_scan_seq) < 2) 3227 return true; 3228 3229 pids = vma->numab_state->pids_active[0] | vma->numab_state->pids_active[1]; 3230 if (test_bit(hash_32(current->pid, ilog2(BITS_PER_LONG)), &pids)) 3231 return true; 3232 3233 /* 3234 * Complete a scan that has already started regardless of PID access, or 3235 * some VMAs may never be scanned in multi-threaded applications: 3236 */ 3237 if (mm->numa_scan_offset > vma->vm_start) { 3238 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_IGNORE_PID); 3239 return true; 3240 } 3241 3242 /* 3243 * This vma has not been accessed for a while, and if the number 3244 * the threads in the same process is low, which means no other 3245 * threads can help scan this vma, force a vma scan. 3246 */ 3247 if (READ_ONCE(mm->numa_scan_seq) > 3248 (vma->numab_state->prev_scan_seq + get_nr_threads(current))) 3249 return true; 3250 3251 return false; 3252 } 3253 3254 #define VMA_PID_RESET_PERIOD (4 * sysctl_numa_balancing_scan_delay) 3255 3256 /* 3257 * The expensive part of numa migration is done from task_work context. 3258 * Triggered from task_tick_numa(). 3259 */ 3260 static void task_numa_work(struct callback_head *work) 3261 { 3262 unsigned long migrate, next_scan, now = jiffies; 3263 struct task_struct *p = current; 3264 struct mm_struct *mm = p->mm; 3265 u64 runtime = p->se.sum_exec_runtime; 3266 struct vm_area_struct *vma; 3267 unsigned long start, end; 3268 unsigned long nr_pte_updates = 0; 3269 long pages, virtpages; 3270 struct vma_iterator vmi; 3271 bool vma_pids_skipped; 3272 bool vma_pids_forced = false; 3273 3274 SCHED_WARN_ON(p != container_of(work, struct task_struct, numa_work)); 3275 3276 work->next = work; 3277 /* 3278 * Who cares about NUMA placement when they're dying. 3279 * 3280 * NOTE: make sure not to dereference p->mm before this check, 3281 * exit_task_work() happens _after_ exit_mm() so we could be called 3282 * without p->mm even though we still had it when we enqueued this 3283 * work. 3284 */ 3285 if (p->flags & PF_EXITING) 3286 return; 3287 3288 if (!mm->numa_next_scan) { 3289 mm->numa_next_scan = now + 3290 msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 3291 } 3292 3293 /* 3294 * Enforce maximal scan/migration frequency.. 3295 */ 3296 migrate = mm->numa_next_scan; 3297 if (time_before(now, migrate)) 3298 return; 3299 3300 if (p->numa_scan_period == 0) { 3301 p->numa_scan_period_max = task_scan_max(p); 3302 p->numa_scan_period = task_scan_start(p); 3303 } 3304 3305 next_scan = now + msecs_to_jiffies(p->numa_scan_period); 3306 if (!try_cmpxchg(&mm->numa_next_scan, &migrate, next_scan)) 3307 return; 3308 3309 /* 3310 * Delay this task enough that another task of this mm will likely win 3311 * the next time around. 3312 */ 3313 p->node_stamp += 2 * TICK_NSEC; 3314 3315 pages = sysctl_numa_balancing_scan_size; 3316 pages <<= 20 - PAGE_SHIFT; /* MB in pages */ 3317 virtpages = pages * 8; /* Scan up to this much virtual space */ 3318 if (!pages) 3319 return; 3320 3321 3322 if (!mmap_read_trylock(mm)) 3323 return; 3324 3325 /* 3326 * VMAs are skipped if the current PID has not trapped a fault within 3327 * the VMA recently. Allow scanning to be forced if there is no 3328 * suitable VMA remaining. 3329 */ 3330 vma_pids_skipped = false; 3331 3332 retry_pids: 3333 start = mm->numa_scan_offset; 3334 vma_iter_init(&vmi, mm, start); 3335 vma = vma_next(&vmi); 3336 if (!vma) { 3337 reset_ptenuma_scan(p); 3338 start = 0; 3339 vma_iter_set(&vmi, start); 3340 vma = vma_next(&vmi); 3341 } 3342 3343 for (; vma; vma = vma_next(&vmi)) { 3344 if (!vma_migratable(vma) || !vma_policy_mof(vma) || 3345 is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) { 3346 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_UNSUITABLE); 3347 continue; 3348 } 3349 3350 /* 3351 * Shared library pages mapped by multiple processes are not 3352 * migrated as it is expected they are cache replicated. Avoid 3353 * hinting faults in read-only file-backed mappings or the vdso 3354 * as migrating the pages will be of marginal benefit. 3355 */ 3356 if (!vma->vm_mm || 3357 (vma->vm_file && (vma->vm_flags & (VM_READ|VM_WRITE)) == (VM_READ))) { 3358 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_SHARED_RO); 3359 continue; 3360 } 3361 3362 /* 3363 * Skip inaccessible VMAs to avoid any confusion between 3364 * PROT_NONE and NUMA hinting ptes 3365 */ 3366 if (!vma_is_accessible(vma)) { 3367 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_INACCESSIBLE); 3368 continue; 3369 } 3370 3371 /* Initialise new per-VMA NUMAB state. */ 3372 if (!vma->numab_state) { 3373 struct vma_numab_state *ptr; 3374 3375 ptr = kzalloc(sizeof(*ptr), GFP_KERNEL); 3376 if (!ptr) 3377 continue; 3378 3379 if (cmpxchg(&vma->numab_state, NULL, ptr)) { 3380 kfree(ptr); 3381 continue; 3382 } 3383 3384 vma->numab_state->start_scan_seq = mm->numa_scan_seq; 3385 3386 vma->numab_state->next_scan = now + 3387 msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 3388 3389 /* Reset happens after 4 times scan delay of scan start */ 3390 vma->numab_state->pids_active_reset = vma->numab_state->next_scan + 3391 msecs_to_jiffies(VMA_PID_RESET_PERIOD); 3392 3393 /* 3394 * Ensure prev_scan_seq does not match numa_scan_seq, 3395 * to prevent VMAs being skipped prematurely on the 3396 * first scan: 3397 */ 3398 vma->numab_state->prev_scan_seq = mm->numa_scan_seq - 1; 3399 } 3400 3401 /* 3402 * Scanning the VMA's of short lived tasks add more overhead. So 3403 * delay the scan for new VMAs. 3404 */ 3405 if (mm->numa_scan_seq && time_before(jiffies, 3406 vma->numab_state->next_scan)) { 3407 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_SCAN_DELAY); 3408 continue; 3409 } 3410 3411 /* RESET access PIDs regularly for old VMAs. */ 3412 if (mm->numa_scan_seq && 3413 time_after(jiffies, vma->numab_state->pids_active_reset)) { 3414 vma->numab_state->pids_active_reset = vma->numab_state->pids_active_reset + 3415 msecs_to_jiffies(VMA_PID_RESET_PERIOD); 3416 vma->numab_state->pids_active[0] = READ_ONCE(vma->numab_state->pids_active[1]); 3417 vma->numab_state->pids_active[1] = 0; 3418 } 3419 3420 /* Do not rescan VMAs twice within the same sequence. */ 3421 if (vma->numab_state->prev_scan_seq == mm->numa_scan_seq) { 3422 mm->numa_scan_offset = vma->vm_end; 3423 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_SEQ_COMPLETED); 3424 continue; 3425 } 3426 3427 /* 3428 * Do not scan the VMA if task has not accessed it, unless no other 3429 * VMA candidate exists. 3430 */ 3431 if (!vma_pids_forced && !vma_is_accessed(mm, vma)) { 3432 vma_pids_skipped = true; 3433 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_PID_INACTIVE); 3434 continue; 3435 } 3436 3437 do { 3438 start = max(start, vma->vm_start); 3439 end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE); 3440 end = min(end, vma->vm_end); 3441 nr_pte_updates = change_prot_numa(vma, start, end); 3442 3443 /* 3444 * Try to scan sysctl_numa_balancing_size worth of 3445 * hpages that have at least one present PTE that 3446 * is not already pte-numa. If the VMA contains 3447 * areas that are unused or already full of prot_numa 3448 * PTEs, scan up to virtpages, to skip through those 3449 * areas faster. 3450 */ 3451 if (nr_pte_updates) 3452 pages -= (end - start) >> PAGE_SHIFT; 3453 virtpages -= (end - start) >> PAGE_SHIFT; 3454 3455 start = end; 3456 if (pages <= 0 || virtpages <= 0) 3457 goto out; 3458 3459 cond_resched(); 3460 } while (end != vma->vm_end); 3461 3462 /* VMA scan is complete, do not scan until next sequence. */ 3463 vma->numab_state->prev_scan_seq = mm->numa_scan_seq; 3464 3465 /* 3466 * Only force scan within one VMA at a time, to limit the 3467 * cost of scanning a potentially uninteresting VMA. 3468 */ 3469 if (vma_pids_forced) 3470 break; 3471 } 3472 3473 /* 3474 * If no VMAs are remaining and VMAs were skipped due to the PID 3475 * not accessing the VMA previously, then force a scan to ensure 3476 * forward progress: 3477 */ 3478 if (!vma && !vma_pids_forced && vma_pids_skipped) { 3479 vma_pids_forced = true; 3480 goto retry_pids; 3481 } 3482 3483 out: 3484 /* 3485 * It is possible to reach the end of the VMA list but the last few 3486 * VMAs are not guaranteed to the vma_migratable. If they are not, we 3487 * would find the !migratable VMA on the next scan but not reset the 3488 * scanner to the start so check it now. 3489 */ 3490 if (vma) 3491 mm->numa_scan_offset = start; 3492 else 3493 reset_ptenuma_scan(p); 3494 mmap_read_unlock(mm); 3495 3496 /* 3497 * Make sure tasks use at least 32x as much time to run other code 3498 * than they used here, to limit NUMA PTE scanning overhead to 3% max. 3499 * Usually update_task_scan_period slows down scanning enough; on an 3500 * overloaded system we need to limit overhead on a per task basis. 3501 */ 3502 if (unlikely(p->se.sum_exec_runtime != runtime)) { 3503 u64 diff = p->se.sum_exec_runtime - runtime; 3504 p->node_stamp += 32 * diff; 3505 } 3506 } 3507 3508 void init_numa_balancing(unsigned long clone_flags, struct task_struct *p) 3509 { 3510 int mm_users = 0; 3511 struct mm_struct *mm = p->mm; 3512 3513 if (mm) { 3514 mm_users = atomic_read(&mm->mm_users); 3515 if (mm_users == 1) { 3516 mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 3517 mm->numa_scan_seq = 0; 3518 } 3519 } 3520 p->node_stamp = 0; 3521 p->numa_scan_seq = mm ? mm->numa_scan_seq : 0; 3522 p->numa_scan_period = sysctl_numa_balancing_scan_delay; 3523 p->numa_migrate_retry = 0; 3524 /* Protect against double add, see task_tick_numa and task_numa_work */ 3525 p->numa_work.next = &p->numa_work; 3526 p->numa_faults = NULL; 3527 p->numa_pages_migrated = 0; 3528 p->total_numa_faults = 0; 3529 RCU_INIT_POINTER(p->numa_group, NULL); 3530 p->last_task_numa_placement = 0; 3531 p->last_sum_exec_runtime = 0; 3532 3533 init_task_work(&p->numa_work, task_numa_work); 3534 3535 /* New address space, reset the preferred nid */ 3536 if (!(clone_flags & CLONE_VM)) { 3537 p->numa_preferred_nid = NUMA_NO_NODE; 3538 return; 3539 } 3540 3541 /* 3542 * New thread, keep existing numa_preferred_nid which should be copied 3543 * already by arch_dup_task_struct but stagger when scans start. 3544 */ 3545 if (mm) { 3546 unsigned int delay; 3547 3548 delay = min_t(unsigned int, task_scan_max(current), 3549 current->numa_scan_period * mm_users * NSEC_PER_MSEC); 3550 delay += 2 * TICK_NSEC; 3551 p->node_stamp = delay; 3552 } 3553 } 3554 3555 /* 3556 * Drive the periodic memory faults.. 3557 */ 3558 static void task_tick_numa(struct rq *rq, struct task_struct *curr) 3559 { 3560 struct callback_head *work = &curr->numa_work; 3561 u64 period, now; 3562 3563 /* 3564 * We don't care about NUMA placement if we don't have memory. 3565 */ 3566 if (!curr->mm || (curr->flags & (PF_EXITING | PF_KTHREAD)) || work->next != work) 3567 return; 3568 3569 /* 3570 * Using runtime rather than walltime has the dual advantage that 3571 * we (mostly) drive the selection from busy threads and that the 3572 * task needs to have done some actual work before we bother with 3573 * NUMA placement. 3574 */ 3575 now = curr->se.sum_exec_runtime; 3576 period = (u64)curr->numa_scan_period * NSEC_PER_MSEC; 3577 3578 if (now > curr->node_stamp + period) { 3579 if (!curr->node_stamp) 3580 curr->numa_scan_period = task_scan_start(curr); 3581 curr->node_stamp += period; 3582 3583 if (!time_before(jiffies, curr->mm->numa_next_scan)) 3584 task_work_add(curr, work, TWA_RESUME); 3585 } 3586 } 3587 3588 static void update_scan_period(struct task_struct *p, int new_cpu) 3589 { 3590 int src_nid = cpu_to_node(task_cpu(p)); 3591 int dst_nid = cpu_to_node(new_cpu); 3592 3593 if (!static_branch_likely(&sched_numa_balancing)) 3594 return; 3595 3596 if (!p->mm || !p->numa_faults || (p->flags & PF_EXITING)) 3597 return; 3598 3599 if (src_nid == dst_nid) 3600 return; 3601 3602 /* 3603 * Allow resets if faults have been trapped before one scan 3604 * has completed. This is most likely due to a new task that 3605 * is pulled cross-node due to wakeups or load balancing. 3606 */ 3607 if (p->numa_scan_seq) { 3608 /* 3609 * Avoid scan adjustments if moving to the preferred 3610 * node or if the task was not previously running on 3611 * the preferred node. 3612 */ 3613 if (dst_nid == p->numa_preferred_nid || 3614 (p->numa_preferred_nid != NUMA_NO_NODE && 3615 src_nid != p->numa_preferred_nid)) 3616 return; 3617 } 3618 3619 p->numa_scan_period = task_scan_start(p); 3620 } 3621 3622 #else 3623 static void task_tick_numa(struct rq *rq, struct task_struct *curr) 3624 { 3625 } 3626 3627 static inline void account_numa_enqueue(struct rq *rq, struct task_struct *p) 3628 { 3629 } 3630 3631 static inline void account_numa_dequeue(struct rq *rq, struct task_struct *p) 3632 { 3633 } 3634 3635 static inline void update_scan_period(struct task_struct *p, int new_cpu) 3636 { 3637 } 3638 3639 #endif /* CONFIG_NUMA_BALANCING */ 3640 3641 static void 3642 account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) 3643 { 3644 update_load_add(&cfs_rq->load, se->load.weight); 3645 #ifdef CONFIG_SMP 3646 if (entity_is_task(se)) { 3647 struct rq *rq = rq_of(cfs_rq); 3648 3649 account_numa_enqueue(rq, task_of(se)); 3650 list_add(&se->group_node, &rq->cfs_tasks); 3651 } 3652 #endif 3653 cfs_rq->nr_running++; 3654 if (se_is_idle(se)) 3655 cfs_rq->idle_nr_running++; 3656 } 3657 3658 static void 3659 account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) 3660 { 3661 update_load_sub(&cfs_rq->load, se->load.weight); 3662 #ifdef CONFIG_SMP 3663 if (entity_is_task(se)) { 3664 account_numa_dequeue(rq_of(cfs_rq), task_of(se)); 3665 list_del_init(&se->group_node); 3666 } 3667 #endif 3668 cfs_rq->nr_running--; 3669 if (se_is_idle(se)) 3670 cfs_rq->idle_nr_running--; 3671 } 3672 3673 /* 3674 * Signed add and clamp on underflow. 3675 * 3676 * Explicitly do a load-store to ensure the intermediate value never hits 3677 * memory. This allows lockless observations without ever seeing the negative 3678 * values. 3679 */ 3680 #define add_positive(_ptr, _val) do { \ 3681 typeof(_ptr) ptr = (_ptr); \ 3682 typeof(_val) val = (_val); \ 3683 typeof(*ptr) res, var = READ_ONCE(*ptr); \ 3684 \ 3685 res = var + val; \ 3686 \ 3687 if (val < 0 && res > var) \ 3688 res = 0; \ 3689 \ 3690 WRITE_ONCE(*ptr, res); \ 3691 } while (0) 3692 3693 /* 3694 * Unsigned subtract and clamp on underflow. 3695 * 3696 * Explicitly do a load-store to ensure the intermediate value never hits 3697 * memory. This allows lockless observations without ever seeing the negative 3698 * values. 3699 */ 3700 #define sub_positive(_ptr, _val) do { \ 3701 typeof(_ptr) ptr = (_ptr); \ 3702 typeof(*ptr) val = (_val); \ 3703 typeof(*ptr) res, var = READ_ONCE(*ptr); \ 3704 res = var - val; \ 3705 if (res > var) \ 3706 res = 0; \ 3707 WRITE_ONCE(*ptr, res); \ 3708 } while (0) 3709 3710 /* 3711 * Remove and clamp on negative, from a local variable. 3712 * 3713 * A variant of sub_positive(), which does not use explicit load-store 3714 * and is thus optimized for local variable updates. 3715 */ 3716 #define lsub_positive(_ptr, _val) do { \ 3717 typeof(_ptr) ptr = (_ptr); \ 3718 *ptr -= min_t(typeof(*ptr), *ptr, _val); \ 3719 } while (0) 3720 3721 #ifdef CONFIG_SMP 3722 static inline void 3723 enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 3724 { 3725 cfs_rq->avg.load_avg += se->avg.load_avg; 3726 cfs_rq->avg.load_sum += se_weight(se) * se->avg.load_sum; 3727 } 3728 3729 static inline void 3730 dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 3731 { 3732 sub_positive(&cfs_rq->avg.load_avg, se->avg.load_avg); 3733 sub_positive(&cfs_rq->avg.load_sum, se_weight(se) * se->avg.load_sum); 3734 /* See update_cfs_rq_load_avg() */ 3735 cfs_rq->avg.load_sum = max_t(u32, cfs_rq->avg.load_sum, 3736 cfs_rq->avg.load_avg * PELT_MIN_DIVIDER); 3737 } 3738 #else 3739 static inline void 3740 enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { } 3741 static inline void 3742 dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { } 3743 #endif 3744 3745 static void reweight_eevdf(struct sched_entity *se, u64 avruntime, 3746 unsigned long weight) 3747 { 3748 unsigned long old_weight = se->load.weight; 3749 s64 vlag, vslice; 3750 3751 /* 3752 * VRUNTIME 3753 * ======== 3754 * 3755 * COROLLARY #1: The virtual runtime of the entity needs to be 3756 * adjusted if re-weight at !0-lag point. 3757 * 3758 * Proof: For contradiction assume this is not true, so we can 3759 * re-weight without changing vruntime at !0-lag point. 3760 * 3761 * Weight VRuntime Avg-VRuntime 3762 * before w v V 3763 * after w' v' V' 3764 * 3765 * Since lag needs to be preserved through re-weight: 3766 * 3767 * lag = (V - v)*w = (V'- v')*w', where v = v' 3768 * ==> V' = (V - v)*w/w' + v (1) 3769 * 3770 * Let W be the total weight of the entities before reweight, 3771 * since V' is the new weighted average of entities: 3772 * 3773 * V' = (WV + w'v - wv) / (W + w' - w) (2) 3774 * 3775 * by using (1) & (2) we obtain: 3776 * 3777 * (WV + w'v - wv) / (W + w' - w) = (V - v)*w/w' + v 3778 * ==> (WV-Wv+Wv+w'v-wv)/(W+w'-w) = (V - v)*w/w' + v 3779 * ==> (WV - Wv)/(W + w' - w) + v = (V - v)*w/w' + v 3780 * ==> (V - v)*W/(W + w' - w) = (V - v)*w/w' (3) 3781 * 3782 * Since we are doing at !0-lag point which means V != v, we 3783 * can simplify (3): 3784 * 3785 * ==> W / (W + w' - w) = w / w' 3786 * ==> Ww' = Ww + ww' - ww 3787 * ==> W * (w' - w) = w * (w' - w) 3788 * ==> W = w (re-weight indicates w' != w) 3789 * 3790 * So the cfs_rq contains only one entity, hence vruntime of 3791 * the entity @v should always equal to the cfs_rq's weighted 3792 * average vruntime @V, which means we will always re-weight 3793 * at 0-lag point, thus breach assumption. Proof completed. 3794 * 3795 * 3796 * COROLLARY #2: Re-weight does NOT affect weighted average 3797 * vruntime of all the entities. 3798 * 3799 * Proof: According to corollary #1, Eq. (1) should be: 3800 * 3801 * (V - v)*w = (V' - v')*w' 3802 * ==> v' = V' - (V - v)*w/w' (4) 3803 * 3804 * According to the weighted average formula, we have: 3805 * 3806 * V' = (WV - wv + w'v') / (W - w + w') 3807 * = (WV - wv + w'(V' - (V - v)w/w')) / (W - w + w') 3808 * = (WV - wv + w'V' - Vw + wv) / (W - w + w') 3809 * = (WV + w'V' - Vw) / (W - w + w') 3810 * 3811 * ==> V'*(W - w + w') = WV + w'V' - Vw 3812 * ==> V' * (W - w) = (W - w) * V (5) 3813 * 3814 * If the entity is the only one in the cfs_rq, then reweight 3815 * always occurs at 0-lag point, so V won't change. Or else 3816 * there are other entities, hence W != w, then Eq. (5) turns 3817 * into V' = V. So V won't change in either case, proof done. 3818 * 3819 * 3820 * So according to corollary #1 & #2, the effect of re-weight 3821 * on vruntime should be: 3822 * 3823 * v' = V' - (V - v) * w / w' (4) 3824 * = V - (V - v) * w / w' 3825 * = V - vl * w / w' 3826 * = V - vl' 3827 */ 3828 if (avruntime != se->vruntime) { 3829 vlag = entity_lag(avruntime, se); 3830 vlag = div_s64(vlag * old_weight, weight); 3831 se->vruntime = avruntime - vlag; 3832 } 3833 3834 /* 3835 * DEADLINE 3836 * ======== 3837 * 3838 * When the weight changes, the virtual time slope changes and 3839 * we should adjust the relative virtual deadline accordingly. 3840 * 3841 * d' = v' + (d - v)*w/w' 3842 * = V' - (V - v)*w/w' + (d - v)*w/w' 3843 * = V - (V - v)*w/w' + (d - v)*w/w' 3844 * = V + (d - V)*w/w' 3845 */ 3846 vslice = (s64)(se->deadline - avruntime); 3847 vslice = div_s64(vslice * old_weight, weight); 3848 se->deadline = avruntime + vslice; 3849 } 3850 3851 static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, 3852 unsigned long weight) 3853 { 3854 bool curr = cfs_rq->curr == se; 3855 u64 avruntime; 3856 3857 if (se->on_rq) { 3858 /* commit outstanding execution time */ 3859 update_curr(cfs_rq); 3860 avruntime = avg_vruntime(cfs_rq); 3861 if (!curr) 3862 __dequeue_entity(cfs_rq, se); 3863 update_load_sub(&cfs_rq->load, se->load.weight); 3864 } 3865 dequeue_load_avg(cfs_rq, se); 3866 3867 if (se->on_rq) { 3868 reweight_eevdf(se, avruntime, weight); 3869 } else { 3870 /* 3871 * Because we keep se->vlag = V - v_i, while: lag_i = w_i*(V - v_i), 3872 * we need to scale se->vlag when w_i changes. 3873 */ 3874 se->vlag = div_s64(se->vlag * se->load.weight, weight); 3875 } 3876 3877 update_load_set(&se->load, weight); 3878 3879 #ifdef CONFIG_SMP 3880 do { 3881 u32 divider = get_pelt_divider(&se->avg); 3882 3883 se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider); 3884 } while (0); 3885 #endif 3886 3887 enqueue_load_avg(cfs_rq, se); 3888 if (se->on_rq) { 3889 update_load_add(&cfs_rq->load, se->load.weight); 3890 if (!curr) 3891 __enqueue_entity(cfs_rq, se); 3892 3893 /* 3894 * The entity's vruntime has been adjusted, so let's check 3895 * whether the rq-wide min_vruntime needs updated too. Since 3896 * the calculations above require stable min_vruntime rather 3897 * than up-to-date one, we do the update at the end of the 3898 * reweight process. 3899 */ 3900 update_min_vruntime(cfs_rq); 3901 } 3902 } 3903 3904 void reweight_task(struct task_struct *p, const struct load_weight *lw) 3905 { 3906 struct sched_entity *se = &p->se; 3907 struct cfs_rq *cfs_rq = cfs_rq_of(se); 3908 struct load_weight *load = &se->load; 3909 3910 reweight_entity(cfs_rq, se, lw->weight); 3911 load->inv_weight = lw->inv_weight; 3912 } 3913 3914 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); 3915 3916 #ifdef CONFIG_FAIR_GROUP_SCHED 3917 #ifdef CONFIG_SMP 3918 /* 3919 * All this does is approximate the hierarchical proportion which includes that 3920 * global sum we all love to hate. 3921 * 3922 * That is, the weight of a group entity, is the proportional share of the 3923 * group weight based on the group runqueue weights. That is: 3924 * 3925 * tg->weight * grq->load.weight 3926 * ge->load.weight = ----------------------------- (1) 3927 * \Sum grq->load.weight 3928 * 3929 * Now, because computing that sum is prohibitively expensive to compute (been 3930 * there, done that) we approximate it with this average stuff. The average 3931 * moves slower and therefore the approximation is cheaper and more stable. 3932 * 3933 * So instead of the above, we substitute: 3934 * 3935 * grq->load.weight -> grq->avg.load_avg (2) 3936 * 3937 * which yields the following: 3938 * 3939 * tg->weight * grq->avg.load_avg 3940 * ge->load.weight = ------------------------------ (3) 3941 * tg->load_avg 3942 * 3943 * Where: tg->load_avg ~= \Sum grq->avg.load_avg 3944 * 3945 * That is shares_avg, and it is right (given the approximation (2)). 3946 * 3947 * The problem with it is that because the average is slow -- it was designed 3948 * to be exactly that of course -- this leads to transients in boundary 3949 * conditions. In specific, the case where the group was idle and we start the 3950 * one task. It takes time for our CPU's grq->avg.load_avg to build up, 3951 * yielding bad latency etc.. 3952 * 3953 * Now, in that special case (1) reduces to: 3954 * 3955 * tg->weight * grq->load.weight 3956 * ge->load.weight = ----------------------------- = tg->weight (4) 3957 * grp->load.weight 3958 * 3959 * That is, the sum collapses because all other CPUs are idle; the UP scenario. 3960 * 3961 * So what we do is modify our approximation (3) to approach (4) in the (near) 3962 * UP case, like: 3963 * 3964 * ge->load.weight = 3965 * 3966 * tg->weight * grq->load.weight 3967 * --------------------------------------------------- (5) 3968 * tg->load_avg - grq->avg.load_avg + grq->load.weight 3969 * 3970 * But because grq->load.weight can drop to 0, resulting in a divide by zero, 3971 * we need to use grq->avg.load_avg as its lower bound, which then gives: 3972 * 3973 * 3974 * tg->weight * grq->load.weight 3975 * ge->load.weight = ----------------------------- (6) 3976 * tg_load_avg' 3977 * 3978 * Where: 3979 * 3980 * tg_load_avg' = tg->load_avg - grq->avg.load_avg + 3981 * max(grq->load.weight, grq->avg.load_avg) 3982 * 3983 * And that is shares_weight and is icky. In the (near) UP case it approaches 3984 * (4) while in the normal case it approaches (3). It consistently 3985 * overestimates the ge->load.weight and therefore: 3986 * 3987 * \Sum ge->load.weight >= tg->weight 3988 * 3989 * hence icky! 3990 */ 3991 static long calc_group_shares(struct cfs_rq *cfs_rq) 3992 { 3993 long tg_weight, tg_shares, load, shares; 3994 struct task_group *tg = cfs_rq->tg; 3995 3996 tg_shares = READ_ONCE(tg->shares); 3997 3998 load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg); 3999 4000 tg_weight = atomic_long_read(&tg->load_avg); 4001 4002 /* Ensure tg_weight >= load */ 4003 tg_weight -= cfs_rq->tg_load_avg_contrib; 4004 tg_weight += load; 4005 4006 shares = (tg_shares * load); 4007 if (tg_weight) 4008 shares /= tg_weight; 4009 4010 /* 4011 * MIN_SHARES has to be unscaled here to support per-CPU partitioning 4012 * of a group with small tg->shares value. It is a floor value which is 4013 * assigned as a minimum load.weight to the sched_entity representing 4014 * the group on a CPU. 4015 * 4016 * E.g. on 64-bit for a group with tg->shares of scale_load(15)=15*1024 4017 * on an 8-core system with 8 tasks each runnable on one CPU shares has 4018 * to be 15*1024*1/8=1920 instead of scale_load(MIN_SHARES)=2*1024. In 4019 * case no task is runnable on a CPU MIN_SHARES=2 should be returned 4020 * instead of 0. 4021 */ 4022 return clamp_t(long, shares, MIN_SHARES, tg_shares); 4023 } 4024 #endif /* CONFIG_SMP */ 4025 4026 /* 4027 * Recomputes the group entity based on the current state of its group 4028 * runqueue. 4029 */ 4030 static void update_cfs_group(struct sched_entity *se) 4031 { 4032 struct cfs_rq *gcfs_rq = group_cfs_rq(se); 4033 long shares; 4034 4035 if (!gcfs_rq) 4036 return; 4037 4038 if (throttled_hierarchy(gcfs_rq)) 4039 return; 4040 4041 #ifndef CONFIG_SMP 4042 shares = READ_ONCE(gcfs_rq->tg->shares); 4043 #else 4044 shares = calc_group_shares(gcfs_rq); 4045 #endif 4046 if (unlikely(se->load.weight != shares)) 4047 reweight_entity(cfs_rq_of(se), se, shares); 4048 } 4049 4050 #else /* CONFIG_FAIR_GROUP_SCHED */ 4051 static inline void update_cfs_group(struct sched_entity *se) 4052 { 4053 } 4054 #endif /* CONFIG_FAIR_GROUP_SCHED */ 4055 4056 static inline void cfs_rq_util_change(struct cfs_rq *cfs_rq, int flags) 4057 { 4058 struct rq *rq = rq_of(cfs_rq); 4059 4060 if (&rq->cfs == cfs_rq) { 4061 /* 4062 * There are a few boundary cases this might miss but it should 4063 * get called often enough that that should (hopefully) not be 4064 * a real problem. 4065 * 4066 * It will not get called when we go idle, because the idle 4067 * thread is a different class (!fair), nor will the utilization 4068 * number include things like RT tasks. 4069 * 4070 * As is, the util number is not freq-invariant (we'd have to 4071 * implement arch_scale_freq_capacity() for that). 4072 * 4073 * See cpu_util_cfs(). 4074 */ 4075 cpufreq_update_util(rq, flags); 4076 } 4077 } 4078 4079 #ifdef CONFIG_SMP 4080 static inline bool load_avg_is_decayed(struct sched_avg *sa) 4081 { 4082 if (sa->load_sum) 4083 return false; 4084 4085 if (sa->util_sum) 4086 return false; 4087 4088 if (sa->runnable_sum) 4089 return false; 4090 4091 /* 4092 * _avg must be null when _sum are null because _avg = _sum / divider 4093 * Make sure that rounding and/or propagation of PELT values never 4094 * break this. 4095 */ 4096 SCHED_WARN_ON(sa->load_avg || 4097 sa->util_avg || 4098 sa->runnable_avg); 4099 4100 return true; 4101 } 4102 4103 static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq) 4104 { 4105 return u64_u32_load_copy(cfs_rq->avg.last_update_time, 4106 cfs_rq->last_update_time_copy); 4107 } 4108 #ifdef CONFIG_FAIR_GROUP_SCHED 4109 /* 4110 * Because list_add_leaf_cfs_rq always places a child cfs_rq on the list 4111 * immediately before a parent cfs_rq, and cfs_rqs are removed from the list 4112 * bottom-up, we only have to test whether the cfs_rq before us on the list 4113 * is our child. 4114 * If cfs_rq is not on the list, test whether a child needs its to be added to 4115 * connect a branch to the tree * (see list_add_leaf_cfs_rq() for details). 4116 */ 4117 static inline bool child_cfs_rq_on_list(struct cfs_rq *cfs_rq) 4118 { 4119 struct cfs_rq *prev_cfs_rq; 4120 struct list_head *prev; 4121 4122 if (cfs_rq->on_list) { 4123 prev = cfs_rq->leaf_cfs_rq_list.prev; 4124 } else { 4125 struct rq *rq = rq_of(cfs_rq); 4126 4127 prev = rq->tmp_alone_branch; 4128 } 4129 4130 prev_cfs_rq = container_of(prev, struct cfs_rq, leaf_cfs_rq_list); 4131 4132 return (prev_cfs_rq->tg->parent == cfs_rq->tg); 4133 } 4134 4135 static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq) 4136 { 4137 if (cfs_rq->load.weight) 4138 return false; 4139 4140 if (!load_avg_is_decayed(&cfs_rq->avg)) 4141 return false; 4142 4143 if (child_cfs_rq_on_list(cfs_rq)) 4144 return false; 4145 4146 return true; 4147 } 4148 4149 /** 4150 * update_tg_load_avg - update the tg's load avg 4151 * @cfs_rq: the cfs_rq whose avg changed 4152 * 4153 * This function 'ensures': tg->load_avg := \Sum tg->cfs_rq[]->avg.load. 4154 * However, because tg->load_avg is a global value there are performance 4155 * considerations. 4156 * 4157 * In order to avoid having to look at the other cfs_rq's, we use a 4158 * differential update where we store the last value we propagated. This in 4159 * turn allows skipping updates if the differential is 'small'. 4160 * 4161 * Updating tg's load_avg is necessary before update_cfs_share(). 4162 */ 4163 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) 4164 { 4165 long delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; 4166 4167 /* 4168 * No need to update load_avg for root_task_group as it is not used. 4169 */ 4170 if (cfs_rq->tg == &root_task_group) 4171 return; 4172 4173 if (abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { 4174 atomic_long_add(delta, &cfs_rq->tg->load_avg); 4175 cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; 4176 } 4177 } 4178 4179 /* 4180 * Called within set_task_rq() right before setting a task's CPU. The 4181 * caller only guarantees p->pi_lock is held; no other assumptions, 4182 * including the state of rq->lock, should be made. 4183 */ 4184 void set_task_rq_fair(struct sched_entity *se, 4185 struct cfs_rq *prev, struct cfs_rq *next) 4186 { 4187 u64 p_last_update_time; 4188 u64 n_last_update_time; 4189 4190 if (!sched_feat(ATTACH_AGE_LOAD)) 4191 return; 4192 4193 /* 4194 * We are supposed to update the task to "current" time, then its up to 4195 * date and ready to go to new CPU/cfs_rq. But we have difficulty in 4196 * getting what current time is, so simply throw away the out-of-date 4197 * time. This will result in the wakee task is less decayed, but giving 4198 * the wakee more load sounds not bad. 4199 */ 4200 if (!(se->avg.last_update_time && prev)) 4201 return; 4202 4203 p_last_update_time = cfs_rq_last_update_time(prev); 4204 n_last_update_time = cfs_rq_last_update_time(next); 4205 4206 __update_load_avg_blocked_se(p_last_update_time, se); 4207 se->avg.last_update_time = n_last_update_time; 4208 } 4209 4210 /* 4211 * When on migration a sched_entity joins/leaves the PELT hierarchy, we need to 4212 * propagate its contribution. The key to this propagation is the invariant 4213 * that for each group: 4214 * 4215 * ge->avg == grq->avg (1) 4216 * 4217 * _IFF_ we look at the pure running and runnable sums. Because they 4218 * represent the very same entity, just at different points in the hierarchy. 4219 * 4220 * Per the above update_tg_cfs_util() and update_tg_cfs_runnable() are trivial 4221 * and simply copies the running/runnable sum over (but still wrong, because 4222 * the group entity and group rq do not have their PELT windows aligned). 4223 * 4224 * However, update_tg_cfs_load() is more complex. So we have: 4225 * 4226 * ge->avg.load_avg = ge->load.weight * ge->avg.runnable_avg (2) 4227 * 4228 * And since, like util, the runnable part should be directly transferable, 4229 * the following would _appear_ to be the straight forward approach: 4230 * 4231 * grq->avg.load_avg = grq->load.weight * grq->avg.runnable_avg (3) 4232 * 4233 * And per (1) we have: 4234 * 4235 * ge->avg.runnable_avg == grq->avg.runnable_avg 4236 * 4237 * Which gives: 4238 * 4239 * ge->load.weight * grq->avg.load_avg 4240 * ge->avg.load_avg = ----------------------------------- (4) 4241 * grq->load.weight 4242 * 4243 * Except that is wrong! 4244 * 4245 * Because while for entities historical weight is not important and we 4246 * really only care about our future and therefore can consider a pure 4247 * runnable sum, runqueues can NOT do this. 4248 * 4249 * We specifically want runqueues to have a load_avg that includes 4250 * historical weights. Those represent the blocked load, the load we expect 4251 * to (shortly) return to us. This only works by keeping the weights as 4252 * integral part of the sum. We therefore cannot decompose as per (3). 4253 * 4254 * Another reason this doesn't work is that runnable isn't a 0-sum entity. 4255 * Imagine a rq with 2 tasks that each are runnable 2/3 of the time. Then the 4256 * rq itself is runnable anywhere between 2/3 and 1 depending on how the 4257 * runnable section of these tasks overlap (or not). If they were to perfectly 4258 * align the rq as a whole would be runnable 2/3 of the time. If however we 4259 * always have at least 1 runnable task, the rq as a whole is always runnable. 4260 * 4261 * So we'll have to approximate.. :/ 4262 * 4263 * Given the constraint: 4264 * 4265 * ge->avg.running_sum <= ge->avg.runnable_sum <= LOAD_AVG_MAX 4266 * 4267 * We can construct a rule that adds runnable to a rq by assuming minimal 4268 * overlap. 4269 * 4270 * On removal, we'll assume each task is equally runnable; which yields: 4271 * 4272 * grq->avg.runnable_sum = grq->avg.load_sum / grq->load.weight 4273 * 4274 * XXX: only do this for the part of runnable > running ? 4275 * 4276 */ 4277 static inline void 4278 update_tg_cfs_util(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 4279 { 4280 long delta_sum, delta_avg = gcfs_rq->avg.util_avg - se->avg.util_avg; 4281 u32 new_sum, divider; 4282 4283 /* Nothing to update */ 4284 if (!delta_avg) 4285 return; 4286 4287 /* 4288 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 4289 * See ___update_load_avg() for details. 4290 */ 4291 divider = get_pelt_divider(&cfs_rq->avg); 4292 4293 4294 /* Set new sched_entity's utilization */ 4295 se->avg.util_avg = gcfs_rq->avg.util_avg; 4296 new_sum = se->avg.util_avg * divider; 4297 delta_sum = (long)new_sum - (long)se->avg.util_sum; 4298 se->avg.util_sum = new_sum; 4299 4300 /* Update parent cfs_rq utilization */ 4301 add_positive(&cfs_rq->avg.util_avg, delta_avg); 4302 add_positive(&cfs_rq->avg.util_sum, delta_sum); 4303 4304 /* See update_cfs_rq_load_avg() */ 4305 cfs_rq->avg.util_sum = max_t(u32, cfs_rq->avg.util_sum, 4306 cfs_rq->avg.util_avg * PELT_MIN_DIVIDER); 4307 } 4308 4309 static inline void 4310 update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 4311 { 4312 long delta_sum, delta_avg = gcfs_rq->avg.runnable_avg - se->avg.runnable_avg; 4313 u32 new_sum, divider; 4314 4315 /* Nothing to update */ 4316 if (!delta_avg) 4317 return; 4318 4319 /* 4320 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 4321 * See ___update_load_avg() for details. 4322 */ 4323 divider = get_pelt_divider(&cfs_rq->avg); 4324 4325 /* Set new sched_entity's runnable */ 4326 se->avg.runnable_avg = gcfs_rq->avg.runnable_avg; 4327 new_sum = se->avg.runnable_avg * divider; 4328 delta_sum = (long)new_sum - (long)se->avg.runnable_sum; 4329 se->avg.runnable_sum = new_sum; 4330 4331 /* Update parent cfs_rq runnable */ 4332 add_positive(&cfs_rq->avg.runnable_avg, delta_avg); 4333 add_positive(&cfs_rq->avg.runnable_sum, delta_sum); 4334 /* See update_cfs_rq_load_avg() */ 4335 cfs_rq->avg.runnable_sum = max_t(u32, cfs_rq->avg.runnable_sum, 4336 cfs_rq->avg.runnable_avg * PELT_MIN_DIVIDER); 4337 } 4338 4339 static inline void 4340 update_tg_cfs_load(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 4341 { 4342 long delta_avg, running_sum, runnable_sum = gcfs_rq->prop_runnable_sum; 4343 unsigned long load_avg; 4344 u64 load_sum = 0; 4345 s64 delta_sum; 4346 u32 divider; 4347 4348 if (!runnable_sum) 4349 return; 4350 4351 gcfs_rq->prop_runnable_sum = 0; 4352 4353 /* 4354 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 4355 * See ___update_load_avg() for details. 4356 */ 4357 divider = get_pelt_divider(&cfs_rq->avg); 4358 4359 if (runnable_sum >= 0) { 4360 /* 4361 * Add runnable; clip at LOAD_AVG_MAX. Reflects that until 4362 * the CPU is saturated running == runnable. 4363 */ 4364 runnable_sum += se->avg.load_sum; 4365 runnable_sum = min_t(long, runnable_sum, divider); 4366 } else { 4367 /* 4368 * Estimate the new unweighted runnable_sum of the gcfs_rq by 4369 * assuming all tasks are equally runnable. 4370 */ 4371 if (scale_load_down(gcfs_rq->load.weight)) { 4372 load_sum = div_u64(gcfs_rq->avg.load_sum, 4373 scale_load_down(gcfs_rq->load.weight)); 4374 } 4375 4376 /* But make sure to not inflate se's runnable */ 4377 runnable_sum = min(se->avg.load_sum, load_sum); 4378 } 4379 4380 /* 4381 * runnable_sum can't be lower than running_sum 4382 * Rescale running sum to be in the same range as runnable sum 4383 * running_sum is in [0 : LOAD_AVG_MAX << SCHED_CAPACITY_SHIFT] 4384 * runnable_sum is in [0 : LOAD_AVG_MAX] 4385 */ 4386 running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT; 4387 runnable_sum = max(runnable_sum, running_sum); 4388 4389 load_sum = se_weight(se) * runnable_sum; 4390 load_avg = div_u64(load_sum, divider); 4391 4392 delta_avg = load_avg - se->avg.load_avg; 4393 if (!delta_avg) 4394 return; 4395 4396 delta_sum = load_sum - (s64)se_weight(se) * se->avg.load_sum; 4397 4398 se->avg.load_sum = runnable_sum; 4399 se->avg.load_avg = load_avg; 4400 add_positive(&cfs_rq->avg.load_avg, delta_avg); 4401 add_positive(&cfs_rq->avg.load_sum, delta_sum); 4402 /* See update_cfs_rq_load_avg() */ 4403 cfs_rq->avg.load_sum = max_t(u32, cfs_rq->avg.load_sum, 4404 cfs_rq->avg.load_avg * PELT_MIN_DIVIDER); 4405 } 4406 4407 static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) 4408 { 4409 cfs_rq->propagate = 1; 4410 cfs_rq->prop_runnable_sum += runnable_sum; 4411 } 4412 4413 /* Update task and its cfs_rq load average */ 4414 static inline int propagate_entity_load_avg(struct sched_entity *se) 4415 { 4416 struct cfs_rq *cfs_rq, *gcfs_rq; 4417 4418 if (entity_is_task(se)) 4419 return 0; 4420 4421 gcfs_rq = group_cfs_rq(se); 4422 if (!gcfs_rq->propagate) 4423 return 0; 4424 4425 gcfs_rq->propagate = 0; 4426 4427 cfs_rq = cfs_rq_of(se); 4428 4429 add_tg_cfs_propagate(cfs_rq, gcfs_rq->prop_runnable_sum); 4430 4431 update_tg_cfs_util(cfs_rq, se, gcfs_rq); 4432 update_tg_cfs_runnable(cfs_rq, se, gcfs_rq); 4433 update_tg_cfs_load(cfs_rq, se, gcfs_rq); 4434 4435 trace_pelt_cfs_tp(cfs_rq); 4436 trace_pelt_se_tp(se); 4437 4438 return 1; 4439 } 4440 4441 /* 4442 * Check if we need to update the load and the utilization of a blocked 4443 * group_entity: 4444 */ 4445 static inline bool skip_blocked_update(struct sched_entity *se) 4446 { 4447 struct cfs_rq *gcfs_rq = group_cfs_rq(se); 4448 4449 /* 4450 * If sched_entity still have not zero load or utilization, we have to 4451 * decay it: 4452 */ 4453 if (se->avg.load_avg || se->avg.util_avg) 4454 return false; 4455 4456 /* 4457 * If there is a pending propagation, we have to update the load and 4458 * the utilization of the sched_entity: 4459 */ 4460 if (gcfs_rq->propagate) 4461 return false; 4462 4463 /* 4464 * Otherwise, the load and the utilization of the sched_entity is 4465 * already zero and there is no pending propagation, so it will be a 4466 * waste of time to try to decay it: 4467 */ 4468 return true; 4469 } 4470 4471 #else /* CONFIG_FAIR_GROUP_SCHED */ 4472 4473 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) {} 4474 4475 static inline int propagate_entity_load_avg(struct sched_entity *se) 4476 { 4477 return 0; 4478 } 4479 4480 static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) {} 4481 4482 #endif /* CONFIG_FAIR_GROUP_SCHED */ 4483 4484 #ifdef CONFIG_NO_HZ_COMMON 4485 static inline void migrate_se_pelt_lag(struct sched_entity *se) 4486 { 4487 u64 throttled = 0, now, lut; 4488 struct cfs_rq *cfs_rq; 4489 struct rq *rq; 4490 bool is_idle; 4491 4492 if (load_avg_is_decayed(&se->avg)) 4493 return; 4494 4495 cfs_rq = cfs_rq_of(se); 4496 rq = rq_of(cfs_rq); 4497 4498 rcu_read_lock(); 4499 is_idle = is_idle_task(rcu_dereference(rq->curr)); 4500 rcu_read_unlock(); 4501 4502 /* 4503 * The lag estimation comes with a cost we don't want to pay all the 4504 * time. Hence, limiting to the case where the source CPU is idle and 4505 * we know we are at the greatest risk to have an outdated clock. 4506 */ 4507 if (!is_idle) 4508 return; 4509 4510 /* 4511 * Estimated "now" is: last_update_time + cfs_idle_lag + rq_idle_lag, where: 4512 * 4513 * last_update_time (the cfs_rq's last_update_time) 4514 * = cfs_rq_clock_pelt()@cfs_rq_idle 4515 * = rq_clock_pelt()@cfs_rq_idle 4516 * - cfs->throttled_clock_pelt_time@cfs_rq_idle 4517 * 4518 * cfs_idle_lag (delta between rq's update and cfs_rq's update) 4519 * = rq_clock_pelt()@rq_idle - rq_clock_pelt()@cfs_rq_idle 4520 * 4521 * rq_idle_lag (delta between now and rq's update) 4522 * = sched_clock_cpu() - rq_clock()@rq_idle 4523 * 4524 * We can then write: 4525 * 4526 * now = rq_clock_pelt()@rq_idle - cfs->throttled_clock_pelt_time + 4527 * sched_clock_cpu() - rq_clock()@rq_idle 4528 * Where: 4529 * rq_clock_pelt()@rq_idle is rq->clock_pelt_idle 4530 * rq_clock()@rq_idle is rq->clock_idle 4531 * cfs->throttled_clock_pelt_time@cfs_rq_idle 4532 * is cfs_rq->throttled_pelt_idle 4533 */ 4534 4535 #ifdef CONFIG_CFS_BANDWIDTH 4536 throttled = u64_u32_load(cfs_rq->throttled_pelt_idle); 4537 /* The clock has been stopped for throttling */ 4538 if (throttled == U64_MAX) 4539 return; 4540 #endif 4541 now = u64_u32_load(rq->clock_pelt_idle); 4542 /* 4543 * Paired with _update_idle_rq_clock_pelt(). It ensures at the worst case 4544 * is observed the old clock_pelt_idle value and the new clock_idle, 4545 * which lead to an underestimation. The opposite would lead to an 4546 * overestimation. 4547 */ 4548 smp_rmb(); 4549 lut = cfs_rq_last_update_time(cfs_rq); 4550 4551 now -= throttled; 4552 if (now < lut) 4553 /* 4554 * cfs_rq->avg.last_update_time is more recent than our 4555 * estimation, let's use it. 4556 */ 4557 now = lut; 4558 else 4559 now += sched_clock_cpu(cpu_of(rq)) - u64_u32_load(rq->clock_idle); 4560 4561 __update_load_avg_blocked_se(now, se); 4562 } 4563 #else 4564 static void migrate_se_pelt_lag(struct sched_entity *se) {} 4565 #endif 4566 4567 /** 4568 * update_cfs_rq_load_avg - update the cfs_rq's load/util averages 4569 * @now: current time, as per cfs_rq_clock_pelt() 4570 * @cfs_rq: cfs_rq to update 4571 * 4572 * The cfs_rq avg is the direct sum of all its entities (blocked and runnable) 4573 * avg. The immediate corollary is that all (fair) tasks must be attached. 4574 * 4575 * cfs_rq->avg is used for task_h_load() and update_cfs_share() for example. 4576 * 4577 * Return: true if the load decayed or we removed load. 4578 * 4579 * Since both these conditions indicate a changed cfs_rq->avg.load we should 4580 * call update_tg_load_avg() when this function returns true. 4581 */ 4582 static inline int 4583 update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) 4584 { 4585 unsigned long removed_load = 0, removed_util = 0, removed_runnable = 0; 4586 struct sched_avg *sa = &cfs_rq->avg; 4587 int decayed = 0; 4588 4589 if (cfs_rq->removed.nr) { 4590 unsigned long r; 4591 u32 divider = get_pelt_divider(&cfs_rq->avg); 4592 4593 raw_spin_lock(&cfs_rq->removed.lock); 4594 swap(cfs_rq->removed.util_avg, removed_util); 4595 swap(cfs_rq->removed.load_avg, removed_load); 4596 swap(cfs_rq->removed.runnable_avg, removed_runnable); 4597 cfs_rq->removed.nr = 0; 4598 raw_spin_unlock(&cfs_rq->removed.lock); 4599 4600 r = removed_load; 4601 sub_positive(&sa->load_avg, r); 4602 sub_positive(&sa->load_sum, r * divider); 4603 /* See sa->util_sum below */ 4604 sa->load_sum = max_t(u32, sa->load_sum, sa->load_avg * PELT_MIN_DIVIDER); 4605 4606 r = removed_util; 4607 sub_positive(&sa->util_avg, r); 4608 sub_positive(&sa->util_sum, r * divider); 4609 /* 4610 * Because of rounding, se->util_sum might ends up being +1 more than 4611 * cfs->util_sum. Although this is not a problem by itself, detaching 4612 * a lot of tasks with the rounding problem between 2 updates of 4613 * util_avg (~1ms) can make cfs->util_sum becoming null whereas 4614 * cfs_util_avg is not. 4615 * Check that util_sum is still above its lower bound for the new 4616 * util_avg. Given that period_contrib might have moved since the last 4617 * sync, we are only sure that util_sum must be above or equal to 4618 * util_avg * minimum possible divider 4619 */ 4620 sa->util_sum = max_t(u32, sa->util_sum, sa->util_avg * PELT_MIN_DIVIDER); 4621 4622 r = removed_runnable; 4623 sub_positive(&sa->runnable_avg, r); 4624 sub_positive(&sa->runnable_sum, r * divider); 4625 /* See sa->util_sum above */ 4626 sa->runnable_sum = max_t(u32, sa->runnable_sum, 4627 sa->runnable_avg * PELT_MIN_DIVIDER); 4628 4629 /* 4630 * removed_runnable is the unweighted version of removed_load so we 4631 * can use it to estimate removed_load_sum. 4632 */ 4633 add_tg_cfs_propagate(cfs_rq, 4634 -(long)(removed_runnable * divider) >> SCHED_CAPACITY_SHIFT); 4635 4636 decayed = 1; 4637 } 4638 4639 decayed |= __update_load_avg_cfs_rq(now, cfs_rq); 4640 u64_u32_store_copy(sa->last_update_time, 4641 cfs_rq->last_update_time_copy, 4642 sa->last_update_time); 4643 return decayed; 4644 } 4645 4646 /** 4647 * attach_entity_load_avg - attach this entity to its cfs_rq load avg 4648 * @cfs_rq: cfs_rq to attach to 4649 * @se: sched_entity to attach 4650 * 4651 * Must call update_cfs_rq_load_avg() before this, since we rely on 4652 * cfs_rq->avg.last_update_time being current. 4653 */ 4654 static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 4655 { 4656 /* 4657 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 4658 * See ___update_load_avg() for details. 4659 */ 4660 u32 divider = get_pelt_divider(&cfs_rq->avg); 4661 4662 /* 4663 * When we attach the @se to the @cfs_rq, we must align the decay 4664 * window because without that, really weird and wonderful things can 4665 * happen. 4666 * 4667 * XXX illustrate 4668 */ 4669 se->avg.last_update_time = cfs_rq->avg.last_update_time; 4670 se->avg.period_contrib = cfs_rq->avg.period_contrib; 4671 4672 /* 4673 * Hell(o) Nasty stuff.. we need to recompute _sum based on the new 4674 * period_contrib. This isn't strictly correct, but since we're 4675 * entirely outside of the PELT hierarchy, nobody cares if we truncate 4676 * _sum a little. 4677 */ 4678 se->avg.util_sum = se->avg.util_avg * divider; 4679 4680 se->avg.runnable_sum = se->avg.runnable_avg * divider; 4681 4682 se->avg.load_sum = se->avg.load_avg * divider; 4683 if (se_weight(se) < se->avg.load_sum) 4684 se->avg.load_sum = div_u64(se->avg.load_sum, se_weight(se)); 4685 else 4686 se->avg.load_sum = 1; 4687 4688 enqueue_load_avg(cfs_rq, se); 4689 cfs_rq->avg.util_avg += se->avg.util_avg; 4690 cfs_rq->avg.util_sum += se->avg.util_sum; 4691 cfs_rq->avg.runnable_avg += se->avg.runnable_avg; 4692 cfs_rq->avg.runnable_sum += se->avg.runnable_sum; 4693 4694 add_tg_cfs_propagate(cfs_rq, se->avg.load_sum); 4695 4696 cfs_rq_util_change(cfs_rq, 0); 4697 4698 trace_pelt_cfs_tp(cfs_rq); 4699 } 4700 4701 /** 4702 * detach_entity_load_avg - detach this entity from its cfs_rq load avg 4703 * @cfs_rq: cfs_rq to detach from 4704 * @se: sched_entity to detach 4705 * 4706 * Must call update_cfs_rq_load_avg() before this, since we rely on 4707 * cfs_rq->avg.last_update_time being current. 4708 */ 4709 static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 4710 { 4711 dequeue_load_avg(cfs_rq, se); 4712 sub_positive(&cfs_rq->avg.util_avg, se->avg.util_avg); 4713 sub_positive(&cfs_rq->avg.util_sum, se->avg.util_sum); 4714 /* See update_cfs_rq_load_avg() */ 4715 cfs_rq->avg.util_sum = max_t(u32, cfs_rq->avg.util_sum, 4716 cfs_rq->avg.util_avg * PELT_MIN_DIVIDER); 4717 4718 sub_positive(&cfs_rq->avg.runnable_avg, se->avg.runnable_avg); 4719 sub_positive(&cfs_rq->avg.runnable_sum, se->avg.runnable_sum); 4720 /* See update_cfs_rq_load_avg() */ 4721 cfs_rq->avg.runnable_sum = max_t(u32, cfs_rq->avg.runnable_sum, 4722 cfs_rq->avg.runnable_avg * PELT_MIN_DIVIDER); 4723 4724 add_tg_cfs_propagate(cfs_rq, -se->avg.load_sum); 4725 4726 cfs_rq_util_change(cfs_rq, 0); 4727 4728 trace_pelt_cfs_tp(cfs_rq); 4729 } 4730 4731 /* 4732 * Optional action to be done while updating the load average 4733 */ 4734 #define UPDATE_TG 0x1 4735 #define SKIP_AGE_LOAD 0x2 4736 #define DO_ATTACH 0x4 4737 #define DO_DETACH 0x8 4738 4739 /* Update task and its cfs_rq load average */ 4740 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 4741 { 4742 u64 now = cfs_rq_clock_pelt(cfs_rq); 4743 int decayed; 4744 4745 /* 4746 * Track task load average for carrying it to new CPU after migrated, and 4747 * track group sched_entity load average for task_h_load calc in migration 4748 */ 4749 if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD)) 4750 __update_load_avg_se(now, cfs_rq, se); 4751 4752 decayed = update_cfs_rq_load_avg(now, cfs_rq); 4753 decayed |= propagate_entity_load_avg(se); 4754 4755 if (!se->avg.last_update_time && (flags & DO_ATTACH)) { 4756 4757 /* 4758 * DO_ATTACH means we're here from enqueue_entity(). 4759 * !last_update_time means we've passed through 4760 * migrate_task_rq_fair() indicating we migrated. 4761 * 4762 * IOW we're enqueueing a task on a new CPU. 4763 */ 4764 attach_entity_load_avg(cfs_rq, se); 4765 update_tg_load_avg(cfs_rq); 4766 4767 } else if (flags & DO_DETACH) { 4768 /* 4769 * DO_DETACH means we're here from dequeue_entity() 4770 * and we are migrating task out of the CPU. 4771 */ 4772 detach_entity_load_avg(cfs_rq, se); 4773 update_tg_load_avg(cfs_rq); 4774 } else if (decayed) { 4775 cfs_rq_util_change(cfs_rq, 0); 4776 4777 if (flags & UPDATE_TG) 4778 update_tg_load_avg(cfs_rq); 4779 } 4780 } 4781 4782 /* 4783 * Synchronize entity load avg of dequeued entity without locking 4784 * the previous rq. 4785 */ 4786 static void sync_entity_load_avg(struct sched_entity *se) 4787 { 4788 struct cfs_rq *cfs_rq = cfs_rq_of(se); 4789 u64 last_update_time; 4790 4791 last_update_time = cfs_rq_last_update_time(cfs_rq); 4792 __update_load_avg_blocked_se(last_update_time, se); 4793 } 4794 4795 /* 4796 * Task first catches up with cfs_rq, and then subtract 4797 * itself from the cfs_rq (task must be off the queue now). 4798 */ 4799 static void remove_entity_load_avg(struct sched_entity *se) 4800 { 4801 struct cfs_rq *cfs_rq = cfs_rq_of(se); 4802 unsigned long flags; 4803 4804 /* 4805 * tasks cannot exit without having gone through wake_up_new_task() -> 4806 * enqueue_task_fair() which will have added things to the cfs_rq, 4807 * so we can remove unconditionally. 4808 */ 4809 4810 sync_entity_load_avg(se); 4811 4812 raw_spin_lock_irqsave(&cfs_rq->removed.lock, flags); 4813 ++cfs_rq->removed.nr; 4814 cfs_rq->removed.util_avg += se->avg.util_avg; 4815 cfs_rq->removed.load_avg += se->avg.load_avg; 4816 cfs_rq->removed.runnable_avg += se->avg.runnable_avg; 4817 raw_spin_unlock_irqrestore(&cfs_rq->removed.lock, flags); 4818 } 4819 4820 static inline unsigned long cfs_rq_runnable_avg(struct cfs_rq *cfs_rq) 4821 { 4822 return cfs_rq->avg.runnable_avg; 4823 } 4824 4825 static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq) 4826 { 4827 return cfs_rq->avg.load_avg; 4828 } 4829 4830 static int newidle_balance(struct rq *this_rq, struct rq_flags *rf); 4831 4832 static inline unsigned long task_util(struct task_struct *p) 4833 { 4834 return READ_ONCE(p->se.avg.util_avg); 4835 } 4836 4837 static inline unsigned long _task_util_est(struct task_struct *p) 4838 { 4839 struct util_est ue = READ_ONCE(p->se.avg.util_est); 4840 4841 return max(ue.ewma, (ue.enqueued & ~UTIL_AVG_UNCHANGED)); 4842 } 4843 4844 static inline unsigned long task_util_est(struct task_struct *p) 4845 { 4846 return max(task_util(p), _task_util_est(p)); 4847 } 4848 4849 static inline void util_est_enqueue(struct cfs_rq *cfs_rq, 4850 struct task_struct *p) 4851 { 4852 unsigned int enqueued; 4853 4854 if (!sched_feat(UTIL_EST)) 4855 return; 4856 4857 /* Update root cfs_rq's estimated utilization */ 4858 enqueued = cfs_rq->avg.util_est.enqueued; 4859 enqueued += _task_util_est(p); 4860 WRITE_ONCE(cfs_rq->avg.util_est.enqueued, enqueued); 4861 4862 trace_sched_util_est_cfs_tp(cfs_rq); 4863 } 4864 4865 static inline void util_est_dequeue(struct cfs_rq *cfs_rq, 4866 struct task_struct *p) 4867 { 4868 unsigned int enqueued; 4869 4870 if (!sched_feat(UTIL_EST)) 4871 return; 4872 4873 /* Update root cfs_rq's estimated utilization */ 4874 enqueued = cfs_rq->avg.util_est.enqueued; 4875 enqueued -= min_t(unsigned int, enqueued, _task_util_est(p)); 4876 WRITE_ONCE(cfs_rq->avg.util_est.enqueued, enqueued); 4877 4878 trace_sched_util_est_cfs_tp(cfs_rq); 4879 } 4880 4881 #define UTIL_EST_MARGIN (SCHED_CAPACITY_SCALE / 100) 4882 4883 /* 4884 * Check if a (signed) value is within a specified (unsigned) margin, 4885 * based on the observation that: 4886 * 4887 * abs(x) < y := (unsigned)(x + y - 1) < (2 * y - 1) 4888 * 4889 * NOTE: this only works when value + margin < INT_MAX. 4890 */ 4891 static inline bool within_margin(int value, int margin) 4892 { 4893 return ((unsigned int)(value + margin - 1) < (2 * margin - 1)); 4894 } 4895 4896 static inline void util_est_update(struct cfs_rq *cfs_rq, 4897 struct task_struct *p, 4898 bool task_sleep) 4899 { 4900 long last_ewma_diff, last_enqueued_diff; 4901 struct util_est ue; 4902 4903 if (!sched_feat(UTIL_EST)) 4904 return; 4905 4906 /* 4907 * Skip update of task's estimated utilization when the task has not 4908 * yet completed an activation, e.g. being migrated. 4909 */ 4910 if (!task_sleep) 4911 return; 4912 4913 /* 4914 * If the PELT values haven't changed since enqueue time, 4915 * skip the util_est update. 4916 */ 4917 ue = p->se.avg.util_est; 4918 if (ue.enqueued & UTIL_AVG_UNCHANGED) 4919 return; 4920 4921 last_enqueued_diff = ue.enqueued; 4922 4923 /* 4924 * Reset EWMA on utilization increases, the moving average is used only 4925 * to smooth utilization decreases. 4926 */ 4927 ue.enqueued = task_util(p); 4928 if (sched_feat(UTIL_EST_FASTUP)) { 4929 if (ue.ewma < ue.enqueued) { 4930 ue.ewma = ue.enqueued; 4931 goto done; 4932 } 4933 } 4934 4935 /* 4936 * Skip update of task's estimated utilization when its members are 4937 * already ~1% close to its last activation value. 4938 */ 4939 last_ewma_diff = ue.enqueued - ue.ewma; 4940 last_enqueued_diff -= ue.enqueued; 4941 if (within_margin(last_ewma_diff, UTIL_EST_MARGIN)) { 4942 if (!within_margin(last_enqueued_diff, UTIL_EST_MARGIN)) 4943 goto done; 4944 4945 return; 4946 } 4947 4948 /* 4949 * To avoid overestimation of actual task utilization, skip updates if 4950 * we cannot grant there is idle time in this CPU. 4951 */ 4952 if (task_util(p) > capacity_orig_of(cpu_of(rq_of(cfs_rq)))) 4953 return; 4954 4955 /* 4956 * Update Task's estimated utilization 4957 * 4958 * When *p completes an activation we can consolidate another sample 4959 * of the task size. This is done by storing the current PELT value 4960 * as ue.enqueued and by using this value to update the Exponential 4961 * Weighted Moving Average (EWMA): 4962 * 4963 * ewma(t) = w * task_util(p) + (1-w) * ewma(t-1) 4964 * = w * task_util(p) + ewma(t-1) - w * ewma(t-1) 4965 * = w * (task_util(p) - ewma(t-1)) + ewma(t-1) 4966 * = w * ( last_ewma_diff ) + ewma(t-1) 4967 * = w * (last_ewma_diff + ewma(t-1) / w) 4968 * 4969 * Where 'w' is the weight of new samples, which is configured to be 4970 * 0.25, thus making w=1/4 ( >>= UTIL_EST_WEIGHT_SHIFT) 4971 */ 4972 ue.ewma <<= UTIL_EST_WEIGHT_SHIFT; 4973 ue.ewma += last_ewma_diff; 4974 ue.ewma >>= UTIL_EST_WEIGHT_SHIFT; 4975 done: 4976 ue.enqueued |= UTIL_AVG_UNCHANGED; 4977 WRITE_ONCE(p->se.avg.util_est, ue); 4978 4979 trace_sched_util_est_se_tp(&p->se); 4980 } 4981 4982 static inline int util_fits_cpu(unsigned long util, 4983 unsigned long uclamp_min, 4984 unsigned long uclamp_max, 4985 int cpu) 4986 { 4987 unsigned long capacity_orig, capacity_orig_thermal; 4988 unsigned long capacity = capacity_of(cpu); 4989 bool fits, uclamp_max_fits; 4990 4991 /* 4992 * Check if the real util fits without any uclamp boost/cap applied. 4993 */ 4994 fits = fits_capacity(util, capacity); 4995 4996 if (!uclamp_is_used()) 4997 return fits; 4998 4999 /* 5000 * We must use capacity_orig_of() for comparing against uclamp_min and 5001 * uclamp_max. We only care about capacity pressure (by using 5002 * capacity_of()) for comparing against the real util. 5003 * 5004 * If a task is boosted to 1024 for example, we don't want a tiny 5005 * pressure to skew the check whether it fits a CPU or not. 5006 * 5007 * Similarly if a task is capped to capacity_orig_of(little_cpu), it 5008 * should fit a little cpu even if there's some pressure. 5009 * 5010 * Only exception is for thermal pressure since it has a direct impact 5011 * on available OPP of the system. 5012 * 5013 * We honour it for uclamp_min only as a drop in performance level 5014 * could result in not getting the requested minimum performance level. 5015 * 5016 * For uclamp_max, we can tolerate a drop in performance level as the 5017 * goal is to cap the task. So it's okay if it's getting less. 5018 */ 5019 capacity_orig = capacity_orig_of(cpu); 5020 capacity_orig_thermal = capacity_orig - arch_scale_thermal_pressure(cpu); 5021 5022 /* 5023 * We want to force a task to fit a cpu as implied by uclamp_max. 5024 * But we do have some corner cases to cater for.. 5025 * 5026 * 5027 * C=z 5028 * | ___ 5029 * | C=y | | 5030 * |_ _ _ _ _ _ _ _ _ ___ _ _ _ | _ | _ _ _ _ _ uclamp_max 5031 * | C=x | | | | 5032 * | ___ | | | | 5033 * | | | | | | | (util somewhere in this region) 5034 * | | | | | | | 5035 * | | | | | | | 5036 * +---------------------------------------- 5037 * cpu0 cpu1 cpu2 5038 * 5039 * In the above example if a task is capped to a specific performance 5040 * point, y, then when: 5041 * 5042 * * util = 80% of x then it does not fit on cpu0 and should migrate 5043 * to cpu1 5044 * * util = 80% of y then it is forced to fit on cpu1 to honour 5045 * uclamp_max request. 5046 * 5047 * which is what we're enforcing here. A task always fits if 5048 * uclamp_max <= capacity_orig. But when uclamp_max > capacity_orig, 5049 * the normal upmigration rules should withhold still. 5050 * 5051 * Only exception is when we are on max capacity, then we need to be 5052 * careful not to block overutilized state. This is so because: 5053 * 5054 * 1. There's no concept of capping at max_capacity! We can't go 5055 * beyond this performance level anyway. 5056 * 2. The system is being saturated when we're operating near 5057 * max capacity, it doesn't make sense to block overutilized. 5058 */ 5059 uclamp_max_fits = (capacity_orig == SCHED_CAPACITY_SCALE) && (uclamp_max == SCHED_CAPACITY_SCALE); 5060 uclamp_max_fits = !uclamp_max_fits && (uclamp_max <= capacity_orig); 5061 fits = fits || uclamp_max_fits; 5062 5063 /* 5064 * 5065 * C=z 5066 * | ___ (region a, capped, util >= uclamp_max) 5067 * | C=y | | 5068 * |_ _ _ _ _ _ _ _ _ ___ _ _ _ | _ | _ _ _ _ _ uclamp_max 5069 * | C=x | | | | 5070 * | ___ | | | | (region b, uclamp_min <= util <= uclamp_max) 5071 * |_ _ _|_ _|_ _ _ _| _ | _ _ _| _ | _ _ _ _ _ uclamp_min 5072 * | | | | | | | 5073 * | | | | | | | (region c, boosted, util < uclamp_min) 5074 * +---------------------------------------- 5075 * cpu0 cpu1 cpu2 5076 * 5077 * a) If util > uclamp_max, then we're capped, we don't care about 5078 * actual fitness value here. We only care if uclamp_max fits 5079 * capacity without taking margin/pressure into account. 5080 * See comment above. 5081 * 5082 * b) If uclamp_min <= util <= uclamp_max, then the normal 5083 * fits_capacity() rules apply. Except we need to ensure that we 5084 * enforce we remain within uclamp_max, see comment above. 5085 * 5086 * c) If util < uclamp_min, then we are boosted. Same as (b) but we 5087 * need to take into account the boosted value fits the CPU without 5088 * taking margin/pressure into account. 5089 * 5090 * Cases (a) and (b) are handled in the 'fits' variable already. We 5091 * just need to consider an extra check for case (c) after ensuring we 5092 * handle the case uclamp_min > uclamp_max. 5093 */ 5094 uclamp_min = min(uclamp_min, uclamp_max); 5095 if (fits && (util < uclamp_min) && (uclamp_min > capacity_orig_thermal)) 5096 return -1; 5097 5098 return fits; 5099 } 5100 5101 static inline int task_fits_cpu(struct task_struct *p, int cpu) 5102 { 5103 unsigned long uclamp_min = uclamp_eff_value(p, UCLAMP_MIN); 5104 unsigned long uclamp_max = uclamp_eff_value(p, UCLAMP_MAX); 5105 unsigned long util = task_util_est(p); 5106 /* 5107 * Return true only if the cpu fully fits the task requirements, which 5108 * include the utilization but also the performance hints. 5109 */ 5110 return (util_fits_cpu(util, uclamp_min, uclamp_max, cpu) > 0); 5111 } 5112 5113 static inline void update_misfit_status(struct task_struct *p, struct rq *rq) 5114 { 5115 if (!sched_asym_cpucap_active()) 5116 return; 5117 5118 if (!p || p->nr_cpus_allowed == 1) { 5119 rq->misfit_task_load = 0; 5120 return; 5121 } 5122 5123 if (task_fits_cpu(p, cpu_of(rq))) { 5124 rq->misfit_task_load = 0; 5125 return; 5126 } 5127 5128 /* 5129 * Make sure that misfit_task_load will not be null even if 5130 * task_h_load() returns 0. 5131 */ 5132 rq->misfit_task_load = max_t(unsigned long, task_h_load(p), 1); 5133 } 5134 5135 #else /* CONFIG_SMP */ 5136 5137 static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq) 5138 { 5139 return !cfs_rq->nr_running; 5140 } 5141 5142 #define UPDATE_TG 0x0 5143 #define SKIP_AGE_LOAD 0x0 5144 #define DO_ATTACH 0x0 5145 #define DO_DETACH 0x0 5146 5147 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int not_used1) 5148 { 5149 cfs_rq_util_change(cfs_rq, 0); 5150 } 5151 5152 static inline void remove_entity_load_avg(struct sched_entity *se) {} 5153 5154 static inline void 5155 attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} 5156 static inline void 5157 detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} 5158 5159 static inline int newidle_balance(struct rq *rq, struct rq_flags *rf) 5160 { 5161 return 0; 5162 } 5163 5164 static inline void 5165 util_est_enqueue(struct cfs_rq *cfs_rq, struct task_struct *p) {} 5166 5167 static inline void 5168 util_est_dequeue(struct cfs_rq *cfs_rq, struct task_struct *p) {} 5169 5170 static inline void 5171 util_est_update(struct cfs_rq *cfs_rq, struct task_struct *p, 5172 bool task_sleep) {} 5173 static inline void update_misfit_status(struct task_struct *p, struct rq *rq) {} 5174 5175 #endif /* CONFIG_SMP */ 5176 5177 static void 5178 place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 5179 { 5180 u64 vslice, vruntime = avg_vruntime(cfs_rq); 5181 s64 lag = 0; 5182 5183 se->slice = sysctl_sched_base_slice; 5184 vslice = calc_delta_fair(se->slice, se); 5185 5186 /* 5187 * Due to how V is constructed as the weighted average of entities, 5188 * adding tasks with positive lag, or removing tasks with negative lag 5189 * will move 'time' backwards, this can screw around with the lag of 5190 * other tasks. 5191 * 5192 * EEVDF: placement strategy #1 / #2 5193 */ 5194 if (sched_feat(PLACE_LAG) && cfs_rq->nr_running) { 5195 struct sched_entity *curr = cfs_rq->curr; 5196 unsigned long load; 5197 5198 lag = se->vlag; 5199 5200 /* 5201 * If we want to place a task and preserve lag, we have to 5202 * consider the effect of the new entity on the weighted 5203 * average and compensate for this, otherwise lag can quickly 5204 * evaporate. 5205 * 5206 * Lag is defined as: 5207 * 5208 * lag_i = S - s_i = w_i * (V - v_i) 5209 * 5210 * To avoid the 'w_i' term all over the place, we only track 5211 * the virtual lag: 5212 * 5213 * vl_i = V - v_i <=> v_i = V - vl_i 5214 * 5215 * And we take V to be the weighted average of all v: 5216 * 5217 * V = (\Sum w_j*v_j) / W 5218 * 5219 * Where W is: \Sum w_j 5220 * 5221 * Then, the weighted average after adding an entity with lag 5222 * vl_i is given by: 5223 * 5224 * V' = (\Sum w_j*v_j + w_i*v_i) / (W + w_i) 5225 * = (W*V + w_i*(V - vl_i)) / (W + w_i) 5226 * = (W*V + w_i*V - w_i*vl_i) / (W + w_i) 5227 * = (V*(W + w_i) - w_i*l) / (W + w_i) 5228 * = V - w_i*vl_i / (W + w_i) 5229 * 5230 * And the actual lag after adding an entity with vl_i is: 5231 * 5232 * vl'_i = V' - v_i 5233 * = V - w_i*vl_i / (W + w_i) - (V - vl_i) 5234 * = vl_i - w_i*vl_i / (W + w_i) 5235 * 5236 * Which is strictly less than vl_i. So in order to preserve lag 5237 * we should inflate the lag before placement such that the 5238 * effective lag after placement comes out right. 5239 * 5240 * As such, invert the above relation for vl'_i to get the vl_i 5241 * we need to use such that the lag after placement is the lag 5242 * we computed before dequeue. 5243 * 5244 * vl'_i = vl_i - w_i*vl_i / (W + w_i) 5245 * = ((W + w_i)*vl_i - w_i*vl_i) / (W + w_i) 5246 * 5247 * (W + w_i)*vl'_i = (W + w_i)*vl_i - w_i*vl_i 5248 * = W*vl_i 5249 * 5250 * vl_i = (W + w_i)*vl'_i / W 5251 */ 5252 load = cfs_rq->avg_load; 5253 if (curr && curr->on_rq) 5254 load += scale_load_down(curr->load.weight); 5255 5256 lag *= load + scale_load_down(se->load.weight); 5257 if (WARN_ON_ONCE(!load)) 5258 load = 1; 5259 lag = div_s64(lag, load); 5260 } 5261 5262 se->vruntime = vruntime - lag; 5263 5264 /* 5265 * When joining the competition; the exisiting tasks will be, 5266 * on average, halfway through their slice, as such start tasks 5267 * off with half a slice to ease into the competition. 5268 */ 5269 if (sched_feat(PLACE_DEADLINE_INITIAL) && (flags & ENQUEUE_INITIAL)) 5270 vslice /= 2; 5271 5272 /* 5273 * EEVDF: vd_i = ve_i + r_i/w_i 5274 */ 5275 se->deadline = se->vruntime + vslice; 5276 } 5277 5278 static void check_enqueue_throttle(struct cfs_rq *cfs_rq); 5279 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq); 5280 5281 static inline bool cfs_bandwidth_used(void); 5282 5283 static void 5284 enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 5285 { 5286 bool curr = cfs_rq->curr == se; 5287 5288 /* 5289 * If we're the current task, we must renormalise before calling 5290 * update_curr(). 5291 */ 5292 if (curr) 5293 place_entity(cfs_rq, se, flags); 5294 5295 update_curr(cfs_rq); 5296 5297 /* 5298 * When enqueuing a sched_entity, we must: 5299 * - Update loads to have both entity and cfs_rq synced with now. 5300 * - For group_entity, update its runnable_weight to reflect the new 5301 * h_nr_running of its group cfs_rq. 5302 * - For group_entity, update its weight to reflect the new share of 5303 * its group cfs_rq 5304 * - Add its new weight to cfs_rq->load.weight 5305 */ 5306 update_load_avg(cfs_rq, se, UPDATE_TG | DO_ATTACH); 5307 se_update_runnable(se); 5308 /* 5309 * XXX update_load_avg() above will have attached us to the pelt sum; 5310 * but update_cfs_group() here will re-adjust the weight and have to 5311 * undo/redo all that. Seems wasteful. 5312 */ 5313 update_cfs_group(se); 5314 5315 /* 5316 * XXX now that the entity has been re-weighted, and it's lag adjusted, 5317 * we can place the entity. 5318 */ 5319 if (!curr) 5320 place_entity(cfs_rq, se, flags); 5321 5322 account_entity_enqueue(cfs_rq, se); 5323 5324 /* Entity has migrated, no longer consider this task hot */ 5325 if (flags & ENQUEUE_MIGRATED) 5326 se->exec_start = 0; 5327 5328 check_schedstat_required(); 5329 update_stats_enqueue_fair(cfs_rq, se, flags); 5330 if (!curr) 5331 __enqueue_entity(cfs_rq, se); 5332 se->on_rq = 1; 5333 5334 if (cfs_rq->nr_running == 1) { 5335 check_enqueue_throttle(cfs_rq); 5336 if (!throttled_hierarchy(cfs_rq)) { 5337 list_add_leaf_cfs_rq(cfs_rq); 5338 } else { 5339 #ifdef CONFIG_CFS_BANDWIDTH 5340 struct rq *rq = rq_of(cfs_rq); 5341 5342 if (cfs_rq_throttled(cfs_rq) && !cfs_rq->throttled_clock) 5343 cfs_rq->throttled_clock = rq_clock(rq); 5344 if (!cfs_rq->throttled_clock_self) 5345 cfs_rq->throttled_clock_self = rq_clock(rq); 5346 #endif 5347 } 5348 } 5349 } 5350 5351 static void __clear_buddies_next(struct sched_entity *se) 5352 { 5353 for_each_sched_entity(se) { 5354 struct cfs_rq *cfs_rq = cfs_rq_of(se); 5355 if (cfs_rq->next != se) 5356 break; 5357 5358 cfs_rq->next = NULL; 5359 } 5360 } 5361 5362 static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) 5363 { 5364 if (cfs_rq->next == se) 5365 __clear_buddies_next(se); 5366 } 5367 5368 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq); 5369 5370 static void 5371 dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 5372 { 5373 int action = UPDATE_TG; 5374 5375 if (entity_is_task(se) && task_on_rq_migrating(task_of(se))) 5376 action |= DO_DETACH; 5377 5378 /* 5379 * Update run-time statistics of the 'current'. 5380 */ 5381 update_curr(cfs_rq); 5382 5383 /* 5384 * When dequeuing a sched_entity, we must: 5385 * - Update loads to have both entity and cfs_rq synced with now. 5386 * - For group_entity, update its runnable_weight to reflect the new 5387 * h_nr_running of its group cfs_rq. 5388 * - Subtract its previous weight from cfs_rq->load.weight. 5389 * - For group entity, update its weight to reflect the new share 5390 * of its group cfs_rq. 5391 */ 5392 update_load_avg(cfs_rq, se, action); 5393 se_update_runnable(se); 5394 5395 update_stats_dequeue_fair(cfs_rq, se, flags); 5396 5397 clear_buddies(cfs_rq, se); 5398 5399 update_entity_lag(cfs_rq, se); 5400 if (se != cfs_rq->curr) 5401 __dequeue_entity(cfs_rq, se); 5402 se->on_rq = 0; 5403 account_entity_dequeue(cfs_rq, se); 5404 5405 /* return excess runtime on last dequeue */ 5406 return_cfs_rq_runtime(cfs_rq); 5407 5408 update_cfs_group(se); 5409 5410 /* 5411 * Now advance min_vruntime if @se was the entity holding it back, 5412 * except when: DEQUEUE_SAVE && !DEQUEUE_MOVE, in this case we'll be 5413 * put back on, and if we advance min_vruntime, we'll be placed back 5414 * further than we started -- ie. we'll be penalized. 5415 */ 5416 if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) != DEQUEUE_SAVE) 5417 update_min_vruntime(cfs_rq); 5418 5419 if (cfs_rq->nr_running == 0) 5420 update_idle_cfs_rq_clock_pelt(cfs_rq); 5421 } 5422 5423 static void 5424 set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 5425 { 5426 clear_buddies(cfs_rq, se); 5427 5428 /* 'current' is not kept within the tree. */ 5429 if (se->on_rq) { 5430 /* 5431 * Any task has to be enqueued before it get to execute on 5432 * a CPU. So account for the time it spent waiting on the 5433 * runqueue. 5434 */ 5435 update_stats_wait_end_fair(cfs_rq, se); 5436 __dequeue_entity(cfs_rq, se); 5437 update_load_avg(cfs_rq, se, UPDATE_TG); 5438 /* 5439 * HACK, stash a copy of deadline at the point of pick in vlag, 5440 * which isn't used until dequeue. 5441 */ 5442 se->vlag = se->deadline; 5443 } 5444 5445 update_stats_curr_start(cfs_rq, se); 5446 cfs_rq->curr = se; 5447 5448 /* 5449 * Track our maximum slice length, if the CPU's load is at 5450 * least twice that of our own weight (i.e. dont track it 5451 * when there are only lesser-weight tasks around): 5452 */ 5453 if (schedstat_enabled() && 5454 rq_of(cfs_rq)->cfs.load.weight >= 2*se->load.weight) { 5455 struct sched_statistics *stats; 5456 5457 stats = __schedstats_from_se(se); 5458 __schedstat_set(stats->slice_max, 5459 max((u64)stats->slice_max, 5460 se->sum_exec_runtime - se->prev_sum_exec_runtime)); 5461 } 5462 5463 se->prev_sum_exec_runtime = se->sum_exec_runtime; 5464 } 5465 5466 /* 5467 * Pick the next process, keeping these things in mind, in this order: 5468 * 1) keep things fair between processes/task groups 5469 * 2) pick the "next" process, since someone really wants that to run 5470 * 3) pick the "last" process, for cache locality 5471 * 4) do not run the "skip" process, if something else is available 5472 */ 5473 static struct sched_entity * 5474 pick_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *curr) 5475 { 5476 /* 5477 * Enabling NEXT_BUDDY will affect latency but not fairness. 5478 */ 5479 if (sched_feat(NEXT_BUDDY) && 5480 cfs_rq->next && entity_eligible(cfs_rq, cfs_rq->next)) 5481 return cfs_rq->next; 5482 5483 return pick_eevdf(cfs_rq); 5484 } 5485 5486 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq); 5487 5488 static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) 5489 { 5490 /* 5491 * If still on the runqueue then deactivate_task() 5492 * was not called and update_curr() has to be done: 5493 */ 5494 if (prev->on_rq) 5495 update_curr(cfs_rq); 5496 5497 /* throttle cfs_rqs exceeding runtime */ 5498 check_cfs_rq_runtime(cfs_rq); 5499 5500 if (prev->on_rq) { 5501 update_stats_wait_start_fair(cfs_rq, prev); 5502 /* Put 'current' back into the tree. */ 5503 __enqueue_entity(cfs_rq, prev); 5504 /* in !on_rq case, update occurred at dequeue */ 5505 update_load_avg(cfs_rq, prev, 0); 5506 } 5507 cfs_rq->curr = NULL; 5508 } 5509 5510 static void 5511 entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) 5512 { 5513 /* 5514 * Update run-time statistics of the 'current'. 5515 */ 5516 update_curr(cfs_rq); 5517 5518 /* 5519 * Ensure that runnable average is periodically updated. 5520 */ 5521 update_load_avg(cfs_rq, curr, UPDATE_TG); 5522 update_cfs_group(curr); 5523 5524 #ifdef CONFIG_SCHED_HRTICK 5525 /* 5526 * queued ticks are scheduled to match the slice, so don't bother 5527 * validating it and just reschedule. 5528 */ 5529 if (queued) { 5530 resched_curr(rq_of(cfs_rq)); 5531 return; 5532 } 5533 /* 5534 * don't let the period tick interfere with the hrtick preemption 5535 */ 5536 if (!sched_feat(DOUBLE_TICK) && 5537 hrtimer_active(&rq_of(cfs_rq)->hrtick_timer)) 5538 return; 5539 #endif 5540 } 5541 5542 5543 /************************************************** 5544 * CFS bandwidth control machinery 5545 */ 5546 5547 #ifdef CONFIG_CFS_BANDWIDTH 5548 5549 #ifdef CONFIG_JUMP_LABEL 5550 static struct static_key __cfs_bandwidth_used; 5551 5552 static inline bool cfs_bandwidth_used(void) 5553 { 5554 return static_key_false(&__cfs_bandwidth_used); 5555 } 5556 5557 void cfs_bandwidth_usage_inc(void) 5558 { 5559 static_key_slow_inc_cpuslocked(&__cfs_bandwidth_used); 5560 } 5561 5562 void cfs_bandwidth_usage_dec(void) 5563 { 5564 static_key_slow_dec_cpuslocked(&__cfs_bandwidth_used); 5565 } 5566 #else /* CONFIG_JUMP_LABEL */ 5567 static bool cfs_bandwidth_used(void) 5568 { 5569 return true; 5570 } 5571 5572 void cfs_bandwidth_usage_inc(void) {} 5573 void cfs_bandwidth_usage_dec(void) {} 5574 #endif /* CONFIG_JUMP_LABEL */ 5575 5576 /* 5577 * default period for cfs group bandwidth. 5578 * default: 0.1s, units: nanoseconds 5579 */ 5580 static inline u64 default_cfs_period(void) 5581 { 5582 return 100000000ULL; 5583 } 5584 5585 static inline u64 sched_cfs_bandwidth_slice(void) 5586 { 5587 return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC; 5588 } 5589 5590 /* 5591 * Replenish runtime according to assigned quota. We use sched_clock_cpu 5592 * directly instead of rq->clock to avoid adding additional synchronization 5593 * around rq->lock. 5594 * 5595 * requires cfs_b->lock 5596 */ 5597 void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b) 5598 { 5599 s64 runtime; 5600 5601 if (unlikely(cfs_b->quota == RUNTIME_INF)) 5602 return; 5603 5604 cfs_b->runtime += cfs_b->quota; 5605 runtime = cfs_b->runtime_snap - cfs_b->runtime; 5606 if (runtime > 0) { 5607 cfs_b->burst_time += runtime; 5608 cfs_b->nr_burst++; 5609 } 5610 5611 cfs_b->runtime = min(cfs_b->runtime, cfs_b->quota + cfs_b->burst); 5612 cfs_b->runtime_snap = cfs_b->runtime; 5613 } 5614 5615 static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) 5616 { 5617 return &tg->cfs_bandwidth; 5618 } 5619 5620 /* returns 0 on failure to allocate runtime */ 5621 static int __assign_cfs_rq_runtime(struct cfs_bandwidth *cfs_b, 5622 struct cfs_rq *cfs_rq, u64 target_runtime) 5623 { 5624 u64 min_amount, amount = 0; 5625 5626 lockdep_assert_held(&cfs_b->lock); 5627 5628 /* note: this is a positive sum as runtime_remaining <= 0 */ 5629 min_amount = target_runtime - cfs_rq->runtime_remaining; 5630 5631 if (cfs_b->quota == RUNTIME_INF) 5632 amount = min_amount; 5633 else { 5634 start_cfs_bandwidth(cfs_b); 5635 5636 if (cfs_b->runtime > 0) { 5637 amount = min(cfs_b->runtime, min_amount); 5638 cfs_b->runtime -= amount; 5639 cfs_b->idle = 0; 5640 } 5641 } 5642 5643 cfs_rq->runtime_remaining += amount; 5644 5645 return cfs_rq->runtime_remaining > 0; 5646 } 5647 5648 /* returns 0 on failure to allocate runtime */ 5649 static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq) 5650 { 5651 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 5652 int ret; 5653 5654 raw_spin_lock(&cfs_b->lock); 5655 ret = __assign_cfs_rq_runtime(cfs_b, cfs_rq, sched_cfs_bandwidth_slice()); 5656 raw_spin_unlock(&cfs_b->lock); 5657 5658 return ret; 5659 } 5660 5661 static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) 5662 { 5663 /* dock delta_exec before expiring quota (as it could span periods) */ 5664 cfs_rq->runtime_remaining -= delta_exec; 5665 5666 if (likely(cfs_rq->runtime_remaining > 0)) 5667 return; 5668 5669 if (cfs_rq->throttled) 5670 return; 5671 /* 5672 * if we're unable to extend our runtime we resched so that the active 5673 * hierarchy can be throttled 5674 */ 5675 if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr)) 5676 resched_curr(rq_of(cfs_rq)); 5677 } 5678 5679 static __always_inline 5680 void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) 5681 { 5682 if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled) 5683 return; 5684 5685 __account_cfs_rq_runtime(cfs_rq, delta_exec); 5686 } 5687 5688 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) 5689 { 5690 return cfs_bandwidth_used() && cfs_rq->throttled; 5691 } 5692 5693 /* check whether cfs_rq, or any parent, is throttled */ 5694 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) 5695 { 5696 return cfs_bandwidth_used() && cfs_rq->throttle_count; 5697 } 5698 5699 /* 5700 * Ensure that neither of the group entities corresponding to src_cpu or 5701 * dest_cpu are members of a throttled hierarchy when performing group 5702 * load-balance operations. 5703 */ 5704 static inline int throttled_lb_pair(struct task_group *tg, 5705 int src_cpu, int dest_cpu) 5706 { 5707 struct cfs_rq *src_cfs_rq, *dest_cfs_rq; 5708 5709 src_cfs_rq = tg->cfs_rq[src_cpu]; 5710 dest_cfs_rq = tg->cfs_rq[dest_cpu]; 5711 5712 return throttled_hierarchy(src_cfs_rq) || 5713 throttled_hierarchy(dest_cfs_rq); 5714 } 5715 5716 static int tg_unthrottle_up(struct task_group *tg, void *data) 5717 { 5718 struct rq *rq = data; 5719 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 5720 5721 cfs_rq->throttle_count--; 5722 if (!cfs_rq->throttle_count) { 5723 cfs_rq->throttled_clock_pelt_time += rq_clock_pelt(rq) - 5724 cfs_rq->throttled_clock_pelt; 5725 5726 /* Add cfs_rq with load or one or more already running entities to the list */ 5727 if (!cfs_rq_is_decayed(cfs_rq)) 5728 list_add_leaf_cfs_rq(cfs_rq); 5729 5730 if (cfs_rq->throttled_clock_self) { 5731 u64 delta = rq_clock(rq) - cfs_rq->throttled_clock_self; 5732 5733 cfs_rq->throttled_clock_self = 0; 5734 5735 if (SCHED_WARN_ON((s64)delta < 0)) 5736 delta = 0; 5737 5738 cfs_rq->throttled_clock_self_time += delta; 5739 } 5740 } 5741 5742 return 0; 5743 } 5744 5745 static int tg_throttle_down(struct task_group *tg, void *data) 5746 { 5747 struct rq *rq = data; 5748 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 5749 5750 /* group is entering throttled state, stop time */ 5751 if (!cfs_rq->throttle_count) { 5752 cfs_rq->throttled_clock_pelt = rq_clock_pelt(rq); 5753 list_del_leaf_cfs_rq(cfs_rq); 5754 5755 SCHED_WARN_ON(cfs_rq->throttled_clock_self); 5756 if (cfs_rq->nr_running) 5757 cfs_rq->throttled_clock_self = rq_clock(rq); 5758 } 5759 cfs_rq->throttle_count++; 5760 5761 return 0; 5762 } 5763 5764 static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) 5765 { 5766 struct rq *rq = rq_of(cfs_rq); 5767 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 5768 struct sched_entity *se; 5769 long task_delta, idle_task_delta, dequeue = 1; 5770 5771 raw_spin_lock(&cfs_b->lock); 5772 /* This will start the period timer if necessary */ 5773 if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, 1)) { 5774 /* 5775 * We have raced with bandwidth becoming available, and if we 5776 * actually throttled the timer might not unthrottle us for an 5777 * entire period. We additionally needed to make sure that any 5778 * subsequent check_cfs_rq_runtime calls agree not to throttle 5779 * us, as we may commit to do cfs put_prev+pick_next, so we ask 5780 * for 1ns of runtime rather than just check cfs_b. 5781 */ 5782 dequeue = 0; 5783 } else { 5784 list_add_tail_rcu(&cfs_rq->throttled_list, 5785 &cfs_b->throttled_cfs_rq); 5786 } 5787 raw_spin_unlock(&cfs_b->lock); 5788 5789 if (!dequeue) 5790 return false; /* Throttle no longer required. */ 5791 5792 se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))]; 5793 5794 /* freeze hierarchy runnable averages while throttled */ 5795 rcu_read_lock(); 5796 walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq); 5797 rcu_read_unlock(); 5798 5799 task_delta = cfs_rq->h_nr_running; 5800 idle_task_delta = cfs_rq->idle_h_nr_running; 5801 for_each_sched_entity(se) { 5802 struct cfs_rq *qcfs_rq = cfs_rq_of(se); 5803 /* throttled entity or throttle-on-deactivate */ 5804 if (!se->on_rq) 5805 goto done; 5806 5807 dequeue_entity(qcfs_rq, se, DEQUEUE_SLEEP); 5808 5809 if (cfs_rq_is_idle(group_cfs_rq(se))) 5810 idle_task_delta = cfs_rq->h_nr_running; 5811 5812 qcfs_rq->h_nr_running -= task_delta; 5813 qcfs_rq->idle_h_nr_running -= idle_task_delta; 5814 5815 if (qcfs_rq->load.weight) { 5816 /* Avoid re-evaluating load for this entity: */ 5817 se = parent_entity(se); 5818 break; 5819 } 5820 } 5821 5822 for_each_sched_entity(se) { 5823 struct cfs_rq *qcfs_rq = cfs_rq_of(se); 5824 /* throttled entity or throttle-on-deactivate */ 5825 if (!se->on_rq) 5826 goto done; 5827 5828 update_load_avg(qcfs_rq, se, 0); 5829 se_update_runnable(se); 5830 5831 if (cfs_rq_is_idle(group_cfs_rq(se))) 5832 idle_task_delta = cfs_rq->h_nr_running; 5833 5834 qcfs_rq->h_nr_running -= task_delta; 5835 qcfs_rq->idle_h_nr_running -= idle_task_delta; 5836 } 5837 5838 /* At this point se is NULL and we are at root level*/ 5839 sub_nr_running(rq, task_delta); 5840 5841 done: 5842 /* 5843 * Note: distribution will already see us throttled via the 5844 * throttled-list. rq->lock protects completion. 5845 */ 5846 cfs_rq->throttled = 1; 5847 SCHED_WARN_ON(cfs_rq->throttled_clock); 5848 if (cfs_rq->nr_running) 5849 cfs_rq->throttled_clock = rq_clock(rq); 5850 return true; 5851 } 5852 5853 void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) 5854 { 5855 struct rq *rq = rq_of(cfs_rq); 5856 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 5857 struct sched_entity *se; 5858 long task_delta, idle_task_delta; 5859 5860 se = cfs_rq->tg->se[cpu_of(rq)]; 5861 5862 cfs_rq->throttled = 0; 5863 5864 update_rq_clock(rq); 5865 5866 raw_spin_lock(&cfs_b->lock); 5867 if (cfs_rq->throttled_clock) { 5868 cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock; 5869 cfs_rq->throttled_clock = 0; 5870 } 5871 list_del_rcu(&cfs_rq->throttled_list); 5872 raw_spin_unlock(&cfs_b->lock); 5873 5874 /* update hierarchical throttle state */ 5875 walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq); 5876 5877 if (!cfs_rq->load.weight) { 5878 if (!cfs_rq->on_list) 5879 return; 5880 /* 5881 * Nothing to run but something to decay (on_list)? 5882 * Complete the branch. 5883 */ 5884 for_each_sched_entity(se) { 5885 if (list_add_leaf_cfs_rq(cfs_rq_of(se))) 5886 break; 5887 } 5888 goto unthrottle_throttle; 5889 } 5890 5891 task_delta = cfs_rq->h_nr_running; 5892 idle_task_delta = cfs_rq->idle_h_nr_running; 5893 for_each_sched_entity(se) { 5894 struct cfs_rq *qcfs_rq = cfs_rq_of(se); 5895 5896 if (se->on_rq) 5897 break; 5898 enqueue_entity(qcfs_rq, se, ENQUEUE_WAKEUP); 5899 5900 if (cfs_rq_is_idle(group_cfs_rq(se))) 5901 idle_task_delta = cfs_rq->h_nr_running; 5902 5903 qcfs_rq->h_nr_running += task_delta; 5904 qcfs_rq->idle_h_nr_running += idle_task_delta; 5905 5906 /* end evaluation on encountering a throttled cfs_rq */ 5907 if (cfs_rq_throttled(qcfs_rq)) 5908 goto unthrottle_throttle; 5909 } 5910 5911 for_each_sched_entity(se) { 5912 struct cfs_rq *qcfs_rq = cfs_rq_of(se); 5913 5914 update_load_avg(qcfs_rq, se, UPDATE_TG); 5915 se_update_runnable(se); 5916 5917 if (cfs_rq_is_idle(group_cfs_rq(se))) 5918 idle_task_delta = cfs_rq->h_nr_running; 5919 5920 qcfs_rq->h_nr_running += task_delta; 5921 qcfs_rq->idle_h_nr_running += idle_task_delta; 5922 5923 /* end evaluation on encountering a throttled cfs_rq */ 5924 if (cfs_rq_throttled(qcfs_rq)) 5925 goto unthrottle_throttle; 5926 } 5927 5928 /* At this point se is NULL and we are at root level*/ 5929 add_nr_running(rq, task_delta); 5930 5931 unthrottle_throttle: 5932 assert_list_leaf_cfs_rq(rq); 5933 5934 /* Determine whether we need to wake up potentially idle CPU: */ 5935 if (rq->curr == rq->idle && rq->cfs.nr_running) 5936 resched_curr(rq); 5937 } 5938 5939 #ifdef CONFIG_SMP 5940 static void __cfsb_csd_unthrottle(void *arg) 5941 { 5942 struct cfs_rq *cursor, *tmp; 5943 struct rq *rq = arg; 5944 struct rq_flags rf; 5945 5946 rq_lock(rq, &rf); 5947 5948 /* 5949 * Iterating over the list can trigger several call to 5950 * update_rq_clock() in unthrottle_cfs_rq(). 5951 * Do it once and skip the potential next ones. 5952 */ 5953 update_rq_clock(rq); 5954 rq_clock_start_loop_update(rq); 5955 5956 /* 5957 * Since we hold rq lock we're safe from concurrent manipulation of 5958 * the CSD list. However, this RCU critical section annotates the 5959 * fact that we pair with sched_free_group_rcu(), so that we cannot 5960 * race with group being freed in the window between removing it 5961 * from the list and advancing to the next entry in the list. 5962 */ 5963 rcu_read_lock(); 5964 5965 list_for_each_entry_safe(cursor, tmp, &rq->cfsb_csd_list, 5966 throttled_csd_list) { 5967 list_del_init(&cursor->throttled_csd_list); 5968 5969 if (cfs_rq_throttled(cursor)) 5970 unthrottle_cfs_rq(cursor); 5971 } 5972 5973 rcu_read_unlock(); 5974 5975 rq_clock_stop_loop_update(rq); 5976 rq_unlock(rq, &rf); 5977 } 5978 5979 static inline void __unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq) 5980 { 5981 struct rq *rq = rq_of(cfs_rq); 5982 bool first; 5983 5984 if (rq == this_rq()) { 5985 unthrottle_cfs_rq(cfs_rq); 5986 return; 5987 } 5988 5989 /* Already enqueued */ 5990 if (SCHED_WARN_ON(!list_empty(&cfs_rq->throttled_csd_list))) 5991 return; 5992 5993 first = list_empty(&rq->cfsb_csd_list); 5994 list_add_tail(&cfs_rq->throttled_csd_list, &rq->cfsb_csd_list); 5995 if (first) 5996 smp_call_function_single_async(cpu_of(rq), &rq->cfsb_csd); 5997 } 5998 #else 5999 static inline void __unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq) 6000 { 6001 unthrottle_cfs_rq(cfs_rq); 6002 } 6003 #endif 6004 6005 static void unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq) 6006 { 6007 lockdep_assert_rq_held(rq_of(cfs_rq)); 6008 6009 if (SCHED_WARN_ON(!cfs_rq_throttled(cfs_rq) || 6010 cfs_rq->runtime_remaining <= 0)) 6011 return; 6012 6013 __unthrottle_cfs_rq_async(cfs_rq); 6014 } 6015 6016 static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b) 6017 { 6018 struct cfs_rq *local_unthrottle = NULL; 6019 int this_cpu = smp_processor_id(); 6020 u64 runtime, remaining = 1; 6021 bool throttled = false; 6022 struct cfs_rq *cfs_rq; 6023 struct rq_flags rf; 6024 struct rq *rq; 6025 6026 rcu_read_lock(); 6027 list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq, 6028 throttled_list) { 6029 rq = rq_of(cfs_rq); 6030 6031 if (!remaining) { 6032 throttled = true; 6033 break; 6034 } 6035 6036 rq_lock_irqsave(rq, &rf); 6037 if (!cfs_rq_throttled(cfs_rq)) 6038 goto next; 6039 6040 #ifdef CONFIG_SMP 6041 /* Already queued for async unthrottle */ 6042 if (!list_empty(&cfs_rq->throttled_csd_list)) 6043 goto next; 6044 #endif 6045 6046 /* By the above checks, this should never be true */ 6047 SCHED_WARN_ON(cfs_rq->runtime_remaining > 0); 6048 6049 raw_spin_lock(&cfs_b->lock); 6050 runtime = -cfs_rq->runtime_remaining + 1; 6051 if (runtime > cfs_b->runtime) 6052 runtime = cfs_b->runtime; 6053 cfs_b->runtime -= runtime; 6054 remaining = cfs_b->runtime; 6055 raw_spin_unlock(&cfs_b->lock); 6056 6057 cfs_rq->runtime_remaining += runtime; 6058 6059 /* we check whether we're throttled above */ 6060 if (cfs_rq->runtime_remaining > 0) { 6061 if (cpu_of(rq) != this_cpu || 6062 SCHED_WARN_ON(local_unthrottle)) 6063 unthrottle_cfs_rq_async(cfs_rq); 6064 else 6065 local_unthrottle = cfs_rq; 6066 } else { 6067 throttled = true; 6068 } 6069 6070 next: 6071 rq_unlock_irqrestore(rq, &rf); 6072 } 6073 rcu_read_unlock(); 6074 6075 if (local_unthrottle) { 6076 rq = cpu_rq(this_cpu); 6077 rq_lock_irqsave(rq, &rf); 6078 if (cfs_rq_throttled(local_unthrottle)) 6079 unthrottle_cfs_rq(local_unthrottle); 6080 rq_unlock_irqrestore(rq, &rf); 6081 } 6082 6083 return throttled; 6084 } 6085 6086 /* 6087 * Responsible for refilling a task_group's bandwidth and unthrottling its 6088 * cfs_rqs as appropriate. If there has been no activity within the last 6089 * period the timer is deactivated until scheduling resumes; cfs_b->idle is 6090 * used to track this state. 6091 */ 6092 static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, unsigned long flags) 6093 { 6094 int throttled; 6095 6096 /* no need to continue the timer with no bandwidth constraint */ 6097 if (cfs_b->quota == RUNTIME_INF) 6098 goto out_deactivate; 6099 6100 throttled = !list_empty(&cfs_b->throttled_cfs_rq); 6101 cfs_b->nr_periods += overrun; 6102 6103 /* Refill extra burst quota even if cfs_b->idle */ 6104 __refill_cfs_bandwidth_runtime(cfs_b); 6105 6106 /* 6107 * idle depends on !throttled (for the case of a large deficit), and if 6108 * we're going inactive then everything else can be deferred 6109 */ 6110 if (cfs_b->idle && !throttled) 6111 goto out_deactivate; 6112 6113 if (!throttled) { 6114 /* mark as potentially idle for the upcoming period */ 6115 cfs_b->idle = 1; 6116 return 0; 6117 } 6118 6119 /* account preceding periods in which throttling occurred */ 6120 cfs_b->nr_throttled += overrun; 6121 6122 /* 6123 * This check is repeated as we release cfs_b->lock while we unthrottle. 6124 */ 6125 while (throttled && cfs_b->runtime > 0) { 6126 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 6127 /* we can't nest cfs_b->lock while distributing bandwidth */ 6128 throttled = distribute_cfs_runtime(cfs_b); 6129 raw_spin_lock_irqsave(&cfs_b->lock, flags); 6130 } 6131 6132 /* 6133 * While we are ensured activity in the period following an 6134 * unthrottle, this also covers the case in which the new bandwidth is 6135 * insufficient to cover the existing bandwidth deficit. (Forcing the 6136 * timer to remain active while there are any throttled entities.) 6137 */ 6138 cfs_b->idle = 0; 6139 6140 return 0; 6141 6142 out_deactivate: 6143 return 1; 6144 } 6145 6146 /* a cfs_rq won't donate quota below this amount */ 6147 static const u64 min_cfs_rq_runtime = 1 * NSEC_PER_MSEC; 6148 /* minimum remaining period time to redistribute slack quota */ 6149 static const u64 min_bandwidth_expiration = 2 * NSEC_PER_MSEC; 6150 /* how long we wait to gather additional slack before distributing */ 6151 static const u64 cfs_bandwidth_slack_period = 5 * NSEC_PER_MSEC; 6152 6153 /* 6154 * Are we near the end of the current quota period? 6155 * 6156 * Requires cfs_b->lock for hrtimer_expires_remaining to be safe against the 6157 * hrtimer base being cleared by hrtimer_start. In the case of 6158 * migrate_hrtimers, base is never cleared, so we are fine. 6159 */ 6160 static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire) 6161 { 6162 struct hrtimer *refresh_timer = &cfs_b->period_timer; 6163 s64 remaining; 6164 6165 /* if the call-back is running a quota refresh is already occurring */ 6166 if (hrtimer_callback_running(refresh_timer)) 6167 return 1; 6168 6169 /* is a quota refresh about to occur? */ 6170 remaining = ktime_to_ns(hrtimer_expires_remaining(refresh_timer)); 6171 if (remaining < (s64)min_expire) 6172 return 1; 6173 6174 return 0; 6175 } 6176 6177 static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b) 6178 { 6179 u64 min_left = cfs_bandwidth_slack_period + min_bandwidth_expiration; 6180 6181 /* if there's a quota refresh soon don't bother with slack */ 6182 if (runtime_refresh_within(cfs_b, min_left)) 6183 return; 6184 6185 /* don't push forwards an existing deferred unthrottle */ 6186 if (cfs_b->slack_started) 6187 return; 6188 cfs_b->slack_started = true; 6189 6190 hrtimer_start(&cfs_b->slack_timer, 6191 ns_to_ktime(cfs_bandwidth_slack_period), 6192 HRTIMER_MODE_REL); 6193 } 6194 6195 /* we know any runtime found here is valid as update_curr() precedes return */ 6196 static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq) 6197 { 6198 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 6199 s64 slack_runtime = cfs_rq->runtime_remaining - min_cfs_rq_runtime; 6200 6201 if (slack_runtime <= 0) 6202 return; 6203 6204 raw_spin_lock(&cfs_b->lock); 6205 if (cfs_b->quota != RUNTIME_INF) { 6206 cfs_b->runtime += slack_runtime; 6207 6208 /* we are under rq->lock, defer unthrottling using a timer */ 6209 if (cfs_b->runtime > sched_cfs_bandwidth_slice() && 6210 !list_empty(&cfs_b->throttled_cfs_rq)) 6211 start_cfs_slack_bandwidth(cfs_b); 6212 } 6213 raw_spin_unlock(&cfs_b->lock); 6214 6215 /* even if it's not valid for return we don't want to try again */ 6216 cfs_rq->runtime_remaining -= slack_runtime; 6217 } 6218 6219 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) 6220 { 6221 if (!cfs_bandwidth_used()) 6222 return; 6223 6224 if (!cfs_rq->runtime_enabled || cfs_rq->nr_running) 6225 return; 6226 6227 __return_cfs_rq_runtime(cfs_rq); 6228 } 6229 6230 /* 6231 * This is done with a timer (instead of inline with bandwidth return) since 6232 * it's necessary to juggle rq->locks to unthrottle their respective cfs_rqs. 6233 */ 6234 static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) 6235 { 6236 u64 runtime = 0, slice = sched_cfs_bandwidth_slice(); 6237 unsigned long flags; 6238 6239 /* confirm we're still not at a refresh boundary */ 6240 raw_spin_lock_irqsave(&cfs_b->lock, flags); 6241 cfs_b->slack_started = false; 6242 6243 if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) { 6244 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 6245 return; 6246 } 6247 6248 if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice) 6249 runtime = cfs_b->runtime; 6250 6251 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 6252 6253 if (!runtime) 6254 return; 6255 6256 distribute_cfs_runtime(cfs_b); 6257 } 6258 6259 /* 6260 * When a group wakes up we want to make sure that its quota is not already 6261 * expired/exceeded, otherwise it may be allowed to steal additional ticks of 6262 * runtime as update_curr() throttling can not trigger until it's on-rq. 6263 */ 6264 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) 6265 { 6266 if (!cfs_bandwidth_used()) 6267 return; 6268 6269 /* an active group must be handled by the update_curr()->put() path */ 6270 if (!cfs_rq->runtime_enabled || cfs_rq->curr) 6271 return; 6272 6273 /* ensure the group is not already throttled */ 6274 if (cfs_rq_throttled(cfs_rq)) 6275 return; 6276 6277 /* update runtime allocation */ 6278 account_cfs_rq_runtime(cfs_rq, 0); 6279 if (cfs_rq->runtime_remaining <= 0) 6280 throttle_cfs_rq(cfs_rq); 6281 } 6282 6283 static void sync_throttle(struct task_group *tg, int cpu) 6284 { 6285 struct cfs_rq *pcfs_rq, *cfs_rq; 6286 6287 if (!cfs_bandwidth_used()) 6288 return; 6289 6290 if (!tg->parent) 6291 return; 6292 6293 cfs_rq = tg->cfs_rq[cpu]; 6294 pcfs_rq = tg->parent->cfs_rq[cpu]; 6295 6296 cfs_rq->throttle_count = pcfs_rq->throttle_count; 6297 cfs_rq->throttled_clock_pelt = rq_clock_pelt(cpu_rq(cpu)); 6298 } 6299 6300 /* conditionally throttle active cfs_rq's from put_prev_entity() */ 6301 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) 6302 { 6303 if (!cfs_bandwidth_used()) 6304 return false; 6305 6306 if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0)) 6307 return false; 6308 6309 /* 6310 * it's possible for a throttled entity to be forced into a running 6311 * state (e.g. set_curr_task), in this case we're finished. 6312 */ 6313 if (cfs_rq_throttled(cfs_rq)) 6314 return true; 6315 6316 return throttle_cfs_rq(cfs_rq); 6317 } 6318 6319 static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer) 6320 { 6321 struct cfs_bandwidth *cfs_b = 6322 container_of(timer, struct cfs_bandwidth, slack_timer); 6323 6324 do_sched_cfs_slack_timer(cfs_b); 6325 6326 return HRTIMER_NORESTART; 6327 } 6328 6329 extern const u64 max_cfs_quota_period; 6330 6331 static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer) 6332 { 6333 struct cfs_bandwidth *cfs_b = 6334 container_of(timer, struct cfs_bandwidth, period_timer); 6335 unsigned long flags; 6336 int overrun; 6337 int idle = 0; 6338 int count = 0; 6339 6340 raw_spin_lock_irqsave(&cfs_b->lock, flags); 6341 for (;;) { 6342 overrun = hrtimer_forward_now(timer, cfs_b->period); 6343 if (!overrun) 6344 break; 6345 6346 idle = do_sched_cfs_period_timer(cfs_b, overrun, flags); 6347 6348 if (++count > 3) { 6349 u64 new, old = ktime_to_ns(cfs_b->period); 6350 6351 /* 6352 * Grow period by a factor of 2 to avoid losing precision. 6353 * Precision loss in the quota/period ratio can cause __cfs_schedulable 6354 * to fail. 6355 */ 6356 new = old * 2; 6357 if (new < max_cfs_quota_period) { 6358 cfs_b->period = ns_to_ktime(new); 6359 cfs_b->quota *= 2; 6360 cfs_b->burst *= 2; 6361 6362 pr_warn_ratelimited( 6363 "cfs_period_timer[cpu%d]: period too short, scaling up (new cfs_period_us = %lld, cfs_quota_us = %lld)\n", 6364 smp_processor_id(), 6365 div_u64(new, NSEC_PER_USEC), 6366 div_u64(cfs_b->quota, NSEC_PER_USEC)); 6367 } else { 6368 pr_warn_ratelimited( 6369 "cfs_period_timer[cpu%d]: period too short, but cannot scale up without losing precision (cfs_period_us = %lld, cfs_quota_us = %lld)\n", 6370 smp_processor_id(), 6371 div_u64(old, NSEC_PER_USEC), 6372 div_u64(cfs_b->quota, NSEC_PER_USEC)); 6373 } 6374 6375 /* reset count so we don't come right back in here */ 6376 count = 0; 6377 } 6378 } 6379 if (idle) 6380 cfs_b->period_active = 0; 6381 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 6382 6383 return idle ? HRTIMER_NORESTART : HRTIMER_RESTART; 6384 } 6385 6386 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b, struct cfs_bandwidth *parent) 6387 { 6388 raw_spin_lock_init(&cfs_b->lock); 6389 cfs_b->runtime = 0; 6390 cfs_b->quota = RUNTIME_INF; 6391 cfs_b->period = ns_to_ktime(default_cfs_period()); 6392 cfs_b->burst = 0; 6393 cfs_b->hierarchical_quota = parent ? parent->hierarchical_quota : RUNTIME_INF; 6394 6395 INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq); 6396 hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED); 6397 cfs_b->period_timer.function = sched_cfs_period_timer; 6398 6399 /* Add a random offset so that timers interleave */ 6400 hrtimer_set_expires(&cfs_b->period_timer, 6401 get_random_u32_below(cfs_b->period)); 6402 hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 6403 cfs_b->slack_timer.function = sched_cfs_slack_timer; 6404 cfs_b->slack_started = false; 6405 } 6406 6407 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) 6408 { 6409 cfs_rq->runtime_enabled = 0; 6410 INIT_LIST_HEAD(&cfs_rq->throttled_list); 6411 #ifdef CONFIG_SMP 6412 INIT_LIST_HEAD(&cfs_rq->throttled_csd_list); 6413 #endif 6414 } 6415 6416 void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b) 6417 { 6418 lockdep_assert_held(&cfs_b->lock); 6419 6420 if (cfs_b->period_active) 6421 return; 6422 6423 cfs_b->period_active = 1; 6424 hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period); 6425 hrtimer_start_expires(&cfs_b->period_timer, HRTIMER_MODE_ABS_PINNED); 6426 } 6427 6428 static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) 6429 { 6430 int __maybe_unused i; 6431 6432 /* init_cfs_bandwidth() was not called */ 6433 if (!cfs_b->throttled_cfs_rq.next) 6434 return; 6435 6436 hrtimer_cancel(&cfs_b->period_timer); 6437 hrtimer_cancel(&cfs_b->slack_timer); 6438 6439 /* 6440 * It is possible that we still have some cfs_rq's pending on a CSD 6441 * list, though this race is very rare. In order for this to occur, we 6442 * must have raced with the last task leaving the group while there 6443 * exist throttled cfs_rq(s), and the period_timer must have queued the 6444 * CSD item but the remote cpu has not yet processed it. To handle this, 6445 * we can simply flush all pending CSD work inline here. We're 6446 * guaranteed at this point that no additional cfs_rq of this group can 6447 * join a CSD list. 6448 */ 6449 #ifdef CONFIG_SMP 6450 for_each_possible_cpu(i) { 6451 struct rq *rq = cpu_rq(i); 6452 unsigned long flags; 6453 6454 if (list_empty(&rq->cfsb_csd_list)) 6455 continue; 6456 6457 local_irq_save(flags); 6458 __cfsb_csd_unthrottle(rq); 6459 local_irq_restore(flags); 6460 } 6461 #endif 6462 } 6463 6464 /* 6465 * Both these CPU hotplug callbacks race against unregister_fair_sched_group() 6466 * 6467 * The race is harmless, since modifying bandwidth settings of unhooked group 6468 * bits doesn't do much. 6469 */ 6470 6471 /* cpu online callback */ 6472 static void __maybe_unused update_runtime_enabled(struct rq *rq) 6473 { 6474 struct task_group *tg; 6475 6476 lockdep_assert_rq_held(rq); 6477 6478 rcu_read_lock(); 6479 list_for_each_entry_rcu(tg, &task_groups, list) { 6480 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 6481 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 6482 6483 raw_spin_lock(&cfs_b->lock); 6484 cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF; 6485 raw_spin_unlock(&cfs_b->lock); 6486 } 6487 rcu_read_unlock(); 6488 } 6489 6490 /* cpu offline callback */ 6491 static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq) 6492 { 6493 struct task_group *tg; 6494 6495 lockdep_assert_rq_held(rq); 6496 6497 /* 6498 * The rq clock has already been updated in the 6499 * set_rq_offline(), so we should skip updating 6500 * the rq clock again in unthrottle_cfs_rq(). 6501 */ 6502 rq_clock_start_loop_update(rq); 6503 6504 rcu_read_lock(); 6505 list_for_each_entry_rcu(tg, &task_groups, list) { 6506 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 6507 6508 if (!cfs_rq->runtime_enabled) 6509 continue; 6510 6511 /* 6512 * clock_task is not advancing so we just need to make sure 6513 * there's some valid quota amount 6514 */ 6515 cfs_rq->runtime_remaining = 1; 6516 /* 6517 * Offline rq is schedulable till CPU is completely disabled 6518 * in take_cpu_down(), so we prevent new cfs throttling here. 6519 */ 6520 cfs_rq->runtime_enabled = 0; 6521 6522 if (cfs_rq_throttled(cfs_rq)) 6523 unthrottle_cfs_rq(cfs_rq); 6524 } 6525 rcu_read_unlock(); 6526 6527 rq_clock_stop_loop_update(rq); 6528 } 6529 6530 bool cfs_task_bw_constrained(struct task_struct *p) 6531 { 6532 struct cfs_rq *cfs_rq = task_cfs_rq(p); 6533 6534 if (!cfs_bandwidth_used()) 6535 return false; 6536 6537 if (cfs_rq->runtime_enabled || 6538 tg_cfs_bandwidth(cfs_rq->tg)->hierarchical_quota != RUNTIME_INF) 6539 return true; 6540 6541 return false; 6542 } 6543 6544 #ifdef CONFIG_NO_HZ_FULL 6545 /* called from pick_next_task_fair() */ 6546 static void sched_fair_update_stop_tick(struct rq *rq, struct task_struct *p) 6547 { 6548 int cpu = cpu_of(rq); 6549 6550 if (!sched_feat(HZ_BW) || !cfs_bandwidth_used()) 6551 return; 6552 6553 if (!tick_nohz_full_cpu(cpu)) 6554 return; 6555 6556 if (rq->nr_running != 1) 6557 return; 6558 6559 /* 6560 * We know there is only one task runnable and we've just picked it. The 6561 * normal enqueue path will have cleared TICK_DEP_BIT_SCHED if we will 6562 * be otherwise able to stop the tick. Just need to check if we are using 6563 * bandwidth control. 6564 */ 6565 if (cfs_task_bw_constrained(p)) 6566 tick_nohz_dep_set_cpu(cpu, TICK_DEP_BIT_SCHED); 6567 } 6568 #endif 6569 6570 #else /* CONFIG_CFS_BANDWIDTH */ 6571 6572 static inline bool cfs_bandwidth_used(void) 6573 { 6574 return false; 6575 } 6576 6577 static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) {} 6578 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) { return false; } 6579 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {} 6580 static inline void sync_throttle(struct task_group *tg, int cpu) {} 6581 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} 6582 6583 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) 6584 { 6585 return 0; 6586 } 6587 6588 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) 6589 { 6590 return 0; 6591 } 6592 6593 static inline int throttled_lb_pair(struct task_group *tg, 6594 int src_cpu, int dest_cpu) 6595 { 6596 return 0; 6597 } 6598 6599 #ifdef CONFIG_FAIR_GROUP_SCHED 6600 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b, struct cfs_bandwidth *parent) {} 6601 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} 6602 #endif 6603 6604 static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) 6605 { 6606 return NULL; 6607 } 6608 static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {} 6609 static inline void update_runtime_enabled(struct rq *rq) {} 6610 static inline void unthrottle_offline_cfs_rqs(struct rq *rq) {} 6611 #ifdef CONFIG_CGROUP_SCHED 6612 bool cfs_task_bw_constrained(struct task_struct *p) 6613 { 6614 return false; 6615 } 6616 #endif 6617 #endif /* CONFIG_CFS_BANDWIDTH */ 6618 6619 #if !defined(CONFIG_CFS_BANDWIDTH) || !defined(CONFIG_NO_HZ_FULL) 6620 static inline void sched_fair_update_stop_tick(struct rq *rq, struct task_struct *p) {} 6621 #endif 6622 6623 /************************************************** 6624 * CFS operations on tasks: 6625 */ 6626 6627 #ifdef CONFIG_SCHED_HRTICK 6628 static void hrtick_start_fair(struct rq *rq, struct task_struct *p) 6629 { 6630 struct sched_entity *se = &p->se; 6631 6632 SCHED_WARN_ON(task_rq(p) != rq); 6633 6634 if (rq->cfs.h_nr_running > 1) { 6635 u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime; 6636 u64 slice = se->slice; 6637 s64 delta = slice - ran; 6638 6639 if (delta < 0) { 6640 if (task_current(rq, p)) 6641 resched_curr(rq); 6642 return; 6643 } 6644 hrtick_start(rq, delta); 6645 } 6646 } 6647 6648 /* 6649 * called from enqueue/dequeue and updates the hrtick when the 6650 * current task is from our class and nr_running is low enough 6651 * to matter. 6652 */ 6653 static void hrtick_update(struct rq *rq) 6654 { 6655 struct task_struct *curr = rq->curr; 6656 6657 if (!hrtick_enabled_fair(rq) || curr->sched_class != &fair_sched_class) 6658 return; 6659 6660 hrtick_start_fair(rq, curr); 6661 } 6662 #else /* !CONFIG_SCHED_HRTICK */ 6663 static inline void 6664 hrtick_start_fair(struct rq *rq, struct task_struct *p) 6665 { 6666 } 6667 6668 static inline void hrtick_update(struct rq *rq) 6669 { 6670 } 6671 #endif 6672 6673 #ifdef CONFIG_SMP 6674 static inline bool cpu_overutilized(int cpu) 6675 { 6676 unsigned long rq_util_min, rq_util_max; 6677 6678 if (!sched_energy_enabled()) 6679 return false; 6680 6681 rq_util_min = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MIN); 6682 rq_util_max = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MAX); 6683 6684 /* Return true only if the utilization doesn't fit CPU's capacity */ 6685 return !util_fits_cpu(cpu_util_cfs(cpu), rq_util_min, rq_util_max, cpu); 6686 } 6687 6688 static inline void set_rd_overutilized_status(struct root_domain *rd, 6689 unsigned int status) 6690 { 6691 if (!sched_energy_enabled()) 6692 return; 6693 6694 WRITE_ONCE(rd->overutilized, status); 6695 trace_sched_overutilized_tp(rd, !!status); 6696 } 6697 6698 static inline void check_update_overutilized_status(struct rq *rq) 6699 { 6700 /* 6701 * overutilized field is used for load balancing decisions only 6702 * if energy aware scheduler is being used 6703 */ 6704 if (!sched_energy_enabled()) 6705 return; 6706 6707 if (!READ_ONCE(rq->rd->overutilized) && cpu_overutilized(rq->cpu)) 6708 set_rd_overutilized_status(rq->rd, SG_OVERUTILIZED); 6709 } 6710 #else 6711 static inline void check_update_overutilized_status(struct rq *rq) { } 6712 #endif 6713 6714 /* Runqueue only has SCHED_IDLE tasks enqueued */ 6715 static int sched_idle_rq(struct rq *rq) 6716 { 6717 return unlikely(rq->nr_running == rq->cfs.idle_h_nr_running && 6718 rq->nr_running); 6719 } 6720 6721 #ifdef CONFIG_SMP 6722 static int sched_idle_cpu(int cpu) 6723 { 6724 return sched_idle_rq(cpu_rq(cpu)); 6725 } 6726 #endif 6727 6728 /* 6729 * The enqueue_task method is called before nr_running is 6730 * increased. Here we update the fair scheduling stats and 6731 * then put the task into the rbtree: 6732 */ 6733 static void 6734 enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) 6735 { 6736 struct cfs_rq *cfs_rq; 6737 struct sched_entity *se = &p->se; 6738 int idle_h_nr_running = task_has_idle_policy(p); 6739 int task_new = !(flags & ENQUEUE_WAKEUP); 6740 6741 /* 6742 * The code below (indirectly) updates schedutil which looks at 6743 * the cfs_rq utilization to select a frequency. 6744 * Let's add the task's estimated utilization to the cfs_rq's 6745 * estimated utilization, before we update schedutil. 6746 */ 6747 util_est_enqueue(&rq->cfs, p); 6748 6749 /* 6750 * If in_iowait is set, the code below may not trigger any cpufreq 6751 * utilization updates, so do it here explicitly with the IOWAIT flag 6752 * passed. 6753 */ 6754 if (p->in_iowait) 6755 cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT); 6756 6757 for_each_sched_entity(se) { 6758 if (se->on_rq) 6759 break; 6760 cfs_rq = cfs_rq_of(se); 6761 enqueue_entity(cfs_rq, se, flags); 6762 6763 cfs_rq->h_nr_running++; 6764 cfs_rq->idle_h_nr_running += idle_h_nr_running; 6765 6766 if (cfs_rq_is_idle(cfs_rq)) 6767 idle_h_nr_running = 1; 6768 6769 /* end evaluation on encountering a throttled cfs_rq */ 6770 if (cfs_rq_throttled(cfs_rq)) 6771 goto enqueue_throttle; 6772 6773 flags = ENQUEUE_WAKEUP; 6774 } 6775 6776 for_each_sched_entity(se) { 6777 cfs_rq = cfs_rq_of(se); 6778 6779 update_load_avg(cfs_rq, se, UPDATE_TG); 6780 se_update_runnable(se); 6781 update_cfs_group(se); 6782 6783 cfs_rq->h_nr_running++; 6784 cfs_rq->idle_h_nr_running += idle_h_nr_running; 6785 6786 if (cfs_rq_is_idle(cfs_rq)) 6787 idle_h_nr_running = 1; 6788 6789 /* end evaluation on encountering a throttled cfs_rq */ 6790 if (cfs_rq_throttled(cfs_rq)) 6791 goto enqueue_throttle; 6792 } 6793 6794 /* At this point se is NULL and we are at root level*/ 6795 add_nr_running(rq, 1); 6796 6797 /* 6798 * Since new tasks are assigned an initial util_avg equal to 6799 * half of the spare capacity of their CPU, tiny tasks have the 6800 * ability to cross the overutilized threshold, which will 6801 * result in the load balancer ruining all the task placement 6802 * done by EAS. As a way to mitigate that effect, do not account 6803 * for the first enqueue operation of new tasks during the 6804 * overutilized flag detection. 6805 * 6806 * A better way of solving this problem would be to wait for 6807 * the PELT signals of tasks to converge before taking them 6808 * into account, but that is not straightforward to implement, 6809 * and the following generally works well enough in practice. 6810 */ 6811 if (!task_new) 6812 check_update_overutilized_status(rq); 6813 6814 enqueue_throttle: 6815 assert_list_leaf_cfs_rq(rq); 6816 6817 hrtick_update(rq); 6818 } 6819 6820 static void set_next_buddy(struct sched_entity *se); 6821 6822 /* 6823 * The dequeue_task method is called before nr_running is 6824 * decreased. We remove the task from the rbtree and 6825 * update the fair scheduling stats: 6826 */ 6827 static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) 6828 { 6829 struct cfs_rq *cfs_rq; 6830 struct sched_entity *se = &p->se; 6831 int task_sleep = flags & DEQUEUE_SLEEP; 6832 int idle_h_nr_running = task_has_idle_policy(p); 6833 bool was_sched_idle = sched_idle_rq(rq); 6834 6835 util_est_dequeue(&rq->cfs, p); 6836 6837 for_each_sched_entity(se) { 6838 cfs_rq = cfs_rq_of(se); 6839 dequeue_entity(cfs_rq, se, flags); 6840 6841 cfs_rq->h_nr_running--; 6842 cfs_rq->idle_h_nr_running -= idle_h_nr_running; 6843 6844 if (cfs_rq_is_idle(cfs_rq)) 6845 idle_h_nr_running = 1; 6846 6847 /* end evaluation on encountering a throttled cfs_rq */ 6848 if (cfs_rq_throttled(cfs_rq)) 6849 goto dequeue_throttle; 6850 6851 /* Don't dequeue parent if it has other entities besides us */ 6852 if (cfs_rq->load.weight) { 6853 /* Avoid re-evaluating load for this entity: */ 6854 se = parent_entity(se); 6855 /* 6856 * Bias pick_next to pick a task from this cfs_rq, as 6857 * p is sleeping when it is within its sched_slice. 6858 */ 6859 if (task_sleep && se && !throttled_hierarchy(cfs_rq)) 6860 set_next_buddy(se); 6861 break; 6862 } 6863 flags |= DEQUEUE_SLEEP; 6864 } 6865 6866 for_each_sched_entity(se) { 6867 cfs_rq = cfs_rq_of(se); 6868 6869 update_load_avg(cfs_rq, se, UPDATE_TG); 6870 se_update_runnable(se); 6871 update_cfs_group(se); 6872 6873 cfs_rq->h_nr_running--; 6874 cfs_rq->idle_h_nr_running -= idle_h_nr_running; 6875 6876 if (cfs_rq_is_idle(cfs_rq)) 6877 idle_h_nr_running = 1; 6878 6879 /* end evaluation on encountering a throttled cfs_rq */ 6880 if (cfs_rq_throttled(cfs_rq)) 6881 goto dequeue_throttle; 6882 6883 } 6884 6885 /* At this point se is NULL and we are at root level*/ 6886 sub_nr_running(rq, 1); 6887 6888 /* balance early to pull high priority tasks */ 6889 if (unlikely(!was_sched_idle && sched_idle_rq(rq))) 6890 rq->next_balance = jiffies; 6891 6892 dequeue_throttle: 6893 util_est_update(&rq->cfs, p, task_sleep); 6894 hrtick_update(rq); 6895 } 6896 6897 #ifdef CONFIG_SMP 6898 6899 /* Working cpumask for: load_balance, load_balance_newidle. */ 6900 static DEFINE_PER_CPU(cpumask_var_t, load_balance_mask); 6901 static DEFINE_PER_CPU(cpumask_var_t, select_rq_mask); 6902 static DEFINE_PER_CPU(cpumask_var_t, should_we_balance_tmpmask); 6903 6904 #ifdef CONFIG_NO_HZ_COMMON 6905 6906 static struct { 6907 cpumask_var_t idle_cpus_mask; 6908 atomic_t nr_cpus; 6909 int has_blocked; /* Idle CPUS has blocked load */ 6910 int needs_update; /* Newly idle CPUs need their next_balance collated */ 6911 unsigned long next_balance; /* in jiffy units */ 6912 unsigned long next_blocked; /* Next update of blocked load in jiffies */ 6913 } nohz ____cacheline_aligned; 6914 6915 #endif /* CONFIG_NO_HZ_COMMON */ 6916 6917 static unsigned long cpu_load(struct rq *rq) 6918 { 6919 return cfs_rq_load_avg(&rq->cfs); 6920 } 6921 6922 /* 6923 * cpu_load_without - compute CPU load without any contributions from *p 6924 * @cpu: the CPU which load is requested 6925 * @p: the task which load should be discounted 6926 * 6927 * The load of a CPU is defined by the load of tasks currently enqueued on that 6928 * CPU as well as tasks which are currently sleeping after an execution on that 6929 * CPU. 6930 * 6931 * This method returns the load of the specified CPU by discounting the load of 6932 * the specified task, whenever the task is currently contributing to the CPU 6933 * load. 6934 */ 6935 static unsigned long cpu_load_without(struct rq *rq, struct task_struct *p) 6936 { 6937 struct cfs_rq *cfs_rq; 6938 unsigned int load; 6939 6940 /* Task has no contribution or is new */ 6941 if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 6942 return cpu_load(rq); 6943 6944 cfs_rq = &rq->cfs; 6945 load = READ_ONCE(cfs_rq->avg.load_avg); 6946 6947 /* Discount task's util from CPU's util */ 6948 lsub_positive(&load, task_h_load(p)); 6949 6950 return load; 6951 } 6952 6953 static unsigned long cpu_runnable(struct rq *rq) 6954 { 6955 return cfs_rq_runnable_avg(&rq->cfs); 6956 } 6957 6958 static unsigned long cpu_runnable_without(struct rq *rq, struct task_struct *p) 6959 { 6960 struct cfs_rq *cfs_rq; 6961 unsigned int runnable; 6962 6963 /* Task has no contribution or is new */ 6964 if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 6965 return cpu_runnable(rq); 6966 6967 cfs_rq = &rq->cfs; 6968 runnable = READ_ONCE(cfs_rq->avg.runnable_avg); 6969 6970 /* Discount task's runnable from CPU's runnable */ 6971 lsub_positive(&runnable, p->se.avg.runnable_avg); 6972 6973 return runnable; 6974 } 6975 6976 static unsigned long capacity_of(int cpu) 6977 { 6978 return cpu_rq(cpu)->cpu_capacity; 6979 } 6980 6981 static void record_wakee(struct task_struct *p) 6982 { 6983 /* 6984 * Only decay a single time; tasks that have less then 1 wakeup per 6985 * jiffy will not have built up many flips. 6986 */ 6987 if (time_after(jiffies, current->wakee_flip_decay_ts + HZ)) { 6988 current->wakee_flips >>= 1; 6989 current->wakee_flip_decay_ts = jiffies; 6990 } 6991 6992 if (current->last_wakee != p) { 6993 current->last_wakee = p; 6994 current->wakee_flips++; 6995 } 6996 } 6997 6998 /* 6999 * Detect M:N waker/wakee relationships via a switching-frequency heuristic. 7000 * 7001 * A waker of many should wake a different task than the one last awakened 7002 * at a frequency roughly N times higher than one of its wakees. 7003 * 7004 * In order to determine whether we should let the load spread vs consolidating 7005 * to shared cache, we look for a minimum 'flip' frequency of llc_size in one 7006 * partner, and a factor of lls_size higher frequency in the other. 7007 * 7008 * With both conditions met, we can be relatively sure that the relationship is 7009 * non-monogamous, with partner count exceeding socket size. 7010 * 7011 * Waker/wakee being client/server, worker/dispatcher, interrupt source or 7012 * whatever is irrelevant, spread criteria is apparent partner count exceeds 7013 * socket size. 7014 */ 7015 static int wake_wide(struct task_struct *p) 7016 { 7017 unsigned int master = current->wakee_flips; 7018 unsigned int slave = p->wakee_flips; 7019 int factor = __this_cpu_read(sd_llc_size); 7020 7021 if (master < slave) 7022 swap(master, slave); 7023 if (slave < factor || master < slave * factor) 7024 return 0; 7025 return 1; 7026 } 7027 7028 /* 7029 * The purpose of wake_affine() is to quickly determine on which CPU we can run 7030 * soonest. For the purpose of speed we only consider the waking and previous 7031 * CPU. 7032 * 7033 * wake_affine_idle() - only considers 'now', it check if the waking CPU is 7034 * cache-affine and is (or will be) idle. 7035 * 7036 * wake_affine_weight() - considers the weight to reflect the average 7037 * scheduling latency of the CPUs. This seems to work 7038 * for the overloaded case. 7039 */ 7040 static int 7041 wake_affine_idle(int this_cpu, int prev_cpu, int sync) 7042 { 7043 /* 7044 * If this_cpu is idle, it implies the wakeup is from interrupt 7045 * context. Only allow the move if cache is shared. Otherwise an 7046 * interrupt intensive workload could force all tasks onto one 7047 * node depending on the IO topology or IRQ affinity settings. 7048 * 7049 * If the prev_cpu is idle and cache affine then avoid a migration. 7050 * There is no guarantee that the cache hot data from an interrupt 7051 * is more important than cache hot data on the prev_cpu and from 7052 * a cpufreq perspective, it's better to have higher utilisation 7053 * on one CPU. 7054 */ 7055 if (available_idle_cpu(this_cpu) && cpus_share_cache(this_cpu, prev_cpu)) 7056 return available_idle_cpu(prev_cpu) ? prev_cpu : this_cpu; 7057 7058 if (sync && cpu_rq(this_cpu)->nr_running == 1) 7059 return this_cpu; 7060 7061 if (available_idle_cpu(prev_cpu)) 7062 return prev_cpu; 7063 7064 return nr_cpumask_bits; 7065 } 7066 7067 static int 7068 wake_affine_weight(struct sched_domain *sd, struct task_struct *p, 7069 int this_cpu, int prev_cpu, int sync) 7070 { 7071 s64 this_eff_load, prev_eff_load; 7072 unsigned long task_load; 7073 7074 this_eff_load = cpu_load(cpu_rq(this_cpu)); 7075 7076 if (sync) { 7077 unsigned long current_load = task_h_load(current); 7078 7079 if (current_load > this_eff_load) 7080 return this_cpu; 7081 7082 this_eff_load -= current_load; 7083 } 7084 7085 task_load = task_h_load(p); 7086 7087 this_eff_load += task_load; 7088 if (sched_feat(WA_BIAS)) 7089 this_eff_load *= 100; 7090 this_eff_load *= capacity_of(prev_cpu); 7091 7092 prev_eff_load = cpu_load(cpu_rq(prev_cpu)); 7093 prev_eff_load -= task_load; 7094 if (sched_feat(WA_BIAS)) 7095 prev_eff_load *= 100 + (sd->imbalance_pct - 100) / 2; 7096 prev_eff_load *= capacity_of(this_cpu); 7097 7098 /* 7099 * If sync, adjust the weight of prev_eff_load such that if 7100 * prev_eff == this_eff that select_idle_sibling() will consider 7101 * stacking the wakee on top of the waker if no other CPU is 7102 * idle. 7103 */ 7104 if (sync) 7105 prev_eff_load += 1; 7106 7107 return this_eff_load < prev_eff_load ? this_cpu : nr_cpumask_bits; 7108 } 7109 7110 static int wake_affine(struct sched_domain *sd, struct task_struct *p, 7111 int this_cpu, int prev_cpu, int sync) 7112 { 7113 int target = nr_cpumask_bits; 7114 7115 if (sched_feat(WA_IDLE)) 7116 target = wake_affine_idle(this_cpu, prev_cpu, sync); 7117 7118 if (sched_feat(WA_WEIGHT) && target == nr_cpumask_bits) 7119 target = wake_affine_weight(sd, p, this_cpu, prev_cpu, sync); 7120 7121 schedstat_inc(p->stats.nr_wakeups_affine_attempts); 7122 if (target != this_cpu) 7123 return prev_cpu; 7124 7125 schedstat_inc(sd->ttwu_move_affine); 7126 schedstat_inc(p->stats.nr_wakeups_affine); 7127 return target; 7128 } 7129 7130 static struct sched_group * 7131 find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu); 7132 7133 /* 7134 * find_idlest_group_cpu - find the idlest CPU among the CPUs in the group. 7135 */ 7136 static int 7137 find_idlest_group_cpu(struct sched_group *group, struct task_struct *p, int this_cpu) 7138 { 7139 unsigned long load, min_load = ULONG_MAX; 7140 unsigned int min_exit_latency = UINT_MAX; 7141 u64 latest_idle_timestamp = 0; 7142 int least_loaded_cpu = this_cpu; 7143 int shallowest_idle_cpu = -1; 7144 int i; 7145 7146 /* Check if we have any choice: */ 7147 if (group->group_weight == 1) 7148 return cpumask_first(sched_group_span(group)); 7149 7150 /* Traverse only the allowed CPUs */ 7151 for_each_cpu_and(i, sched_group_span(group), p->cpus_ptr) { 7152 struct rq *rq = cpu_rq(i); 7153 7154 if (!sched_core_cookie_match(rq, p)) 7155 continue; 7156 7157 if (sched_idle_cpu(i)) 7158 return i; 7159 7160 if (available_idle_cpu(i)) { 7161 struct cpuidle_state *idle = idle_get_state(rq); 7162 if (idle && idle->exit_latency < min_exit_latency) { 7163 /* 7164 * We give priority to a CPU whose idle state 7165 * has the smallest exit latency irrespective 7166 * of any idle timestamp. 7167 */ 7168 min_exit_latency = idle->exit_latency; 7169 latest_idle_timestamp = rq->idle_stamp; 7170 shallowest_idle_cpu = i; 7171 } else if ((!idle || idle->exit_latency == min_exit_latency) && 7172 rq->idle_stamp > latest_idle_timestamp) { 7173 /* 7174 * If equal or no active idle state, then 7175 * the most recently idled CPU might have 7176 * a warmer cache. 7177 */ 7178 latest_idle_timestamp = rq->idle_stamp; 7179 shallowest_idle_cpu = i; 7180 } 7181 } else if (shallowest_idle_cpu == -1) { 7182 load = cpu_load(cpu_rq(i)); 7183 if (load < min_load) { 7184 min_load = load; 7185 least_loaded_cpu = i; 7186 } 7187 } 7188 } 7189 7190 return shallowest_idle_cpu != -1 ? shallowest_idle_cpu : least_loaded_cpu; 7191 } 7192 7193 static inline int find_idlest_cpu(struct sched_domain *sd, struct task_struct *p, 7194 int cpu, int prev_cpu, int sd_flag) 7195 { 7196 int new_cpu = cpu; 7197 7198 if (!cpumask_intersects(sched_domain_span(sd), p->cpus_ptr)) 7199 return prev_cpu; 7200 7201 /* 7202 * We need task's util for cpu_util_without, sync it up to 7203 * prev_cpu's last_update_time. 7204 */ 7205 if (!(sd_flag & SD_BALANCE_FORK)) 7206 sync_entity_load_avg(&p->se); 7207 7208 while (sd) { 7209 struct sched_group *group; 7210 struct sched_domain *tmp; 7211 int weight; 7212 7213 if (!(sd->flags & sd_flag)) { 7214 sd = sd->child; 7215 continue; 7216 } 7217 7218 group = find_idlest_group(sd, p, cpu); 7219 if (!group) { 7220 sd = sd->child; 7221 continue; 7222 } 7223 7224 new_cpu = find_idlest_group_cpu(group, p, cpu); 7225 if (new_cpu == cpu) { 7226 /* Now try balancing at a lower domain level of 'cpu': */ 7227 sd = sd->child; 7228 continue; 7229 } 7230 7231 /* Now try balancing at a lower domain level of 'new_cpu': */ 7232 cpu = new_cpu; 7233 weight = sd->span_weight; 7234 sd = NULL; 7235 for_each_domain(cpu, tmp) { 7236 if (weight <= tmp->span_weight) 7237 break; 7238 if (tmp->flags & sd_flag) 7239 sd = tmp; 7240 } 7241 } 7242 7243 return new_cpu; 7244 } 7245 7246 static inline int __select_idle_cpu(int cpu, struct task_struct *p) 7247 { 7248 if ((available_idle_cpu(cpu) || sched_idle_cpu(cpu)) && 7249 sched_cpu_cookie_match(cpu_rq(cpu), p)) 7250 return cpu; 7251 7252 return -1; 7253 } 7254 7255 #ifdef CONFIG_SCHED_SMT 7256 DEFINE_STATIC_KEY_FALSE(sched_smt_present); 7257 EXPORT_SYMBOL_GPL(sched_smt_present); 7258 7259 static inline void set_idle_cores(int cpu, int val) 7260 { 7261 struct sched_domain_shared *sds; 7262 7263 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); 7264 if (sds) 7265 WRITE_ONCE(sds->has_idle_cores, val); 7266 } 7267 7268 static inline bool test_idle_cores(int cpu) 7269 { 7270 struct sched_domain_shared *sds; 7271 7272 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); 7273 if (sds) 7274 return READ_ONCE(sds->has_idle_cores); 7275 7276 return false; 7277 } 7278 7279 /* 7280 * Scans the local SMT mask to see if the entire core is idle, and records this 7281 * information in sd_llc_shared->has_idle_cores. 7282 * 7283 * Since SMT siblings share all cache levels, inspecting this limited remote 7284 * state should be fairly cheap. 7285 */ 7286 void __update_idle_core(struct rq *rq) 7287 { 7288 int core = cpu_of(rq); 7289 int cpu; 7290 7291 rcu_read_lock(); 7292 if (test_idle_cores(core)) 7293 goto unlock; 7294 7295 for_each_cpu(cpu, cpu_smt_mask(core)) { 7296 if (cpu == core) 7297 continue; 7298 7299 if (!available_idle_cpu(cpu)) 7300 goto unlock; 7301 } 7302 7303 set_idle_cores(core, 1); 7304 unlock: 7305 rcu_read_unlock(); 7306 } 7307 7308 /* 7309 * Scan the entire LLC domain for idle cores; this dynamically switches off if 7310 * there are no idle cores left in the system; tracked through 7311 * sd_llc->shared->has_idle_cores and enabled through update_idle_core() above. 7312 */ 7313 static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu) 7314 { 7315 bool idle = true; 7316 int cpu; 7317 7318 for_each_cpu(cpu, cpu_smt_mask(core)) { 7319 if (!available_idle_cpu(cpu)) { 7320 idle = false; 7321 if (*idle_cpu == -1) { 7322 if (sched_idle_cpu(cpu) && cpumask_test_cpu(cpu, cpus)) { 7323 *idle_cpu = cpu; 7324 break; 7325 } 7326 continue; 7327 } 7328 break; 7329 } 7330 if (*idle_cpu == -1 && cpumask_test_cpu(cpu, cpus)) 7331 *idle_cpu = cpu; 7332 } 7333 7334 if (idle) 7335 return core; 7336 7337 cpumask_andnot(cpus, cpus, cpu_smt_mask(core)); 7338 return -1; 7339 } 7340 7341 /* 7342 * Scan the local SMT mask for idle CPUs. 7343 */ 7344 static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target) 7345 { 7346 int cpu; 7347 7348 for_each_cpu_and(cpu, cpu_smt_mask(target), p->cpus_ptr) { 7349 if (cpu == target) 7350 continue; 7351 /* 7352 * Check if the CPU is in the LLC scheduling domain of @target. 7353 * Due to isolcpus, there is no guarantee that all the siblings are in the domain. 7354 */ 7355 if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) 7356 continue; 7357 if (available_idle_cpu(cpu) || sched_idle_cpu(cpu)) 7358 return cpu; 7359 } 7360 7361 return -1; 7362 } 7363 7364 #else /* CONFIG_SCHED_SMT */ 7365 7366 static inline void set_idle_cores(int cpu, int val) 7367 { 7368 } 7369 7370 static inline bool test_idle_cores(int cpu) 7371 { 7372 return false; 7373 } 7374 7375 static inline int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu) 7376 { 7377 return __select_idle_cpu(core, p); 7378 } 7379 7380 static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target) 7381 { 7382 return -1; 7383 } 7384 7385 #endif /* CONFIG_SCHED_SMT */ 7386 7387 /* 7388 * Scan the LLC domain for idle CPUs; this is dynamically regulated by 7389 * comparing the average scan cost (tracked in sd->avg_scan_cost) against the 7390 * average idle time for this rq (as found in rq->avg_idle). 7391 */ 7392 static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool has_idle_core, int target) 7393 { 7394 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 7395 int i, cpu, idle_cpu = -1, nr = INT_MAX; 7396 struct sched_domain_shared *sd_share; 7397 struct rq *this_rq = this_rq(); 7398 int this = smp_processor_id(); 7399 struct sched_domain *this_sd = NULL; 7400 u64 time = 0; 7401 7402 cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr); 7403 7404 if (sched_feat(SIS_PROP) && !has_idle_core) { 7405 u64 avg_cost, avg_idle, span_avg; 7406 unsigned long now = jiffies; 7407 7408 this_sd = rcu_dereference(*this_cpu_ptr(&sd_llc)); 7409 if (!this_sd) 7410 return -1; 7411 7412 /* 7413 * If we're busy, the assumption that the last idle period 7414 * predicts the future is flawed; age away the remaining 7415 * predicted idle time. 7416 */ 7417 if (unlikely(this_rq->wake_stamp < now)) { 7418 while (this_rq->wake_stamp < now && this_rq->wake_avg_idle) { 7419 this_rq->wake_stamp++; 7420 this_rq->wake_avg_idle >>= 1; 7421 } 7422 } 7423 7424 avg_idle = this_rq->wake_avg_idle; 7425 avg_cost = this_sd->avg_scan_cost + 1; 7426 7427 span_avg = sd->span_weight * avg_idle; 7428 if (span_avg > 4*avg_cost) 7429 nr = div_u64(span_avg, avg_cost); 7430 else 7431 nr = 4; 7432 7433 time = cpu_clock(this); 7434 } 7435 7436 if (sched_feat(SIS_UTIL)) { 7437 sd_share = rcu_dereference(per_cpu(sd_llc_shared, target)); 7438 if (sd_share) { 7439 /* because !--nr is the condition to stop scan */ 7440 nr = READ_ONCE(sd_share->nr_idle_scan) + 1; 7441 /* overloaded LLC is unlikely to have idle cpu/core */ 7442 if (nr == 1) 7443 return -1; 7444 } 7445 } 7446 7447 for_each_cpu_wrap(cpu, cpus, target + 1) { 7448 if (has_idle_core) { 7449 i = select_idle_core(p, cpu, cpus, &idle_cpu); 7450 if ((unsigned int)i < nr_cpumask_bits) 7451 return i; 7452 7453 } else { 7454 if (!--nr) 7455 return -1; 7456 idle_cpu = __select_idle_cpu(cpu, p); 7457 if ((unsigned int)idle_cpu < nr_cpumask_bits) 7458 break; 7459 } 7460 } 7461 7462 if (has_idle_core) 7463 set_idle_cores(target, false); 7464 7465 if (sched_feat(SIS_PROP) && this_sd && !has_idle_core) { 7466 time = cpu_clock(this) - time; 7467 7468 /* 7469 * Account for the scan cost of wakeups against the average 7470 * idle time. 7471 */ 7472 this_rq->wake_avg_idle -= min(this_rq->wake_avg_idle, time); 7473 7474 update_avg(&this_sd->avg_scan_cost, time); 7475 } 7476 7477 return idle_cpu; 7478 } 7479 7480 /* 7481 * Scan the asym_capacity domain for idle CPUs; pick the first idle one on which 7482 * the task fits. If no CPU is big enough, but there are idle ones, try to 7483 * maximize capacity. 7484 */ 7485 static int 7486 select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) 7487 { 7488 unsigned long task_util, util_min, util_max, best_cap = 0; 7489 int fits, best_fits = 0; 7490 int cpu, best_cpu = -1; 7491 struct cpumask *cpus; 7492 7493 cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 7494 cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr); 7495 7496 task_util = task_util_est(p); 7497 util_min = uclamp_eff_value(p, UCLAMP_MIN); 7498 util_max = uclamp_eff_value(p, UCLAMP_MAX); 7499 7500 for_each_cpu_wrap(cpu, cpus, target) { 7501 unsigned long cpu_cap = capacity_of(cpu); 7502 7503 if (!available_idle_cpu(cpu) && !sched_idle_cpu(cpu)) 7504 continue; 7505 7506 fits = util_fits_cpu(task_util, util_min, util_max, cpu); 7507 7508 /* This CPU fits with all requirements */ 7509 if (fits > 0) 7510 return cpu; 7511 /* 7512 * Only the min performance hint (i.e. uclamp_min) doesn't fit. 7513 * Look for the CPU with best capacity. 7514 */ 7515 else if (fits < 0) 7516 cpu_cap = capacity_orig_of(cpu) - thermal_load_avg(cpu_rq(cpu)); 7517 7518 /* 7519 * First, select CPU which fits better (-1 being better than 0). 7520 * Then, select the one with best capacity at same level. 7521 */ 7522 if ((fits < best_fits) || 7523 ((fits == best_fits) && (cpu_cap > best_cap))) { 7524 best_cap = cpu_cap; 7525 best_cpu = cpu; 7526 best_fits = fits; 7527 } 7528 } 7529 7530 return best_cpu; 7531 } 7532 7533 static inline bool asym_fits_cpu(unsigned long util, 7534 unsigned long util_min, 7535 unsigned long util_max, 7536 int cpu) 7537 { 7538 if (sched_asym_cpucap_active()) 7539 /* 7540 * Return true only if the cpu fully fits the task requirements 7541 * which include the utilization and the performance hints. 7542 */ 7543 return (util_fits_cpu(util, util_min, util_max, cpu) > 0); 7544 7545 return true; 7546 } 7547 7548 /* 7549 * Try and locate an idle core/thread in the LLC cache domain. 7550 */ 7551 static int select_idle_sibling(struct task_struct *p, int prev, int target) 7552 { 7553 bool has_idle_core = false; 7554 struct sched_domain *sd; 7555 unsigned long task_util, util_min, util_max; 7556 int i, recent_used_cpu; 7557 7558 /* 7559 * On asymmetric system, update task utilization because we will check 7560 * that the task fits with cpu's capacity. 7561 */ 7562 if (sched_asym_cpucap_active()) { 7563 sync_entity_load_avg(&p->se); 7564 task_util = task_util_est(p); 7565 util_min = uclamp_eff_value(p, UCLAMP_MIN); 7566 util_max = uclamp_eff_value(p, UCLAMP_MAX); 7567 } 7568 7569 /* 7570 * per-cpu select_rq_mask usage 7571 */ 7572 lockdep_assert_irqs_disabled(); 7573 7574 if ((available_idle_cpu(target) || sched_idle_cpu(target)) && 7575 asym_fits_cpu(task_util, util_min, util_max, target)) 7576 return target; 7577 7578 /* 7579 * If the previous CPU is cache affine and idle, don't be stupid: 7580 */ 7581 if (prev != target && cpus_share_cache(prev, target) && 7582 (available_idle_cpu(prev) || sched_idle_cpu(prev)) && 7583 asym_fits_cpu(task_util, util_min, util_max, prev)) 7584 return prev; 7585 7586 /* 7587 * Allow a per-cpu kthread to stack with the wakee if the 7588 * kworker thread and the tasks previous CPUs are the same. 7589 * The assumption is that the wakee queued work for the 7590 * per-cpu kthread that is now complete and the wakeup is 7591 * essentially a sync wakeup. An obvious example of this 7592 * pattern is IO completions. 7593 */ 7594 if (is_per_cpu_kthread(current) && 7595 in_task() && 7596 prev == smp_processor_id() && 7597 this_rq()->nr_running <= 1 && 7598 asym_fits_cpu(task_util, util_min, util_max, prev)) { 7599 return prev; 7600 } 7601 7602 /* Check a recently used CPU as a potential idle candidate: */ 7603 recent_used_cpu = p->recent_used_cpu; 7604 p->recent_used_cpu = prev; 7605 if (recent_used_cpu != prev && 7606 recent_used_cpu != target && 7607 cpus_share_cache(recent_used_cpu, target) && 7608 (available_idle_cpu(recent_used_cpu) || sched_idle_cpu(recent_used_cpu)) && 7609 cpumask_test_cpu(recent_used_cpu, p->cpus_ptr) && 7610 asym_fits_cpu(task_util, util_min, util_max, recent_used_cpu)) { 7611 return recent_used_cpu; 7612 } 7613 7614 /* 7615 * For asymmetric CPU capacity systems, our domain of interest is 7616 * sd_asym_cpucapacity rather than sd_llc. 7617 */ 7618 if (sched_asym_cpucap_active()) { 7619 sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, target)); 7620 /* 7621 * On an asymmetric CPU capacity system where an exclusive 7622 * cpuset defines a symmetric island (i.e. one unique 7623 * capacity_orig value through the cpuset), the key will be set 7624 * but the CPUs within that cpuset will not have a domain with 7625 * SD_ASYM_CPUCAPACITY. These should follow the usual symmetric 7626 * capacity path. 7627 */ 7628 if (sd) { 7629 i = select_idle_capacity(p, sd, target); 7630 return ((unsigned)i < nr_cpumask_bits) ? i : target; 7631 } 7632 } 7633 7634 sd = rcu_dereference(per_cpu(sd_llc, target)); 7635 if (!sd) 7636 return target; 7637 7638 if (sched_smt_active()) { 7639 has_idle_core = test_idle_cores(target); 7640 7641 if (!has_idle_core && cpus_share_cache(prev, target)) { 7642 i = select_idle_smt(p, sd, prev); 7643 if ((unsigned int)i < nr_cpumask_bits) 7644 return i; 7645 } 7646 } 7647 7648 i = select_idle_cpu(p, sd, has_idle_core, target); 7649 if ((unsigned)i < nr_cpumask_bits) 7650 return i; 7651 7652 return target; 7653 } 7654 7655 /** 7656 * cpu_util() - Estimates the amount of CPU capacity used by CFS tasks. 7657 * @cpu: the CPU to get the utilization for 7658 * @p: task for which the CPU utilization should be predicted or NULL 7659 * @dst_cpu: CPU @p migrates to, -1 if @p moves from @cpu or @p == NULL 7660 * @boost: 1 to enable boosting, otherwise 0 7661 * 7662 * The unit of the return value must be the same as the one of CPU capacity 7663 * so that CPU utilization can be compared with CPU capacity. 7664 * 7665 * CPU utilization is the sum of running time of runnable tasks plus the 7666 * recent utilization of currently non-runnable tasks on that CPU. 7667 * It represents the amount of CPU capacity currently used by CFS tasks in 7668 * the range [0..max CPU capacity] with max CPU capacity being the CPU 7669 * capacity at f_max. 7670 * 7671 * The estimated CPU utilization is defined as the maximum between CPU 7672 * utilization and sum of the estimated utilization of the currently 7673 * runnable tasks on that CPU. It preserves a utilization "snapshot" of 7674 * previously-executed tasks, which helps better deduce how busy a CPU will 7675 * be when a long-sleeping task wakes up. The contribution to CPU utilization 7676 * of such a task would be significantly decayed at this point of time. 7677 * 7678 * Boosted CPU utilization is defined as max(CPU runnable, CPU utilization). 7679 * CPU contention for CFS tasks can be detected by CPU runnable > CPU 7680 * utilization. Boosting is implemented in cpu_util() so that internal 7681 * users (e.g. EAS) can use it next to external users (e.g. schedutil), 7682 * latter via cpu_util_cfs_boost(). 7683 * 7684 * CPU utilization can be higher than the current CPU capacity 7685 * (f_curr/f_max * max CPU capacity) or even the max CPU capacity because 7686 * of rounding errors as well as task migrations or wakeups of new tasks. 7687 * CPU utilization has to be capped to fit into the [0..max CPU capacity] 7688 * range. Otherwise a group of CPUs (CPU0 util = 121% + CPU1 util = 80%) 7689 * could be seen as over-utilized even though CPU1 has 20% of spare CPU 7690 * capacity. CPU utilization is allowed to overshoot current CPU capacity 7691 * though since this is useful for predicting the CPU capacity required 7692 * after task migrations (scheduler-driven DVFS). 7693 * 7694 * Return: (Boosted) (estimated) utilization for the specified CPU. 7695 */ 7696 static unsigned long 7697 cpu_util(int cpu, struct task_struct *p, int dst_cpu, int boost) 7698 { 7699 struct cfs_rq *cfs_rq = &cpu_rq(cpu)->cfs; 7700 unsigned long util = READ_ONCE(cfs_rq->avg.util_avg); 7701 unsigned long runnable; 7702 7703 if (boost) { 7704 runnable = READ_ONCE(cfs_rq->avg.runnable_avg); 7705 util = max(util, runnable); 7706 } 7707 7708 /* 7709 * If @dst_cpu is -1 or @p migrates from @cpu to @dst_cpu remove its 7710 * contribution. If @p migrates from another CPU to @cpu add its 7711 * contribution. In all the other cases @cpu is not impacted by the 7712 * migration so its util_avg is already correct. 7713 */ 7714 if (p && task_cpu(p) == cpu && dst_cpu != cpu) 7715 lsub_positive(&util, task_util(p)); 7716 else if (p && task_cpu(p) != cpu && dst_cpu == cpu) 7717 util += task_util(p); 7718 7719 if (sched_feat(UTIL_EST)) { 7720 unsigned long util_est; 7721 7722 util_est = READ_ONCE(cfs_rq->avg.util_est.enqueued); 7723 7724 /* 7725 * During wake-up @p isn't enqueued yet and doesn't contribute 7726 * to any cpu_rq(cpu)->cfs.avg.util_est.enqueued. 7727 * If @dst_cpu == @cpu add it to "simulate" cpu_util after @p 7728 * has been enqueued. 7729 * 7730 * During exec (@dst_cpu = -1) @p is enqueued and does 7731 * contribute to cpu_rq(cpu)->cfs.util_est.enqueued. 7732 * Remove it to "simulate" cpu_util without @p's contribution. 7733 * 7734 * Despite the task_on_rq_queued(@p) check there is still a 7735 * small window for a possible race when an exec 7736 * select_task_rq_fair() races with LB's detach_task(). 7737 * 7738 * detach_task() 7739 * deactivate_task() 7740 * p->on_rq = TASK_ON_RQ_MIGRATING; 7741 * -------------------------------- A 7742 * dequeue_task() \ 7743 * dequeue_task_fair() + Race Time 7744 * util_est_dequeue() / 7745 * -------------------------------- B 7746 * 7747 * The additional check "current == p" is required to further 7748 * reduce the race window. 7749 */ 7750 if (dst_cpu == cpu) 7751 util_est += _task_util_est(p); 7752 else if (p && unlikely(task_on_rq_queued(p) || current == p)) 7753 lsub_positive(&util_est, _task_util_est(p)); 7754 7755 util = max(util, util_est); 7756 } 7757 7758 return min(util, capacity_orig_of(cpu)); 7759 } 7760 7761 unsigned long cpu_util_cfs(int cpu) 7762 { 7763 return cpu_util(cpu, NULL, -1, 0); 7764 } 7765 7766 unsigned long cpu_util_cfs_boost(int cpu) 7767 { 7768 return cpu_util(cpu, NULL, -1, 1); 7769 } 7770 7771 /* 7772 * cpu_util_without: compute cpu utilization without any contributions from *p 7773 * @cpu: the CPU which utilization is requested 7774 * @p: the task which utilization should be discounted 7775 * 7776 * The utilization of a CPU is defined by the utilization of tasks currently 7777 * enqueued on that CPU as well as tasks which are currently sleeping after an 7778 * execution on that CPU. 7779 * 7780 * This method returns the utilization of the specified CPU by discounting the 7781 * utilization of the specified task, whenever the task is currently 7782 * contributing to the CPU utilization. 7783 */ 7784 static unsigned long cpu_util_without(int cpu, struct task_struct *p) 7785 { 7786 /* Task has no contribution or is new */ 7787 if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 7788 p = NULL; 7789 7790 return cpu_util(cpu, p, -1, 0); 7791 } 7792 7793 /* 7794 * energy_env - Utilization landscape for energy estimation. 7795 * @task_busy_time: Utilization contribution by the task for which we test the 7796 * placement. Given by eenv_task_busy_time(). 7797 * @pd_busy_time: Utilization of the whole perf domain without the task 7798 * contribution. Given by eenv_pd_busy_time(). 7799 * @cpu_cap: Maximum CPU capacity for the perf domain. 7800 * @pd_cap: Entire perf domain capacity. (pd->nr_cpus * cpu_cap). 7801 */ 7802 struct energy_env { 7803 unsigned long task_busy_time; 7804 unsigned long pd_busy_time; 7805 unsigned long cpu_cap; 7806 unsigned long pd_cap; 7807 }; 7808 7809 /* 7810 * Compute the task busy time for compute_energy(). This time cannot be 7811 * injected directly into effective_cpu_util() because of the IRQ scaling. 7812 * The latter only makes sense with the most recent CPUs where the task has 7813 * run. 7814 */ 7815 static inline void eenv_task_busy_time(struct energy_env *eenv, 7816 struct task_struct *p, int prev_cpu) 7817 { 7818 unsigned long busy_time, max_cap = arch_scale_cpu_capacity(prev_cpu); 7819 unsigned long irq = cpu_util_irq(cpu_rq(prev_cpu)); 7820 7821 if (unlikely(irq >= max_cap)) 7822 busy_time = max_cap; 7823 else 7824 busy_time = scale_irq_capacity(task_util_est(p), irq, max_cap); 7825 7826 eenv->task_busy_time = busy_time; 7827 } 7828 7829 /* 7830 * Compute the perf_domain (PD) busy time for compute_energy(). Based on the 7831 * utilization for each @pd_cpus, it however doesn't take into account 7832 * clamping since the ratio (utilization / cpu_capacity) is already enough to 7833 * scale the EM reported power consumption at the (eventually clamped) 7834 * cpu_capacity. 7835 * 7836 * The contribution of the task @p for which we want to estimate the 7837 * energy cost is removed (by cpu_util()) and must be calculated 7838 * separately (see eenv_task_busy_time). This ensures: 7839 * 7840 * - A stable PD utilization, no matter which CPU of that PD we want to place 7841 * the task on. 7842 * 7843 * - A fair comparison between CPUs as the task contribution (task_util()) 7844 * will always be the same no matter which CPU utilization we rely on 7845 * (util_avg or util_est). 7846 * 7847 * Set @eenv busy time for the PD that spans @pd_cpus. This busy time can't 7848 * exceed @eenv->pd_cap. 7849 */ 7850 static inline void eenv_pd_busy_time(struct energy_env *eenv, 7851 struct cpumask *pd_cpus, 7852 struct task_struct *p) 7853 { 7854 unsigned long busy_time = 0; 7855 int cpu; 7856 7857 for_each_cpu(cpu, pd_cpus) { 7858 unsigned long util = cpu_util(cpu, p, -1, 0); 7859 7860 busy_time += effective_cpu_util(cpu, util, ENERGY_UTIL, NULL); 7861 } 7862 7863 eenv->pd_busy_time = min(eenv->pd_cap, busy_time); 7864 } 7865 7866 /* 7867 * Compute the maximum utilization for compute_energy() when the task @p 7868 * is placed on the cpu @dst_cpu. 7869 * 7870 * Returns the maximum utilization among @eenv->cpus. This utilization can't 7871 * exceed @eenv->cpu_cap. 7872 */ 7873 static inline unsigned long 7874 eenv_pd_max_util(struct energy_env *eenv, struct cpumask *pd_cpus, 7875 struct task_struct *p, int dst_cpu) 7876 { 7877 unsigned long max_util = 0; 7878 int cpu; 7879 7880 for_each_cpu(cpu, pd_cpus) { 7881 struct task_struct *tsk = (cpu == dst_cpu) ? p : NULL; 7882 unsigned long util = cpu_util(cpu, p, dst_cpu, 1); 7883 unsigned long eff_util; 7884 7885 /* 7886 * Performance domain frequency: utilization clamping 7887 * must be considered since it affects the selection 7888 * of the performance domain frequency. 7889 * NOTE: in case RT tasks are running, by default the 7890 * FREQUENCY_UTIL's utilization can be max OPP. 7891 */ 7892 eff_util = effective_cpu_util(cpu, util, FREQUENCY_UTIL, tsk); 7893 max_util = max(max_util, eff_util); 7894 } 7895 7896 return min(max_util, eenv->cpu_cap); 7897 } 7898 7899 /* 7900 * compute_energy(): Use the Energy Model to estimate the energy that @pd would 7901 * consume for a given utilization landscape @eenv. When @dst_cpu < 0, the task 7902 * contribution is ignored. 7903 */ 7904 static inline unsigned long 7905 compute_energy(struct energy_env *eenv, struct perf_domain *pd, 7906 struct cpumask *pd_cpus, struct task_struct *p, int dst_cpu) 7907 { 7908 unsigned long max_util = eenv_pd_max_util(eenv, pd_cpus, p, dst_cpu); 7909 unsigned long busy_time = eenv->pd_busy_time; 7910 7911 if (dst_cpu >= 0) 7912 busy_time = min(eenv->pd_cap, busy_time + eenv->task_busy_time); 7913 7914 return em_cpu_energy(pd->em_pd, max_util, busy_time, eenv->cpu_cap); 7915 } 7916 7917 /* 7918 * find_energy_efficient_cpu(): Find most energy-efficient target CPU for the 7919 * waking task. find_energy_efficient_cpu() looks for the CPU with maximum 7920 * spare capacity in each performance domain and uses it as a potential 7921 * candidate to execute the task. Then, it uses the Energy Model to figure 7922 * out which of the CPU candidates is the most energy-efficient. 7923 * 7924 * The rationale for this heuristic is as follows. In a performance domain, 7925 * all the most energy efficient CPU candidates (according to the Energy 7926 * Model) are those for which we'll request a low frequency. When there are 7927 * several CPUs for which the frequency request will be the same, we don't 7928 * have enough data to break the tie between them, because the Energy Model 7929 * only includes active power costs. With this model, if we assume that 7930 * frequency requests follow utilization (e.g. using schedutil), the CPU with 7931 * the maximum spare capacity in a performance domain is guaranteed to be among 7932 * the best candidates of the performance domain. 7933 * 7934 * In practice, it could be preferable from an energy standpoint to pack 7935 * small tasks on a CPU in order to let other CPUs go in deeper idle states, 7936 * but that could also hurt our chances to go cluster idle, and we have no 7937 * ways to tell with the current Energy Model if this is actually a good 7938 * idea or not. So, find_energy_efficient_cpu() basically favors 7939 * cluster-packing, and spreading inside a cluster. That should at least be 7940 * a good thing for latency, and this is consistent with the idea that most 7941 * of the energy savings of EAS come from the asymmetry of the system, and 7942 * not so much from breaking the tie between identical CPUs. That's also the 7943 * reason why EAS is enabled in the topology code only for systems where 7944 * SD_ASYM_CPUCAPACITY is set. 7945 * 7946 * NOTE: Forkees are not accepted in the energy-aware wake-up path because 7947 * they don't have any useful utilization data yet and it's not possible to 7948 * forecast their impact on energy consumption. Consequently, they will be 7949 * placed by find_idlest_cpu() on the least loaded CPU, which might turn out 7950 * to be energy-inefficient in some use-cases. The alternative would be to 7951 * bias new tasks towards specific types of CPUs first, or to try to infer 7952 * their util_avg from the parent task, but those heuristics could hurt 7953 * other use-cases too. So, until someone finds a better way to solve this, 7954 * let's keep things simple by re-using the existing slow path. 7955 */ 7956 static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu) 7957 { 7958 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 7959 unsigned long prev_delta = ULONG_MAX, best_delta = ULONG_MAX; 7960 unsigned long p_util_min = uclamp_is_used() ? uclamp_eff_value(p, UCLAMP_MIN) : 0; 7961 unsigned long p_util_max = uclamp_is_used() ? uclamp_eff_value(p, UCLAMP_MAX) : 1024; 7962 struct root_domain *rd = this_rq()->rd; 7963 int cpu, best_energy_cpu, target = -1; 7964 int prev_fits = -1, best_fits = -1; 7965 unsigned long best_thermal_cap = 0; 7966 unsigned long prev_thermal_cap = 0; 7967 struct sched_domain *sd; 7968 struct perf_domain *pd; 7969 struct energy_env eenv; 7970 7971 rcu_read_lock(); 7972 pd = rcu_dereference(rd->pd); 7973 if (!pd || READ_ONCE(rd->overutilized)) 7974 goto unlock; 7975 7976 /* 7977 * Energy-aware wake-up happens on the lowest sched_domain starting 7978 * from sd_asym_cpucapacity spanning over this_cpu and prev_cpu. 7979 */ 7980 sd = rcu_dereference(*this_cpu_ptr(&sd_asym_cpucapacity)); 7981 while (sd && !cpumask_test_cpu(prev_cpu, sched_domain_span(sd))) 7982 sd = sd->parent; 7983 if (!sd) 7984 goto unlock; 7985 7986 target = prev_cpu; 7987 7988 sync_entity_load_avg(&p->se); 7989 if (!task_util_est(p) && p_util_min == 0) 7990 goto unlock; 7991 7992 eenv_task_busy_time(&eenv, p, prev_cpu); 7993 7994 for (; pd; pd = pd->next) { 7995 unsigned long util_min = p_util_min, util_max = p_util_max; 7996 unsigned long cpu_cap, cpu_thermal_cap, util; 7997 long prev_spare_cap = -1, max_spare_cap = -1; 7998 unsigned long rq_util_min, rq_util_max; 7999 unsigned long cur_delta, base_energy; 8000 int max_spare_cap_cpu = -1; 8001 int fits, max_fits = -1; 8002 8003 cpumask_and(cpus, perf_domain_span(pd), cpu_online_mask); 8004 8005 if (cpumask_empty(cpus)) 8006 continue; 8007 8008 /* Account thermal pressure for the energy estimation */ 8009 cpu = cpumask_first(cpus); 8010 cpu_thermal_cap = arch_scale_cpu_capacity(cpu); 8011 cpu_thermal_cap -= arch_scale_thermal_pressure(cpu); 8012 8013 eenv.cpu_cap = cpu_thermal_cap; 8014 eenv.pd_cap = 0; 8015 8016 for_each_cpu(cpu, cpus) { 8017 struct rq *rq = cpu_rq(cpu); 8018 8019 eenv.pd_cap += cpu_thermal_cap; 8020 8021 if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) 8022 continue; 8023 8024 if (!cpumask_test_cpu(cpu, p->cpus_ptr)) 8025 continue; 8026 8027 util = cpu_util(cpu, p, cpu, 0); 8028 cpu_cap = capacity_of(cpu); 8029 8030 /* 8031 * Skip CPUs that cannot satisfy the capacity request. 8032 * IOW, placing the task there would make the CPU 8033 * overutilized. Take uclamp into account to see how 8034 * much capacity we can get out of the CPU; this is 8035 * aligned with sched_cpu_util(). 8036 */ 8037 if (uclamp_is_used() && !uclamp_rq_is_idle(rq)) { 8038 /* 8039 * Open code uclamp_rq_util_with() except for 8040 * the clamp() part. Ie: apply max aggregation 8041 * only. util_fits_cpu() logic requires to 8042 * operate on non clamped util but must use the 8043 * max-aggregated uclamp_{min, max}. 8044 */ 8045 rq_util_min = uclamp_rq_get(rq, UCLAMP_MIN); 8046 rq_util_max = uclamp_rq_get(rq, UCLAMP_MAX); 8047 8048 util_min = max(rq_util_min, p_util_min); 8049 util_max = max(rq_util_max, p_util_max); 8050 } 8051 8052 fits = util_fits_cpu(util, util_min, util_max, cpu); 8053 if (!fits) 8054 continue; 8055 8056 lsub_positive(&cpu_cap, util); 8057 8058 if (cpu == prev_cpu) { 8059 /* Always use prev_cpu as a candidate. */ 8060 prev_spare_cap = cpu_cap; 8061 prev_fits = fits; 8062 } else if ((fits > max_fits) || 8063 ((fits == max_fits) && ((long)cpu_cap > max_spare_cap))) { 8064 /* 8065 * Find the CPU with the maximum spare capacity 8066 * among the remaining CPUs in the performance 8067 * domain. 8068 */ 8069 max_spare_cap = cpu_cap; 8070 max_spare_cap_cpu = cpu; 8071 max_fits = fits; 8072 } 8073 } 8074 8075 if (max_spare_cap_cpu < 0 && prev_spare_cap < 0) 8076 continue; 8077 8078 eenv_pd_busy_time(&eenv, cpus, p); 8079 /* Compute the 'base' energy of the pd, without @p */ 8080 base_energy = compute_energy(&eenv, pd, cpus, p, -1); 8081 8082 /* Evaluate the energy impact of using prev_cpu. */ 8083 if (prev_spare_cap > -1) { 8084 prev_delta = compute_energy(&eenv, pd, cpus, p, 8085 prev_cpu); 8086 /* CPU utilization has changed */ 8087 if (prev_delta < base_energy) 8088 goto unlock; 8089 prev_delta -= base_energy; 8090 prev_thermal_cap = cpu_thermal_cap; 8091 best_delta = min(best_delta, prev_delta); 8092 } 8093 8094 /* Evaluate the energy impact of using max_spare_cap_cpu. */ 8095 if (max_spare_cap_cpu >= 0 && max_spare_cap > prev_spare_cap) { 8096 /* Current best energy cpu fits better */ 8097 if (max_fits < best_fits) 8098 continue; 8099 8100 /* 8101 * Both don't fit performance hint (i.e. uclamp_min) 8102 * but best energy cpu has better capacity. 8103 */ 8104 if ((max_fits < 0) && 8105 (cpu_thermal_cap <= best_thermal_cap)) 8106 continue; 8107 8108 cur_delta = compute_energy(&eenv, pd, cpus, p, 8109 max_spare_cap_cpu); 8110 /* CPU utilization has changed */ 8111 if (cur_delta < base_energy) 8112 goto unlock; 8113 cur_delta -= base_energy; 8114 8115 /* 8116 * Both fit for the task but best energy cpu has lower 8117 * energy impact. 8118 */ 8119 if ((max_fits > 0) && (best_fits > 0) && 8120 (cur_delta >= best_delta)) 8121 continue; 8122 8123 best_delta = cur_delta; 8124 best_energy_cpu = max_spare_cap_cpu; 8125 best_fits = max_fits; 8126 best_thermal_cap = cpu_thermal_cap; 8127 } 8128 } 8129 rcu_read_unlock(); 8130 8131 if ((best_fits > prev_fits) || 8132 ((best_fits > 0) && (best_delta < prev_delta)) || 8133 ((best_fits < 0) && (best_thermal_cap > prev_thermal_cap))) 8134 target = best_energy_cpu; 8135 8136 return target; 8137 8138 unlock: 8139 rcu_read_unlock(); 8140 8141 return target; 8142 } 8143 8144 /* 8145 * select_task_rq_fair: Select target runqueue for the waking task in domains 8146 * that have the relevant SD flag set. In practice, this is SD_BALANCE_WAKE, 8147 * SD_BALANCE_FORK, or SD_BALANCE_EXEC. 8148 * 8149 * Balances load by selecting the idlest CPU in the idlest group, or under 8150 * certain conditions an idle sibling CPU if the domain has SD_WAKE_AFFINE set. 8151 * 8152 * Returns the target CPU number. 8153 */ 8154 static int 8155 select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags) 8156 { 8157 int sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING); 8158 struct sched_domain *tmp, *sd = NULL; 8159 int cpu = smp_processor_id(); 8160 int new_cpu = prev_cpu; 8161 int want_affine = 0; 8162 /* SD_flags and WF_flags share the first nibble */ 8163 int sd_flag = wake_flags & 0xF; 8164 8165 /* 8166 * required for stable ->cpus_allowed 8167 */ 8168 lockdep_assert_held(&p->pi_lock); 8169 if (wake_flags & WF_TTWU) { 8170 record_wakee(p); 8171 8172 if ((wake_flags & WF_CURRENT_CPU) && 8173 cpumask_test_cpu(cpu, p->cpus_ptr)) 8174 return cpu; 8175 8176 if (sched_energy_enabled()) { 8177 new_cpu = find_energy_efficient_cpu(p, prev_cpu); 8178 if (new_cpu >= 0) 8179 return new_cpu; 8180 new_cpu = prev_cpu; 8181 } 8182 8183 want_affine = !wake_wide(p) && cpumask_test_cpu(cpu, p->cpus_ptr); 8184 } 8185 8186 rcu_read_lock(); 8187 for_each_domain(cpu, tmp) { 8188 /* 8189 * If both 'cpu' and 'prev_cpu' are part of this domain, 8190 * cpu is a valid SD_WAKE_AFFINE target. 8191 */ 8192 if (want_affine && (tmp->flags & SD_WAKE_AFFINE) && 8193 cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) { 8194 if (cpu != prev_cpu) 8195 new_cpu = wake_affine(tmp, p, cpu, prev_cpu, sync); 8196 8197 sd = NULL; /* Prefer wake_affine over balance flags */ 8198 break; 8199 } 8200 8201 /* 8202 * Usually only true for WF_EXEC and WF_FORK, as sched_domains 8203 * usually do not have SD_BALANCE_WAKE set. That means wakeup 8204 * will usually go to the fast path. 8205 */ 8206 if (tmp->flags & sd_flag) 8207 sd = tmp; 8208 else if (!want_affine) 8209 break; 8210 } 8211 8212 if (unlikely(sd)) { 8213 /* Slow path */ 8214 new_cpu = find_idlest_cpu(sd, p, cpu, prev_cpu, sd_flag); 8215 } else if (wake_flags & WF_TTWU) { /* XXX always ? */ 8216 /* Fast path */ 8217 new_cpu = select_idle_sibling(p, prev_cpu, new_cpu); 8218 } 8219 rcu_read_unlock(); 8220 8221 return new_cpu; 8222 } 8223 8224 /* 8225 * Called immediately before a task is migrated to a new CPU; task_cpu(p) and 8226 * cfs_rq_of(p) references at time of call are still valid and identify the 8227 * previous CPU. The caller guarantees p->pi_lock or task_rq(p)->lock is held. 8228 */ 8229 static void migrate_task_rq_fair(struct task_struct *p, int new_cpu) 8230 { 8231 struct sched_entity *se = &p->se; 8232 8233 if (!task_on_rq_migrating(p)) { 8234 remove_entity_load_avg(se); 8235 8236 /* 8237 * Here, the task's PELT values have been updated according to 8238 * the current rq's clock. But if that clock hasn't been 8239 * updated in a while, a substantial idle time will be missed, 8240 * leading to an inflation after wake-up on the new rq. 8241 * 8242 * Estimate the missing time from the cfs_rq last_update_time 8243 * and update sched_avg to improve the PELT continuity after 8244 * migration. 8245 */ 8246 migrate_se_pelt_lag(se); 8247 } 8248 8249 /* Tell new CPU we are migrated */ 8250 se->avg.last_update_time = 0; 8251 8252 update_scan_period(p, new_cpu); 8253 } 8254 8255 static void task_dead_fair(struct task_struct *p) 8256 { 8257 remove_entity_load_avg(&p->se); 8258 } 8259 8260 static int 8261 balance_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 8262 { 8263 if (rq->nr_running) 8264 return 1; 8265 8266 return newidle_balance(rq, rf) != 0; 8267 } 8268 #endif /* CONFIG_SMP */ 8269 8270 static void set_next_buddy(struct sched_entity *se) 8271 { 8272 for_each_sched_entity(se) { 8273 if (SCHED_WARN_ON(!se->on_rq)) 8274 return; 8275 if (se_is_idle(se)) 8276 return; 8277 cfs_rq_of(se)->next = se; 8278 } 8279 } 8280 8281 /* 8282 * Preempt the current task with a newly woken task if needed: 8283 */ 8284 static void check_preempt_wakeup_fair(struct rq *rq, struct task_struct *p, int wake_flags) 8285 { 8286 struct task_struct *curr = rq->curr; 8287 struct sched_entity *se = &curr->se, *pse = &p->se; 8288 struct cfs_rq *cfs_rq = task_cfs_rq(curr); 8289 int next_buddy_marked = 0; 8290 int cse_is_idle, pse_is_idle; 8291 8292 if (unlikely(se == pse)) 8293 return; 8294 8295 /* 8296 * This is possible from callers such as attach_tasks(), in which we 8297 * unconditionally wakeup_preempt() after an enqueue (which may have 8298 * lead to a throttle). This both saves work and prevents false 8299 * next-buddy nomination below. 8300 */ 8301 if (unlikely(throttled_hierarchy(cfs_rq_of(pse)))) 8302 return; 8303 8304 if (sched_feat(NEXT_BUDDY) && !(wake_flags & WF_FORK)) { 8305 set_next_buddy(pse); 8306 next_buddy_marked = 1; 8307 } 8308 8309 /* 8310 * We can come here with TIF_NEED_RESCHED already set from new task 8311 * wake up path. 8312 * 8313 * Note: this also catches the edge-case of curr being in a throttled 8314 * group (e.g. via set_curr_task), since update_curr() (in the 8315 * enqueue of curr) will have resulted in resched being set. This 8316 * prevents us from potentially nominating it as a false LAST_BUDDY 8317 * below. 8318 */ 8319 if (test_tsk_need_resched(curr)) 8320 return; 8321 8322 if (!sched_feat(WAKEUP_PREEMPTION)) 8323 return; 8324 8325 find_matching_se(&se, &pse); 8326 WARN_ON_ONCE(!pse); 8327 8328 cse_is_idle = se_is_idle(se); 8329 pse_is_idle = se_is_idle(pse); 8330 8331 /* 8332 * Preempt an idle entity in favor of a non-idle entity (and don't preempt 8333 * in the inverse case). 8334 */ 8335 if (cse_is_idle && !pse_is_idle) 8336 goto preempt; 8337 if (cse_is_idle != pse_is_idle) 8338 return; 8339 8340 /* 8341 * BATCH and IDLE tasks do not preempt others. 8342 */ 8343 if (unlikely(p->policy != SCHED_NORMAL)) 8344 return; 8345 8346 cfs_rq = cfs_rq_of(se); 8347 update_curr(cfs_rq); 8348 /* 8349 * XXX pick_eevdf(cfs_rq) != se ? 8350 */ 8351 if (pick_eevdf(cfs_rq) == pse) 8352 goto preempt; 8353 8354 return; 8355 8356 preempt: 8357 resched_curr(rq); 8358 } 8359 8360 #ifdef CONFIG_SMP 8361 static struct task_struct *pick_task_fair(struct rq *rq) 8362 { 8363 struct sched_entity *se; 8364 struct cfs_rq *cfs_rq; 8365 8366 again: 8367 cfs_rq = &rq->cfs; 8368 if (!cfs_rq->nr_running) 8369 return NULL; 8370 8371 do { 8372 struct sched_entity *curr = cfs_rq->curr; 8373 8374 /* When we pick for a remote RQ, we'll not have done put_prev_entity() */ 8375 if (curr) { 8376 if (curr->on_rq) 8377 update_curr(cfs_rq); 8378 else 8379 curr = NULL; 8380 8381 if (unlikely(check_cfs_rq_runtime(cfs_rq))) 8382 goto again; 8383 } 8384 8385 se = pick_next_entity(cfs_rq, curr); 8386 cfs_rq = group_cfs_rq(se); 8387 } while (cfs_rq); 8388 8389 return task_of(se); 8390 } 8391 #endif 8392 8393 struct task_struct * 8394 pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 8395 { 8396 struct cfs_rq *cfs_rq = &rq->cfs; 8397 struct sched_entity *se; 8398 struct task_struct *p; 8399 int new_tasks; 8400 8401 again: 8402 if (!sched_fair_runnable(rq)) 8403 goto idle; 8404 8405 #ifdef CONFIG_FAIR_GROUP_SCHED 8406 if (!prev || prev->sched_class != &fair_sched_class) 8407 goto simple; 8408 8409 /* 8410 * Because of the set_next_buddy() in dequeue_task_fair() it is rather 8411 * likely that a next task is from the same cgroup as the current. 8412 * 8413 * Therefore attempt to avoid putting and setting the entire cgroup 8414 * hierarchy, only change the part that actually changes. 8415 */ 8416 8417 do { 8418 struct sched_entity *curr = cfs_rq->curr; 8419 8420 /* 8421 * Since we got here without doing put_prev_entity() we also 8422 * have to consider cfs_rq->curr. If it is still a runnable 8423 * entity, update_curr() will update its vruntime, otherwise 8424 * forget we've ever seen it. 8425 */ 8426 if (curr) { 8427 if (curr->on_rq) 8428 update_curr(cfs_rq); 8429 else 8430 curr = NULL; 8431 8432 /* 8433 * This call to check_cfs_rq_runtime() will do the 8434 * throttle and dequeue its entity in the parent(s). 8435 * Therefore the nr_running test will indeed 8436 * be correct. 8437 */ 8438 if (unlikely(check_cfs_rq_runtime(cfs_rq))) { 8439 cfs_rq = &rq->cfs; 8440 8441 if (!cfs_rq->nr_running) 8442 goto idle; 8443 8444 goto simple; 8445 } 8446 } 8447 8448 se = pick_next_entity(cfs_rq, curr); 8449 cfs_rq = group_cfs_rq(se); 8450 } while (cfs_rq); 8451 8452 p = task_of(se); 8453 8454 /* 8455 * Since we haven't yet done put_prev_entity and if the selected task 8456 * is a different task than we started out with, try and touch the 8457 * least amount of cfs_rqs. 8458 */ 8459 if (prev != p) { 8460 struct sched_entity *pse = &prev->se; 8461 8462 while (!(cfs_rq = is_same_group(se, pse))) { 8463 int se_depth = se->depth; 8464 int pse_depth = pse->depth; 8465 8466 if (se_depth <= pse_depth) { 8467 put_prev_entity(cfs_rq_of(pse), pse); 8468 pse = parent_entity(pse); 8469 } 8470 if (se_depth >= pse_depth) { 8471 set_next_entity(cfs_rq_of(se), se); 8472 se = parent_entity(se); 8473 } 8474 } 8475 8476 put_prev_entity(cfs_rq, pse); 8477 set_next_entity(cfs_rq, se); 8478 } 8479 8480 goto done; 8481 simple: 8482 #endif 8483 if (prev) 8484 put_prev_task(rq, prev); 8485 8486 do { 8487 se = pick_next_entity(cfs_rq, NULL); 8488 set_next_entity(cfs_rq, se); 8489 cfs_rq = group_cfs_rq(se); 8490 } while (cfs_rq); 8491 8492 p = task_of(se); 8493 8494 done: __maybe_unused; 8495 #ifdef CONFIG_SMP 8496 /* 8497 * Move the next running task to the front of 8498 * the list, so our cfs_tasks list becomes MRU 8499 * one. 8500 */ 8501 list_move(&p->se.group_node, &rq->cfs_tasks); 8502 #endif 8503 8504 if (hrtick_enabled_fair(rq)) 8505 hrtick_start_fair(rq, p); 8506 8507 update_misfit_status(p, rq); 8508 sched_fair_update_stop_tick(rq, p); 8509 8510 return p; 8511 8512 idle: 8513 if (!rf) 8514 return NULL; 8515 8516 new_tasks = newidle_balance(rq, rf); 8517 8518 /* 8519 * Because newidle_balance() releases (and re-acquires) rq->lock, it is 8520 * possible for any higher priority task to appear. In that case we 8521 * must re-start the pick_next_entity() loop. 8522 */ 8523 if (new_tasks < 0) 8524 return RETRY_TASK; 8525 8526 if (new_tasks > 0) 8527 goto again; 8528 8529 /* 8530 * rq is about to be idle, check if we need to update the 8531 * lost_idle_time of clock_pelt 8532 */ 8533 update_idle_rq_clock_pelt(rq); 8534 8535 return NULL; 8536 } 8537 8538 static struct task_struct *__pick_next_task_fair(struct rq *rq) 8539 { 8540 return pick_next_task_fair(rq, NULL, NULL); 8541 } 8542 8543 /* 8544 * Account for a descheduled task: 8545 */ 8546 static void put_prev_task_fair(struct rq *rq, struct task_struct *prev) 8547 { 8548 struct sched_entity *se = &prev->se; 8549 struct cfs_rq *cfs_rq; 8550 8551 for_each_sched_entity(se) { 8552 cfs_rq = cfs_rq_of(se); 8553 put_prev_entity(cfs_rq, se); 8554 } 8555 } 8556 8557 /* 8558 * sched_yield() is very simple 8559 */ 8560 static void yield_task_fair(struct rq *rq) 8561 { 8562 struct task_struct *curr = rq->curr; 8563 struct cfs_rq *cfs_rq = task_cfs_rq(curr); 8564 struct sched_entity *se = &curr->se; 8565 8566 /* 8567 * Are we the only task in the tree? 8568 */ 8569 if (unlikely(rq->nr_running == 1)) 8570 return; 8571 8572 clear_buddies(cfs_rq, se); 8573 8574 update_rq_clock(rq); 8575 /* 8576 * Update run-time statistics of the 'current'. 8577 */ 8578 update_curr(cfs_rq); 8579 /* 8580 * Tell update_rq_clock() that we've just updated, 8581 * so we don't do microscopic update in schedule() 8582 * and double the fastpath cost. 8583 */ 8584 rq_clock_skip_update(rq); 8585 8586 se->deadline += calc_delta_fair(se->slice, se); 8587 } 8588 8589 static bool yield_to_task_fair(struct rq *rq, struct task_struct *p) 8590 { 8591 struct sched_entity *se = &p->se; 8592 8593 /* throttled hierarchies are not runnable */ 8594 if (!se->on_rq || throttled_hierarchy(cfs_rq_of(se))) 8595 return false; 8596 8597 /* Tell the scheduler that we'd really like pse to run next. */ 8598 set_next_buddy(se); 8599 8600 yield_task_fair(rq); 8601 8602 return true; 8603 } 8604 8605 #ifdef CONFIG_SMP 8606 /************************************************** 8607 * Fair scheduling class load-balancing methods. 8608 * 8609 * BASICS 8610 * 8611 * The purpose of load-balancing is to achieve the same basic fairness the 8612 * per-CPU scheduler provides, namely provide a proportional amount of compute 8613 * time to each task. This is expressed in the following equation: 8614 * 8615 * W_i,n/P_i == W_j,n/P_j for all i,j (1) 8616 * 8617 * Where W_i,n is the n-th weight average for CPU i. The instantaneous weight 8618 * W_i,0 is defined as: 8619 * 8620 * W_i,0 = \Sum_j w_i,j (2) 8621 * 8622 * Where w_i,j is the weight of the j-th runnable task on CPU i. This weight 8623 * is derived from the nice value as per sched_prio_to_weight[]. 8624 * 8625 * The weight average is an exponential decay average of the instantaneous 8626 * weight: 8627 * 8628 * W'_i,n = (2^n - 1) / 2^n * W_i,n + 1 / 2^n * W_i,0 (3) 8629 * 8630 * C_i is the compute capacity of CPU i, typically it is the 8631 * fraction of 'recent' time available for SCHED_OTHER task execution. But it 8632 * can also include other factors [XXX]. 8633 * 8634 * To achieve this balance we define a measure of imbalance which follows 8635 * directly from (1): 8636 * 8637 * imb_i,j = max{ avg(W/C), W_i/C_i } - min{ avg(W/C), W_j/C_j } (4) 8638 * 8639 * We them move tasks around to minimize the imbalance. In the continuous 8640 * function space it is obvious this converges, in the discrete case we get 8641 * a few fun cases generally called infeasible weight scenarios. 8642 * 8643 * [XXX expand on: 8644 * - infeasible weights; 8645 * - local vs global optima in the discrete case. ] 8646 * 8647 * 8648 * SCHED DOMAINS 8649 * 8650 * In order to solve the imbalance equation (4), and avoid the obvious O(n^2) 8651 * for all i,j solution, we create a tree of CPUs that follows the hardware 8652 * topology where each level pairs two lower groups (or better). This results 8653 * in O(log n) layers. Furthermore we reduce the number of CPUs going up the 8654 * tree to only the first of the previous level and we decrease the frequency 8655 * of load-balance at each level inv. proportional to the number of CPUs in 8656 * the groups. 8657 * 8658 * This yields: 8659 * 8660 * log_2 n 1 n 8661 * \Sum { --- * --- * 2^i } = O(n) (5) 8662 * i = 0 2^i 2^i 8663 * `- size of each group 8664 * | | `- number of CPUs doing load-balance 8665 * | `- freq 8666 * `- sum over all levels 8667 * 8668 * Coupled with a limit on how many tasks we can migrate every balance pass, 8669 * this makes (5) the runtime complexity of the balancer. 8670 * 8671 * An important property here is that each CPU is still (indirectly) connected 8672 * to every other CPU in at most O(log n) steps: 8673 * 8674 * The adjacency matrix of the resulting graph is given by: 8675 * 8676 * log_2 n 8677 * A_i,j = \Union (i % 2^k == 0) && i / 2^(k+1) == j / 2^(k+1) (6) 8678 * k = 0 8679 * 8680 * And you'll find that: 8681 * 8682 * A^(log_2 n)_i,j != 0 for all i,j (7) 8683 * 8684 * Showing there's indeed a path between every CPU in at most O(log n) steps. 8685 * The task movement gives a factor of O(m), giving a convergence complexity 8686 * of: 8687 * 8688 * O(nm log n), n := nr_cpus, m := nr_tasks (8) 8689 * 8690 * 8691 * WORK CONSERVING 8692 * 8693 * In order to avoid CPUs going idle while there's still work to do, new idle 8694 * balancing is more aggressive and has the newly idle CPU iterate up the domain 8695 * tree itself instead of relying on other CPUs to bring it work. 8696 * 8697 * This adds some complexity to both (5) and (8) but it reduces the total idle 8698 * time. 8699 * 8700 * [XXX more?] 8701 * 8702 * 8703 * CGROUPS 8704 * 8705 * Cgroups make a horror show out of (2), instead of a simple sum we get: 8706 * 8707 * s_k,i 8708 * W_i,0 = \Sum_j \Prod_k w_k * ----- (9) 8709 * S_k 8710 * 8711 * Where 8712 * 8713 * s_k,i = \Sum_j w_i,j,k and S_k = \Sum_i s_k,i (10) 8714 * 8715 * w_i,j,k is the weight of the j-th runnable task in the k-th cgroup on CPU i. 8716 * 8717 * The big problem is S_k, its a global sum needed to compute a local (W_i) 8718 * property. 8719 * 8720 * [XXX write more on how we solve this.. _after_ merging pjt's patches that 8721 * rewrite all of this once again.] 8722 */ 8723 8724 static unsigned long __read_mostly max_load_balance_interval = HZ/10; 8725 8726 enum fbq_type { regular, remote, all }; 8727 8728 /* 8729 * 'group_type' describes the group of CPUs at the moment of load balancing. 8730 * 8731 * The enum is ordered by pulling priority, with the group with lowest priority 8732 * first so the group_type can simply be compared when selecting the busiest 8733 * group. See update_sd_pick_busiest(). 8734 */ 8735 enum group_type { 8736 /* The group has spare capacity that can be used to run more tasks. */ 8737 group_has_spare = 0, 8738 /* 8739 * The group is fully used and the tasks don't compete for more CPU 8740 * cycles. Nevertheless, some tasks might wait before running. 8741 */ 8742 group_fully_busy, 8743 /* 8744 * One task doesn't fit with CPU's capacity and must be migrated to a 8745 * more powerful CPU. 8746 */ 8747 group_misfit_task, 8748 /* 8749 * Balance SMT group that's fully busy. Can benefit from migration 8750 * a task on SMT with busy sibling to another CPU on idle core. 8751 */ 8752 group_smt_balance, 8753 /* 8754 * SD_ASYM_PACKING only: One local CPU with higher capacity is available, 8755 * and the task should be migrated to it instead of running on the 8756 * current CPU. 8757 */ 8758 group_asym_packing, 8759 /* 8760 * The tasks' affinity constraints previously prevented the scheduler 8761 * from balancing the load across the system. 8762 */ 8763 group_imbalanced, 8764 /* 8765 * The CPU is overloaded and can't provide expected CPU cycles to all 8766 * tasks. 8767 */ 8768 group_overloaded 8769 }; 8770 8771 enum migration_type { 8772 migrate_load = 0, 8773 migrate_util, 8774 migrate_task, 8775 migrate_misfit 8776 }; 8777 8778 #define LBF_ALL_PINNED 0x01 8779 #define LBF_NEED_BREAK 0x02 8780 #define LBF_DST_PINNED 0x04 8781 #define LBF_SOME_PINNED 0x08 8782 #define LBF_ACTIVE_LB 0x10 8783 8784 struct lb_env { 8785 struct sched_domain *sd; 8786 8787 struct rq *src_rq; 8788 int src_cpu; 8789 8790 int dst_cpu; 8791 struct rq *dst_rq; 8792 8793 struct cpumask *dst_grpmask; 8794 int new_dst_cpu; 8795 enum cpu_idle_type idle; 8796 long imbalance; 8797 /* The set of CPUs under consideration for load-balancing */ 8798 struct cpumask *cpus; 8799 8800 unsigned int flags; 8801 8802 unsigned int loop; 8803 unsigned int loop_break; 8804 unsigned int loop_max; 8805 8806 enum fbq_type fbq_type; 8807 enum migration_type migration_type; 8808 struct list_head tasks; 8809 }; 8810 8811 /* 8812 * Is this task likely cache-hot: 8813 */ 8814 static int task_hot(struct task_struct *p, struct lb_env *env) 8815 { 8816 s64 delta; 8817 8818 lockdep_assert_rq_held(env->src_rq); 8819 8820 if (p->sched_class != &fair_sched_class) 8821 return 0; 8822 8823 if (unlikely(task_has_idle_policy(p))) 8824 return 0; 8825 8826 /* SMT siblings share cache */ 8827 if (env->sd->flags & SD_SHARE_CPUCAPACITY) 8828 return 0; 8829 8830 /* 8831 * Buddy candidates are cache hot: 8832 */ 8833 if (sched_feat(CACHE_HOT_BUDDY) && env->dst_rq->nr_running && 8834 (&p->se == cfs_rq_of(&p->se)->next)) 8835 return 1; 8836 8837 if (sysctl_sched_migration_cost == -1) 8838 return 1; 8839 8840 /* 8841 * Don't migrate task if the task's cookie does not match 8842 * with the destination CPU's core cookie. 8843 */ 8844 if (!sched_core_cookie_match(cpu_rq(env->dst_cpu), p)) 8845 return 1; 8846 8847 if (sysctl_sched_migration_cost == 0) 8848 return 0; 8849 8850 delta = rq_clock_task(env->src_rq) - p->se.exec_start; 8851 8852 return delta < (s64)sysctl_sched_migration_cost; 8853 } 8854 8855 #ifdef CONFIG_NUMA_BALANCING 8856 /* 8857 * Returns 1, if task migration degrades locality 8858 * Returns 0, if task migration improves locality i.e migration preferred. 8859 * Returns -1, if task migration is not affected by locality. 8860 */ 8861 static int migrate_degrades_locality(struct task_struct *p, struct lb_env *env) 8862 { 8863 struct numa_group *numa_group = rcu_dereference(p->numa_group); 8864 unsigned long src_weight, dst_weight; 8865 int src_nid, dst_nid, dist; 8866 8867 if (!static_branch_likely(&sched_numa_balancing)) 8868 return -1; 8869 8870 if (!p->numa_faults || !(env->sd->flags & SD_NUMA)) 8871 return -1; 8872 8873 src_nid = cpu_to_node(env->src_cpu); 8874 dst_nid = cpu_to_node(env->dst_cpu); 8875 8876 if (src_nid == dst_nid) 8877 return -1; 8878 8879 /* Migrating away from the preferred node is always bad. */ 8880 if (src_nid == p->numa_preferred_nid) { 8881 if (env->src_rq->nr_running > env->src_rq->nr_preferred_running) 8882 return 1; 8883 else 8884 return -1; 8885 } 8886 8887 /* Encourage migration to the preferred node. */ 8888 if (dst_nid == p->numa_preferred_nid) 8889 return 0; 8890 8891 /* Leaving a core idle is often worse than degrading locality. */ 8892 if (env->idle == CPU_IDLE) 8893 return -1; 8894 8895 dist = node_distance(src_nid, dst_nid); 8896 if (numa_group) { 8897 src_weight = group_weight(p, src_nid, dist); 8898 dst_weight = group_weight(p, dst_nid, dist); 8899 } else { 8900 src_weight = task_weight(p, src_nid, dist); 8901 dst_weight = task_weight(p, dst_nid, dist); 8902 } 8903 8904 return dst_weight < src_weight; 8905 } 8906 8907 #else 8908 static inline int migrate_degrades_locality(struct task_struct *p, 8909 struct lb_env *env) 8910 { 8911 return -1; 8912 } 8913 #endif 8914 8915 /* 8916 * can_migrate_task - may task p from runqueue rq be migrated to this_cpu? 8917 */ 8918 static 8919 int can_migrate_task(struct task_struct *p, struct lb_env *env) 8920 { 8921 int tsk_cache_hot; 8922 8923 lockdep_assert_rq_held(env->src_rq); 8924 8925 /* 8926 * We do not migrate tasks that are: 8927 * 1) throttled_lb_pair, or 8928 * 2) cannot be migrated to this CPU due to cpus_ptr, or 8929 * 3) running (obviously), or 8930 * 4) are cache-hot on their current CPU. 8931 */ 8932 if (throttled_lb_pair(task_group(p), env->src_cpu, env->dst_cpu)) 8933 return 0; 8934 8935 /* Disregard pcpu kthreads; they are where they need to be. */ 8936 if (kthread_is_per_cpu(p)) 8937 return 0; 8938 8939 if (!cpumask_test_cpu(env->dst_cpu, p->cpus_ptr)) { 8940 int cpu; 8941 8942 schedstat_inc(p->stats.nr_failed_migrations_affine); 8943 8944 env->flags |= LBF_SOME_PINNED; 8945 8946 /* 8947 * Remember if this task can be migrated to any other CPU in 8948 * our sched_group. We may want to revisit it if we couldn't 8949 * meet load balance goals by pulling other tasks on src_cpu. 8950 * 8951 * Avoid computing new_dst_cpu 8952 * - for NEWLY_IDLE 8953 * - if we have already computed one in current iteration 8954 * - if it's an active balance 8955 */ 8956 if (env->idle == CPU_NEWLY_IDLE || 8957 env->flags & (LBF_DST_PINNED | LBF_ACTIVE_LB)) 8958 return 0; 8959 8960 /* Prevent to re-select dst_cpu via env's CPUs: */ 8961 for_each_cpu_and(cpu, env->dst_grpmask, env->cpus) { 8962 if (cpumask_test_cpu(cpu, p->cpus_ptr)) { 8963 env->flags |= LBF_DST_PINNED; 8964 env->new_dst_cpu = cpu; 8965 break; 8966 } 8967 } 8968 8969 return 0; 8970 } 8971 8972 /* Record that we found at least one task that could run on dst_cpu */ 8973 env->flags &= ~LBF_ALL_PINNED; 8974 8975 if (task_on_cpu(env->src_rq, p)) { 8976 schedstat_inc(p->stats.nr_failed_migrations_running); 8977 return 0; 8978 } 8979 8980 /* 8981 * Aggressive migration if: 8982 * 1) active balance 8983 * 2) destination numa is preferred 8984 * 3) task is cache cold, or 8985 * 4) too many balance attempts have failed. 8986 */ 8987 if (env->flags & LBF_ACTIVE_LB) 8988 return 1; 8989 8990 tsk_cache_hot = migrate_degrades_locality(p, env); 8991 if (tsk_cache_hot == -1) 8992 tsk_cache_hot = task_hot(p, env); 8993 8994 if (tsk_cache_hot <= 0 || 8995 env->sd->nr_balance_failed > env->sd->cache_nice_tries) { 8996 if (tsk_cache_hot == 1) { 8997 schedstat_inc(env->sd->lb_hot_gained[env->idle]); 8998 schedstat_inc(p->stats.nr_forced_migrations); 8999 } 9000 return 1; 9001 } 9002 9003 schedstat_inc(p->stats.nr_failed_migrations_hot); 9004 return 0; 9005 } 9006 9007 /* 9008 * detach_task() -- detach the task for the migration specified in env 9009 */ 9010 static void detach_task(struct task_struct *p, struct lb_env *env) 9011 { 9012 lockdep_assert_rq_held(env->src_rq); 9013 9014 deactivate_task(env->src_rq, p, DEQUEUE_NOCLOCK); 9015 set_task_cpu(p, env->dst_cpu); 9016 } 9017 9018 /* 9019 * detach_one_task() -- tries to dequeue exactly one task from env->src_rq, as 9020 * part of active balancing operations within "domain". 9021 * 9022 * Returns a task if successful and NULL otherwise. 9023 */ 9024 static struct task_struct *detach_one_task(struct lb_env *env) 9025 { 9026 struct task_struct *p; 9027 9028 lockdep_assert_rq_held(env->src_rq); 9029 9030 list_for_each_entry_reverse(p, 9031 &env->src_rq->cfs_tasks, se.group_node) { 9032 if (!can_migrate_task(p, env)) 9033 continue; 9034 9035 detach_task(p, env); 9036 9037 /* 9038 * Right now, this is only the second place where 9039 * lb_gained[env->idle] is updated (other is detach_tasks) 9040 * so we can safely collect stats here rather than 9041 * inside detach_tasks(). 9042 */ 9043 schedstat_inc(env->sd->lb_gained[env->idle]); 9044 return p; 9045 } 9046 return NULL; 9047 } 9048 9049 /* 9050 * detach_tasks() -- tries to detach up to imbalance load/util/tasks from 9051 * busiest_rq, as part of a balancing operation within domain "sd". 9052 * 9053 * Returns number of detached tasks if successful and 0 otherwise. 9054 */ 9055 static int detach_tasks(struct lb_env *env) 9056 { 9057 struct list_head *tasks = &env->src_rq->cfs_tasks; 9058 unsigned long util, load; 9059 struct task_struct *p; 9060 int detached = 0; 9061 9062 lockdep_assert_rq_held(env->src_rq); 9063 9064 /* 9065 * Source run queue has been emptied by another CPU, clear 9066 * LBF_ALL_PINNED flag as we will not test any task. 9067 */ 9068 if (env->src_rq->nr_running <= 1) { 9069 env->flags &= ~LBF_ALL_PINNED; 9070 return 0; 9071 } 9072 9073 if (env->imbalance <= 0) 9074 return 0; 9075 9076 while (!list_empty(tasks)) { 9077 /* 9078 * We don't want to steal all, otherwise we may be treated likewise, 9079 * which could at worst lead to a livelock crash. 9080 */ 9081 if (env->idle != CPU_NOT_IDLE && env->src_rq->nr_running <= 1) 9082 break; 9083 9084 env->loop++; 9085 /* We've more or less seen every task there is, call it quits */ 9086 if (env->loop > env->loop_max) 9087 break; 9088 9089 /* take a breather every nr_migrate tasks */ 9090 if (env->loop > env->loop_break) { 9091 env->loop_break += SCHED_NR_MIGRATE_BREAK; 9092 env->flags |= LBF_NEED_BREAK; 9093 break; 9094 } 9095 9096 p = list_last_entry(tasks, struct task_struct, se.group_node); 9097 9098 if (!can_migrate_task(p, env)) 9099 goto next; 9100 9101 switch (env->migration_type) { 9102 case migrate_load: 9103 /* 9104 * Depending of the number of CPUs and tasks and the 9105 * cgroup hierarchy, task_h_load() can return a null 9106 * value. Make sure that env->imbalance decreases 9107 * otherwise detach_tasks() will stop only after 9108 * detaching up to loop_max tasks. 9109 */ 9110 load = max_t(unsigned long, task_h_load(p), 1); 9111 9112 if (sched_feat(LB_MIN) && 9113 load < 16 && !env->sd->nr_balance_failed) 9114 goto next; 9115 9116 /* 9117 * Make sure that we don't migrate too much load. 9118 * Nevertheless, let relax the constraint if 9119 * scheduler fails to find a good waiting task to 9120 * migrate. 9121 */ 9122 if (shr_bound(load, env->sd->nr_balance_failed) > env->imbalance) 9123 goto next; 9124 9125 env->imbalance -= load; 9126 break; 9127 9128 case migrate_util: 9129 util = task_util_est(p); 9130 9131 if (util > env->imbalance) 9132 goto next; 9133 9134 env->imbalance -= util; 9135 break; 9136 9137 case migrate_task: 9138 env->imbalance--; 9139 break; 9140 9141 case migrate_misfit: 9142 /* This is not a misfit task */ 9143 if (task_fits_cpu(p, env->src_cpu)) 9144 goto next; 9145 9146 env->imbalance = 0; 9147 break; 9148 } 9149 9150 detach_task(p, env); 9151 list_add(&p->se.group_node, &env->tasks); 9152 9153 detached++; 9154 9155 #ifdef CONFIG_PREEMPTION 9156 /* 9157 * NEWIDLE balancing is a source of latency, so preemptible 9158 * kernels will stop after the first task is detached to minimize 9159 * the critical section. 9160 */ 9161 if (env->idle == CPU_NEWLY_IDLE) 9162 break; 9163 #endif 9164 9165 /* 9166 * We only want to steal up to the prescribed amount of 9167 * load/util/tasks. 9168 */ 9169 if (env->imbalance <= 0) 9170 break; 9171 9172 continue; 9173 next: 9174 list_move(&p->se.group_node, tasks); 9175 } 9176 9177 /* 9178 * Right now, this is one of only two places we collect this stat 9179 * so we can safely collect detach_one_task() stats here rather 9180 * than inside detach_one_task(). 9181 */ 9182 schedstat_add(env->sd->lb_gained[env->idle], detached); 9183 9184 return detached; 9185 } 9186 9187 /* 9188 * attach_task() -- attach the task detached by detach_task() to its new rq. 9189 */ 9190 static void attach_task(struct rq *rq, struct task_struct *p) 9191 { 9192 lockdep_assert_rq_held(rq); 9193 9194 WARN_ON_ONCE(task_rq(p) != rq); 9195 activate_task(rq, p, ENQUEUE_NOCLOCK); 9196 wakeup_preempt(rq, p, 0); 9197 } 9198 9199 /* 9200 * attach_one_task() -- attaches the task returned from detach_one_task() to 9201 * its new rq. 9202 */ 9203 static void attach_one_task(struct rq *rq, struct task_struct *p) 9204 { 9205 struct rq_flags rf; 9206 9207 rq_lock(rq, &rf); 9208 update_rq_clock(rq); 9209 attach_task(rq, p); 9210 rq_unlock(rq, &rf); 9211 } 9212 9213 /* 9214 * attach_tasks() -- attaches all tasks detached by detach_tasks() to their 9215 * new rq. 9216 */ 9217 static void attach_tasks(struct lb_env *env) 9218 { 9219 struct list_head *tasks = &env->tasks; 9220 struct task_struct *p; 9221 struct rq_flags rf; 9222 9223 rq_lock(env->dst_rq, &rf); 9224 update_rq_clock(env->dst_rq); 9225 9226 while (!list_empty(tasks)) { 9227 p = list_first_entry(tasks, struct task_struct, se.group_node); 9228 list_del_init(&p->se.group_node); 9229 9230 attach_task(env->dst_rq, p); 9231 } 9232 9233 rq_unlock(env->dst_rq, &rf); 9234 } 9235 9236 #ifdef CONFIG_NO_HZ_COMMON 9237 static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq) 9238 { 9239 if (cfs_rq->avg.load_avg) 9240 return true; 9241 9242 if (cfs_rq->avg.util_avg) 9243 return true; 9244 9245 return false; 9246 } 9247 9248 static inline bool others_have_blocked(struct rq *rq) 9249 { 9250 if (READ_ONCE(rq->avg_rt.util_avg)) 9251 return true; 9252 9253 if (READ_ONCE(rq->avg_dl.util_avg)) 9254 return true; 9255 9256 if (thermal_load_avg(rq)) 9257 return true; 9258 9259 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ 9260 if (READ_ONCE(rq->avg_irq.util_avg)) 9261 return true; 9262 #endif 9263 9264 return false; 9265 } 9266 9267 static inline void update_blocked_load_tick(struct rq *rq) 9268 { 9269 WRITE_ONCE(rq->last_blocked_load_update_tick, jiffies); 9270 } 9271 9272 static inline void update_blocked_load_status(struct rq *rq, bool has_blocked) 9273 { 9274 if (!has_blocked) 9275 rq->has_blocked_load = 0; 9276 } 9277 #else 9278 static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq) { return false; } 9279 static inline bool others_have_blocked(struct rq *rq) { return false; } 9280 static inline void update_blocked_load_tick(struct rq *rq) {} 9281 static inline void update_blocked_load_status(struct rq *rq, bool has_blocked) {} 9282 #endif 9283 9284 static bool __update_blocked_others(struct rq *rq, bool *done) 9285 { 9286 const struct sched_class *curr_class; 9287 u64 now = rq_clock_pelt(rq); 9288 unsigned long thermal_pressure; 9289 bool decayed; 9290 9291 /* 9292 * update_load_avg() can call cpufreq_update_util(). Make sure that RT, 9293 * DL and IRQ signals have been updated before updating CFS. 9294 */ 9295 curr_class = rq->curr->sched_class; 9296 9297 thermal_pressure = arch_scale_thermal_pressure(cpu_of(rq)); 9298 9299 decayed = update_rt_rq_load_avg(now, rq, curr_class == &rt_sched_class) | 9300 update_dl_rq_load_avg(now, rq, curr_class == &dl_sched_class) | 9301 update_thermal_load_avg(rq_clock_thermal(rq), rq, thermal_pressure) | 9302 update_irq_load_avg(rq, 0); 9303 9304 if (others_have_blocked(rq)) 9305 *done = false; 9306 9307 return decayed; 9308 } 9309 9310 #ifdef CONFIG_FAIR_GROUP_SCHED 9311 9312 static bool __update_blocked_fair(struct rq *rq, bool *done) 9313 { 9314 struct cfs_rq *cfs_rq, *pos; 9315 bool decayed = false; 9316 int cpu = cpu_of(rq); 9317 9318 /* 9319 * Iterates the task_group tree in a bottom up fashion, see 9320 * list_add_leaf_cfs_rq() for details. 9321 */ 9322 for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) { 9323 struct sched_entity *se; 9324 9325 if (update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq)) { 9326 update_tg_load_avg(cfs_rq); 9327 9328 if (cfs_rq->nr_running == 0) 9329 update_idle_cfs_rq_clock_pelt(cfs_rq); 9330 9331 if (cfs_rq == &rq->cfs) 9332 decayed = true; 9333 } 9334 9335 /* Propagate pending load changes to the parent, if any: */ 9336 se = cfs_rq->tg->se[cpu]; 9337 if (se && !skip_blocked_update(se)) 9338 update_load_avg(cfs_rq_of(se), se, UPDATE_TG); 9339 9340 /* 9341 * There can be a lot of idle CPU cgroups. Don't let fully 9342 * decayed cfs_rqs linger on the list. 9343 */ 9344 if (cfs_rq_is_decayed(cfs_rq)) 9345 list_del_leaf_cfs_rq(cfs_rq); 9346 9347 /* Don't need periodic decay once load/util_avg are null */ 9348 if (cfs_rq_has_blocked(cfs_rq)) 9349 *done = false; 9350 } 9351 9352 return decayed; 9353 } 9354 9355 /* 9356 * Compute the hierarchical load factor for cfs_rq and all its ascendants. 9357 * This needs to be done in a top-down fashion because the load of a child 9358 * group is a fraction of its parents load. 9359 */ 9360 static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq) 9361 { 9362 struct rq *rq = rq_of(cfs_rq); 9363 struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)]; 9364 unsigned long now = jiffies; 9365 unsigned long load; 9366 9367 if (cfs_rq->last_h_load_update == now) 9368 return; 9369 9370 WRITE_ONCE(cfs_rq->h_load_next, NULL); 9371 for_each_sched_entity(se) { 9372 cfs_rq = cfs_rq_of(se); 9373 WRITE_ONCE(cfs_rq->h_load_next, se); 9374 if (cfs_rq->last_h_load_update == now) 9375 break; 9376 } 9377 9378 if (!se) { 9379 cfs_rq->h_load = cfs_rq_load_avg(cfs_rq); 9380 cfs_rq->last_h_load_update = now; 9381 } 9382 9383 while ((se = READ_ONCE(cfs_rq->h_load_next)) != NULL) { 9384 load = cfs_rq->h_load; 9385 load = div64_ul(load * se->avg.load_avg, 9386 cfs_rq_load_avg(cfs_rq) + 1); 9387 cfs_rq = group_cfs_rq(se); 9388 cfs_rq->h_load = load; 9389 cfs_rq->last_h_load_update = now; 9390 } 9391 } 9392 9393 static unsigned long task_h_load(struct task_struct *p) 9394 { 9395 struct cfs_rq *cfs_rq = task_cfs_rq(p); 9396 9397 update_cfs_rq_h_load(cfs_rq); 9398 return div64_ul(p->se.avg.load_avg * cfs_rq->h_load, 9399 cfs_rq_load_avg(cfs_rq) + 1); 9400 } 9401 #else 9402 static bool __update_blocked_fair(struct rq *rq, bool *done) 9403 { 9404 struct cfs_rq *cfs_rq = &rq->cfs; 9405 bool decayed; 9406 9407 decayed = update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq); 9408 if (cfs_rq_has_blocked(cfs_rq)) 9409 *done = false; 9410 9411 return decayed; 9412 } 9413 9414 static unsigned long task_h_load(struct task_struct *p) 9415 { 9416 return p->se.avg.load_avg; 9417 } 9418 #endif 9419 9420 static void update_blocked_averages(int cpu) 9421 { 9422 bool decayed = false, done = true; 9423 struct rq *rq = cpu_rq(cpu); 9424 struct rq_flags rf; 9425 9426 rq_lock_irqsave(rq, &rf); 9427 update_blocked_load_tick(rq); 9428 update_rq_clock(rq); 9429 9430 decayed |= __update_blocked_others(rq, &done); 9431 decayed |= __update_blocked_fair(rq, &done); 9432 9433 update_blocked_load_status(rq, !done); 9434 if (decayed) 9435 cpufreq_update_util(rq, 0); 9436 rq_unlock_irqrestore(rq, &rf); 9437 } 9438 9439 /********** Helpers for find_busiest_group ************************/ 9440 9441 /* 9442 * sg_lb_stats - stats of a sched_group required for load_balancing 9443 */ 9444 struct sg_lb_stats { 9445 unsigned long avg_load; /*Avg load across the CPUs of the group */ 9446 unsigned long group_load; /* Total load over the CPUs of the group */ 9447 unsigned long group_capacity; 9448 unsigned long group_util; /* Total utilization over the CPUs of the group */ 9449 unsigned long group_runnable; /* Total runnable time over the CPUs of the group */ 9450 unsigned int sum_nr_running; /* Nr of tasks running in the group */ 9451 unsigned int sum_h_nr_running; /* Nr of CFS tasks running in the group */ 9452 unsigned int idle_cpus; 9453 unsigned int group_weight; 9454 enum group_type group_type; 9455 unsigned int group_asym_packing; /* Tasks should be moved to preferred CPU */ 9456 unsigned int group_smt_balance; /* Task on busy SMT be moved */ 9457 unsigned long group_misfit_task_load; /* A CPU has a task too big for its capacity */ 9458 #ifdef CONFIG_NUMA_BALANCING 9459 unsigned int nr_numa_running; 9460 unsigned int nr_preferred_running; 9461 #endif 9462 }; 9463 9464 /* 9465 * sd_lb_stats - Structure to store the statistics of a sched_domain 9466 * during load balancing. 9467 */ 9468 struct sd_lb_stats { 9469 struct sched_group *busiest; /* Busiest group in this sd */ 9470 struct sched_group *local; /* Local group in this sd */ 9471 unsigned long total_load; /* Total load of all groups in sd */ 9472 unsigned long total_capacity; /* Total capacity of all groups in sd */ 9473 unsigned long avg_load; /* Average load across all groups in sd */ 9474 unsigned int prefer_sibling; /* tasks should go to sibling first */ 9475 9476 struct sg_lb_stats busiest_stat;/* Statistics of the busiest group */ 9477 struct sg_lb_stats local_stat; /* Statistics of the local group */ 9478 }; 9479 9480 static inline void init_sd_lb_stats(struct sd_lb_stats *sds) 9481 { 9482 /* 9483 * Skimp on the clearing to avoid duplicate work. We can avoid clearing 9484 * local_stat because update_sg_lb_stats() does a full clear/assignment. 9485 * We must however set busiest_stat::group_type and 9486 * busiest_stat::idle_cpus to the worst busiest group because 9487 * update_sd_pick_busiest() reads these before assignment. 9488 */ 9489 *sds = (struct sd_lb_stats){ 9490 .busiest = NULL, 9491 .local = NULL, 9492 .total_load = 0UL, 9493 .total_capacity = 0UL, 9494 .busiest_stat = { 9495 .idle_cpus = UINT_MAX, 9496 .group_type = group_has_spare, 9497 }, 9498 }; 9499 } 9500 9501 static unsigned long scale_rt_capacity(int cpu) 9502 { 9503 struct rq *rq = cpu_rq(cpu); 9504 unsigned long max = arch_scale_cpu_capacity(cpu); 9505 unsigned long used, free; 9506 unsigned long irq; 9507 9508 irq = cpu_util_irq(rq); 9509 9510 if (unlikely(irq >= max)) 9511 return 1; 9512 9513 /* 9514 * avg_rt.util_avg and avg_dl.util_avg track binary signals 9515 * (running and not running) with weights 0 and 1024 respectively. 9516 * avg_thermal.load_avg tracks thermal pressure and the weighted 9517 * average uses the actual delta max capacity(load). 9518 */ 9519 used = READ_ONCE(rq->avg_rt.util_avg); 9520 used += READ_ONCE(rq->avg_dl.util_avg); 9521 used += thermal_load_avg(rq); 9522 9523 if (unlikely(used >= max)) 9524 return 1; 9525 9526 free = max - used; 9527 9528 return scale_irq_capacity(free, irq, max); 9529 } 9530 9531 static void update_cpu_capacity(struct sched_domain *sd, int cpu) 9532 { 9533 unsigned long capacity = scale_rt_capacity(cpu); 9534 struct sched_group *sdg = sd->groups; 9535 9536 cpu_rq(cpu)->cpu_capacity_orig = arch_scale_cpu_capacity(cpu); 9537 9538 if (!capacity) 9539 capacity = 1; 9540 9541 cpu_rq(cpu)->cpu_capacity = capacity; 9542 trace_sched_cpu_capacity_tp(cpu_rq(cpu)); 9543 9544 sdg->sgc->capacity = capacity; 9545 sdg->sgc->min_capacity = capacity; 9546 sdg->sgc->max_capacity = capacity; 9547 } 9548 9549 void update_group_capacity(struct sched_domain *sd, int cpu) 9550 { 9551 struct sched_domain *child = sd->child; 9552 struct sched_group *group, *sdg = sd->groups; 9553 unsigned long capacity, min_capacity, max_capacity; 9554 unsigned long interval; 9555 9556 interval = msecs_to_jiffies(sd->balance_interval); 9557 interval = clamp(interval, 1UL, max_load_balance_interval); 9558 sdg->sgc->next_update = jiffies + interval; 9559 9560 if (!child) { 9561 update_cpu_capacity(sd, cpu); 9562 return; 9563 } 9564 9565 capacity = 0; 9566 min_capacity = ULONG_MAX; 9567 max_capacity = 0; 9568 9569 if (child->flags & SD_OVERLAP) { 9570 /* 9571 * SD_OVERLAP domains cannot assume that child groups 9572 * span the current group. 9573 */ 9574 9575 for_each_cpu(cpu, sched_group_span(sdg)) { 9576 unsigned long cpu_cap = capacity_of(cpu); 9577 9578 capacity += cpu_cap; 9579 min_capacity = min(cpu_cap, min_capacity); 9580 max_capacity = max(cpu_cap, max_capacity); 9581 } 9582 } else { 9583 /* 9584 * !SD_OVERLAP domains can assume that child groups 9585 * span the current group. 9586 */ 9587 9588 group = child->groups; 9589 do { 9590 struct sched_group_capacity *sgc = group->sgc; 9591 9592 capacity += sgc->capacity; 9593 min_capacity = min(sgc->min_capacity, min_capacity); 9594 max_capacity = max(sgc->max_capacity, max_capacity); 9595 group = group->next; 9596 } while (group != child->groups); 9597 } 9598 9599 sdg->sgc->capacity = capacity; 9600 sdg->sgc->min_capacity = min_capacity; 9601 sdg->sgc->max_capacity = max_capacity; 9602 } 9603 9604 /* 9605 * Check whether the capacity of the rq has been noticeably reduced by side 9606 * activity. The imbalance_pct is used for the threshold. 9607 * Return true is the capacity is reduced 9608 */ 9609 static inline int 9610 check_cpu_capacity(struct rq *rq, struct sched_domain *sd) 9611 { 9612 return ((rq->cpu_capacity * sd->imbalance_pct) < 9613 (rq->cpu_capacity_orig * 100)); 9614 } 9615 9616 /* 9617 * Check whether a rq has a misfit task and if it looks like we can actually 9618 * help that task: we can migrate the task to a CPU of higher capacity, or 9619 * the task's current CPU is heavily pressured. 9620 */ 9621 static inline int check_misfit_status(struct rq *rq, struct sched_domain *sd) 9622 { 9623 return rq->misfit_task_load && 9624 (rq->cpu_capacity_orig < rq->rd->max_cpu_capacity || 9625 check_cpu_capacity(rq, sd)); 9626 } 9627 9628 /* 9629 * Group imbalance indicates (and tries to solve) the problem where balancing 9630 * groups is inadequate due to ->cpus_ptr constraints. 9631 * 9632 * Imagine a situation of two groups of 4 CPUs each and 4 tasks each with a 9633 * cpumask covering 1 CPU of the first group and 3 CPUs of the second group. 9634 * Something like: 9635 * 9636 * { 0 1 2 3 } { 4 5 6 7 } 9637 * * * * * 9638 * 9639 * If we were to balance group-wise we'd place two tasks in the first group and 9640 * two tasks in the second group. Clearly this is undesired as it will overload 9641 * cpu 3 and leave one of the CPUs in the second group unused. 9642 * 9643 * The current solution to this issue is detecting the skew in the first group 9644 * by noticing the lower domain failed to reach balance and had difficulty 9645 * moving tasks due to affinity constraints. 9646 * 9647 * When this is so detected; this group becomes a candidate for busiest; see 9648 * update_sd_pick_busiest(). And calculate_imbalance() and 9649 * find_busiest_group() avoid some of the usual balance conditions to allow it 9650 * to create an effective group imbalance. 9651 * 9652 * This is a somewhat tricky proposition since the next run might not find the 9653 * group imbalance and decide the groups need to be balanced again. A most 9654 * subtle and fragile situation. 9655 */ 9656 9657 static inline int sg_imbalanced(struct sched_group *group) 9658 { 9659 return group->sgc->imbalance; 9660 } 9661 9662 /* 9663 * group_has_capacity returns true if the group has spare capacity that could 9664 * be used by some tasks. 9665 * We consider that a group has spare capacity if the number of task is 9666 * smaller than the number of CPUs or if the utilization is lower than the 9667 * available capacity for CFS tasks. 9668 * For the latter, we use a threshold to stabilize the state, to take into 9669 * account the variance of the tasks' load and to return true if the available 9670 * capacity in meaningful for the load balancer. 9671 * As an example, an available capacity of 1% can appear but it doesn't make 9672 * any benefit for the load balance. 9673 */ 9674 static inline bool 9675 group_has_capacity(unsigned int imbalance_pct, struct sg_lb_stats *sgs) 9676 { 9677 if (sgs->sum_nr_running < sgs->group_weight) 9678 return true; 9679 9680 if ((sgs->group_capacity * imbalance_pct) < 9681 (sgs->group_runnable * 100)) 9682 return false; 9683 9684 if ((sgs->group_capacity * 100) > 9685 (sgs->group_util * imbalance_pct)) 9686 return true; 9687 9688 return false; 9689 } 9690 9691 /* 9692 * group_is_overloaded returns true if the group has more tasks than it can 9693 * handle. 9694 * group_is_overloaded is not equals to !group_has_capacity because a group 9695 * with the exact right number of tasks, has no more spare capacity but is not 9696 * overloaded so both group_has_capacity and group_is_overloaded return 9697 * false. 9698 */ 9699 static inline bool 9700 group_is_overloaded(unsigned int imbalance_pct, struct sg_lb_stats *sgs) 9701 { 9702 if (sgs->sum_nr_running <= sgs->group_weight) 9703 return false; 9704 9705 if ((sgs->group_capacity * 100) < 9706 (sgs->group_util * imbalance_pct)) 9707 return true; 9708 9709 if ((sgs->group_capacity * imbalance_pct) < 9710 (sgs->group_runnable * 100)) 9711 return true; 9712 9713 return false; 9714 } 9715 9716 static inline enum 9717 group_type group_classify(unsigned int imbalance_pct, 9718 struct sched_group *group, 9719 struct sg_lb_stats *sgs) 9720 { 9721 if (group_is_overloaded(imbalance_pct, sgs)) 9722 return group_overloaded; 9723 9724 if (sg_imbalanced(group)) 9725 return group_imbalanced; 9726 9727 if (sgs->group_asym_packing) 9728 return group_asym_packing; 9729 9730 if (sgs->group_smt_balance) 9731 return group_smt_balance; 9732 9733 if (sgs->group_misfit_task_load) 9734 return group_misfit_task; 9735 9736 if (!group_has_capacity(imbalance_pct, sgs)) 9737 return group_fully_busy; 9738 9739 return group_has_spare; 9740 } 9741 9742 /** 9743 * sched_use_asym_prio - Check whether asym_packing priority must be used 9744 * @sd: The scheduling domain of the load balancing 9745 * @cpu: A CPU 9746 * 9747 * Always use CPU priority when balancing load between SMT siblings. When 9748 * balancing load between cores, it is not sufficient that @cpu is idle. Only 9749 * use CPU priority if the whole core is idle. 9750 * 9751 * Returns: True if the priority of @cpu must be followed. False otherwise. 9752 */ 9753 static bool sched_use_asym_prio(struct sched_domain *sd, int cpu) 9754 { 9755 if (!sched_smt_active()) 9756 return true; 9757 9758 return sd->flags & SD_SHARE_CPUCAPACITY || is_core_idle(cpu); 9759 } 9760 9761 /** 9762 * sched_asym - Check if the destination CPU can do asym_packing load balance 9763 * @env: The load balancing environment 9764 * @sds: Load-balancing data with statistics of the local group 9765 * @sgs: Load-balancing statistics of the candidate busiest group 9766 * @group: The candidate busiest group 9767 * 9768 * @env::dst_cpu can do asym_packing if it has higher priority than the 9769 * preferred CPU of @group. 9770 * 9771 * SMT is a special case. If we are balancing load between cores, @env::dst_cpu 9772 * can do asym_packing balance only if all its SMT siblings are idle. Also, it 9773 * can only do it if @group is an SMT group and has exactly on busy CPU. Larger 9774 * imbalances in the number of CPUS are dealt with in find_busiest_group(). 9775 * 9776 * If we are balancing load within an SMT core, or at DIE domain level, always 9777 * proceed. 9778 * 9779 * Return: true if @env::dst_cpu can do with asym_packing load balance. False 9780 * otherwise. 9781 */ 9782 static inline bool 9783 sched_asym(struct lb_env *env, struct sd_lb_stats *sds, struct sg_lb_stats *sgs, 9784 struct sched_group *group) 9785 { 9786 /* Ensure that the whole local core is idle, if applicable. */ 9787 if (!sched_use_asym_prio(env->sd, env->dst_cpu)) 9788 return false; 9789 9790 /* 9791 * CPU priorities does not make sense for SMT cores with more than one 9792 * busy sibling. 9793 */ 9794 if (group->flags & SD_SHARE_CPUCAPACITY) { 9795 if (sgs->group_weight - sgs->idle_cpus != 1) 9796 return false; 9797 } 9798 9799 return sched_asym_prefer(env->dst_cpu, group->asym_prefer_cpu); 9800 } 9801 9802 /* One group has more than one SMT CPU while the other group does not */ 9803 static inline bool smt_vs_nonsmt_groups(struct sched_group *sg1, 9804 struct sched_group *sg2) 9805 { 9806 if (!sg1 || !sg2) 9807 return false; 9808 9809 return (sg1->flags & SD_SHARE_CPUCAPACITY) != 9810 (sg2->flags & SD_SHARE_CPUCAPACITY); 9811 } 9812 9813 static inline bool smt_balance(struct lb_env *env, struct sg_lb_stats *sgs, 9814 struct sched_group *group) 9815 { 9816 if (env->idle == CPU_NOT_IDLE) 9817 return false; 9818 9819 /* 9820 * For SMT source group, it is better to move a task 9821 * to a CPU that doesn't have multiple tasks sharing its CPU capacity. 9822 * Note that if a group has a single SMT, SD_SHARE_CPUCAPACITY 9823 * will not be on. 9824 */ 9825 if (group->flags & SD_SHARE_CPUCAPACITY && 9826 sgs->sum_h_nr_running > 1) 9827 return true; 9828 9829 return false; 9830 } 9831 9832 static inline long sibling_imbalance(struct lb_env *env, 9833 struct sd_lb_stats *sds, 9834 struct sg_lb_stats *busiest, 9835 struct sg_lb_stats *local) 9836 { 9837 int ncores_busiest, ncores_local; 9838 long imbalance; 9839 9840 if (env->idle == CPU_NOT_IDLE || !busiest->sum_nr_running) 9841 return 0; 9842 9843 ncores_busiest = sds->busiest->cores; 9844 ncores_local = sds->local->cores; 9845 9846 if (ncores_busiest == ncores_local) { 9847 imbalance = busiest->sum_nr_running; 9848 lsub_positive(&imbalance, local->sum_nr_running); 9849 return imbalance; 9850 } 9851 9852 /* Balance such that nr_running/ncores ratio are same on both groups */ 9853 imbalance = ncores_local * busiest->sum_nr_running; 9854 lsub_positive(&imbalance, ncores_busiest * local->sum_nr_running); 9855 /* Normalize imbalance and do rounding on normalization */ 9856 imbalance = 2 * imbalance + ncores_local + ncores_busiest; 9857 imbalance /= ncores_local + ncores_busiest; 9858 9859 /* Take advantage of resource in an empty sched group */ 9860 if (imbalance <= 1 && local->sum_nr_running == 0 && 9861 busiest->sum_nr_running > 1) 9862 imbalance = 2; 9863 9864 return imbalance; 9865 } 9866 9867 static inline bool 9868 sched_reduced_capacity(struct rq *rq, struct sched_domain *sd) 9869 { 9870 /* 9871 * When there is more than 1 task, the group_overloaded case already 9872 * takes care of cpu with reduced capacity 9873 */ 9874 if (rq->cfs.h_nr_running != 1) 9875 return false; 9876 9877 return check_cpu_capacity(rq, sd); 9878 } 9879 9880 /** 9881 * update_sg_lb_stats - Update sched_group's statistics for load balancing. 9882 * @env: The load balancing environment. 9883 * @sds: Load-balancing data with statistics of the local group. 9884 * @group: sched_group whose statistics are to be updated. 9885 * @sgs: variable to hold the statistics for this group. 9886 * @sg_status: Holds flag indicating the status of the sched_group 9887 */ 9888 static inline void update_sg_lb_stats(struct lb_env *env, 9889 struct sd_lb_stats *sds, 9890 struct sched_group *group, 9891 struct sg_lb_stats *sgs, 9892 int *sg_status) 9893 { 9894 int i, nr_running, local_group; 9895 9896 memset(sgs, 0, sizeof(*sgs)); 9897 9898 local_group = group == sds->local; 9899 9900 for_each_cpu_and(i, sched_group_span(group), env->cpus) { 9901 struct rq *rq = cpu_rq(i); 9902 unsigned long load = cpu_load(rq); 9903 9904 sgs->group_load += load; 9905 sgs->group_util += cpu_util_cfs(i); 9906 sgs->group_runnable += cpu_runnable(rq); 9907 sgs->sum_h_nr_running += rq->cfs.h_nr_running; 9908 9909 nr_running = rq->nr_running; 9910 sgs->sum_nr_running += nr_running; 9911 9912 if (nr_running > 1) 9913 *sg_status |= SG_OVERLOAD; 9914 9915 if (cpu_overutilized(i)) 9916 *sg_status |= SG_OVERUTILIZED; 9917 9918 #ifdef CONFIG_NUMA_BALANCING 9919 sgs->nr_numa_running += rq->nr_numa_running; 9920 sgs->nr_preferred_running += rq->nr_preferred_running; 9921 #endif 9922 /* 9923 * No need to call idle_cpu() if nr_running is not 0 9924 */ 9925 if (!nr_running && idle_cpu(i)) { 9926 sgs->idle_cpus++; 9927 /* Idle cpu can't have misfit task */ 9928 continue; 9929 } 9930 9931 if (local_group) 9932 continue; 9933 9934 if (env->sd->flags & SD_ASYM_CPUCAPACITY) { 9935 /* Check for a misfit task on the cpu */ 9936 if (sgs->group_misfit_task_load < rq->misfit_task_load) { 9937 sgs->group_misfit_task_load = rq->misfit_task_load; 9938 *sg_status |= SG_OVERLOAD; 9939 } 9940 } else if ((env->idle != CPU_NOT_IDLE) && 9941 sched_reduced_capacity(rq, env->sd)) { 9942 /* Check for a task running on a CPU with reduced capacity */ 9943 if (sgs->group_misfit_task_load < load) 9944 sgs->group_misfit_task_load = load; 9945 } 9946 } 9947 9948 sgs->group_capacity = group->sgc->capacity; 9949 9950 sgs->group_weight = group->group_weight; 9951 9952 /* Check if dst CPU is idle and preferred to this group */ 9953 if (!local_group && env->sd->flags & SD_ASYM_PACKING && 9954 env->idle != CPU_NOT_IDLE && sgs->sum_h_nr_running && 9955 sched_asym(env, sds, sgs, group)) { 9956 sgs->group_asym_packing = 1; 9957 } 9958 9959 /* Check for loaded SMT group to be balanced to dst CPU */ 9960 if (!local_group && smt_balance(env, sgs, group)) 9961 sgs->group_smt_balance = 1; 9962 9963 sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs); 9964 9965 /* Computing avg_load makes sense only when group is overloaded */ 9966 if (sgs->group_type == group_overloaded) 9967 sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) / 9968 sgs->group_capacity; 9969 } 9970 9971 /** 9972 * update_sd_pick_busiest - return 1 on busiest group 9973 * @env: The load balancing environment. 9974 * @sds: sched_domain statistics 9975 * @sg: sched_group candidate to be checked for being the busiest 9976 * @sgs: sched_group statistics 9977 * 9978 * Determine if @sg is a busier group than the previously selected 9979 * busiest group. 9980 * 9981 * Return: %true if @sg is a busier group than the previously selected 9982 * busiest group. %false otherwise. 9983 */ 9984 static bool update_sd_pick_busiest(struct lb_env *env, 9985 struct sd_lb_stats *sds, 9986 struct sched_group *sg, 9987 struct sg_lb_stats *sgs) 9988 { 9989 struct sg_lb_stats *busiest = &sds->busiest_stat; 9990 9991 /* Make sure that there is at least one task to pull */ 9992 if (!sgs->sum_h_nr_running) 9993 return false; 9994 9995 /* 9996 * Don't try to pull misfit tasks we can't help. 9997 * We can use max_capacity here as reduction in capacity on some 9998 * CPUs in the group should either be possible to resolve 9999 * internally or be covered by avg_load imbalance (eventually). 10000 */ 10001 if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && 10002 (sgs->group_type == group_misfit_task) && 10003 (!capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) || 10004 sds->local_stat.group_type != group_has_spare)) 10005 return false; 10006 10007 if (sgs->group_type > busiest->group_type) 10008 return true; 10009 10010 if (sgs->group_type < busiest->group_type) 10011 return false; 10012 10013 /* 10014 * The candidate and the current busiest group are the same type of 10015 * group. Let check which one is the busiest according to the type. 10016 */ 10017 10018 switch (sgs->group_type) { 10019 case group_overloaded: 10020 /* Select the overloaded group with highest avg_load. */ 10021 if (sgs->avg_load <= busiest->avg_load) 10022 return false; 10023 break; 10024 10025 case group_imbalanced: 10026 /* 10027 * Select the 1st imbalanced group as we don't have any way to 10028 * choose one more than another. 10029 */ 10030 return false; 10031 10032 case group_asym_packing: 10033 /* Prefer to move from lowest priority CPU's work */ 10034 if (sched_asym_prefer(sg->asym_prefer_cpu, sds->busiest->asym_prefer_cpu)) 10035 return false; 10036 break; 10037 10038 case group_misfit_task: 10039 /* 10040 * If we have more than one misfit sg go with the biggest 10041 * misfit. 10042 */ 10043 if (sgs->group_misfit_task_load < busiest->group_misfit_task_load) 10044 return false; 10045 break; 10046 10047 case group_smt_balance: 10048 /* 10049 * Check if we have spare CPUs on either SMT group to 10050 * choose has spare or fully busy handling. 10051 */ 10052 if (sgs->idle_cpus != 0 || busiest->idle_cpus != 0) 10053 goto has_spare; 10054 10055 fallthrough; 10056 10057 case group_fully_busy: 10058 /* 10059 * Select the fully busy group with highest avg_load. In 10060 * theory, there is no need to pull task from such kind of 10061 * group because tasks have all compute capacity that they need 10062 * but we can still improve the overall throughput by reducing 10063 * contention when accessing shared HW resources. 10064 * 10065 * XXX for now avg_load is not computed and always 0 so we 10066 * select the 1st one, except if @sg is composed of SMT 10067 * siblings. 10068 */ 10069 10070 if (sgs->avg_load < busiest->avg_load) 10071 return false; 10072 10073 if (sgs->avg_load == busiest->avg_load) { 10074 /* 10075 * SMT sched groups need more help than non-SMT groups. 10076 * If @sg happens to also be SMT, either choice is good. 10077 */ 10078 if (sds->busiest->flags & SD_SHARE_CPUCAPACITY) 10079 return false; 10080 } 10081 10082 break; 10083 10084 case group_has_spare: 10085 /* 10086 * Do not pick sg with SMT CPUs over sg with pure CPUs, 10087 * as we do not want to pull task off SMT core with one task 10088 * and make the core idle. 10089 */ 10090 if (smt_vs_nonsmt_groups(sds->busiest, sg)) { 10091 if (sg->flags & SD_SHARE_CPUCAPACITY && sgs->sum_h_nr_running <= 1) 10092 return false; 10093 else 10094 return true; 10095 } 10096 has_spare: 10097 10098 /* 10099 * Select not overloaded group with lowest number of idle cpus 10100 * and highest number of running tasks. We could also compare 10101 * the spare capacity which is more stable but it can end up 10102 * that the group has less spare capacity but finally more idle 10103 * CPUs which means less opportunity to pull tasks. 10104 */ 10105 if (sgs->idle_cpus > busiest->idle_cpus) 10106 return false; 10107 else if ((sgs->idle_cpus == busiest->idle_cpus) && 10108 (sgs->sum_nr_running <= busiest->sum_nr_running)) 10109 return false; 10110 10111 break; 10112 } 10113 10114 /* 10115 * Candidate sg has no more than one task per CPU and has higher 10116 * per-CPU capacity. Migrating tasks to less capable CPUs may harm 10117 * throughput. Maximize throughput, power/energy consequences are not 10118 * considered. 10119 */ 10120 if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && 10121 (sgs->group_type <= group_fully_busy) && 10122 (capacity_greater(sg->sgc->min_capacity, capacity_of(env->dst_cpu)))) 10123 return false; 10124 10125 return true; 10126 } 10127 10128 #ifdef CONFIG_NUMA_BALANCING 10129 static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs) 10130 { 10131 if (sgs->sum_h_nr_running > sgs->nr_numa_running) 10132 return regular; 10133 if (sgs->sum_h_nr_running > sgs->nr_preferred_running) 10134 return remote; 10135 return all; 10136 } 10137 10138 static inline enum fbq_type fbq_classify_rq(struct rq *rq) 10139 { 10140 if (rq->nr_running > rq->nr_numa_running) 10141 return regular; 10142 if (rq->nr_running > rq->nr_preferred_running) 10143 return remote; 10144 return all; 10145 } 10146 #else 10147 static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs) 10148 { 10149 return all; 10150 } 10151 10152 static inline enum fbq_type fbq_classify_rq(struct rq *rq) 10153 { 10154 return regular; 10155 } 10156 #endif /* CONFIG_NUMA_BALANCING */ 10157 10158 10159 struct sg_lb_stats; 10160 10161 /* 10162 * task_running_on_cpu - return 1 if @p is running on @cpu. 10163 */ 10164 10165 static unsigned int task_running_on_cpu(int cpu, struct task_struct *p) 10166 { 10167 /* Task has no contribution or is new */ 10168 if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 10169 return 0; 10170 10171 if (task_on_rq_queued(p)) 10172 return 1; 10173 10174 return 0; 10175 } 10176 10177 /** 10178 * idle_cpu_without - would a given CPU be idle without p ? 10179 * @cpu: the processor on which idleness is tested. 10180 * @p: task which should be ignored. 10181 * 10182 * Return: 1 if the CPU would be idle. 0 otherwise. 10183 */ 10184 static int idle_cpu_without(int cpu, struct task_struct *p) 10185 { 10186 struct rq *rq = cpu_rq(cpu); 10187 10188 if (rq->curr != rq->idle && rq->curr != p) 10189 return 0; 10190 10191 /* 10192 * rq->nr_running can't be used but an updated version without the 10193 * impact of p on cpu must be used instead. The updated nr_running 10194 * be computed and tested before calling idle_cpu_without(). 10195 */ 10196 10197 #ifdef CONFIG_SMP 10198 if (rq->ttwu_pending) 10199 return 0; 10200 #endif 10201 10202 return 1; 10203 } 10204 10205 /* 10206 * update_sg_wakeup_stats - Update sched_group's statistics for wakeup. 10207 * @sd: The sched_domain level to look for idlest group. 10208 * @group: sched_group whose statistics are to be updated. 10209 * @sgs: variable to hold the statistics for this group. 10210 * @p: The task for which we look for the idlest group/CPU. 10211 */ 10212 static inline void update_sg_wakeup_stats(struct sched_domain *sd, 10213 struct sched_group *group, 10214 struct sg_lb_stats *sgs, 10215 struct task_struct *p) 10216 { 10217 int i, nr_running; 10218 10219 memset(sgs, 0, sizeof(*sgs)); 10220 10221 /* Assume that task can't fit any CPU of the group */ 10222 if (sd->flags & SD_ASYM_CPUCAPACITY) 10223 sgs->group_misfit_task_load = 1; 10224 10225 for_each_cpu(i, sched_group_span(group)) { 10226 struct rq *rq = cpu_rq(i); 10227 unsigned int local; 10228 10229 sgs->group_load += cpu_load_without(rq, p); 10230 sgs->group_util += cpu_util_without(i, p); 10231 sgs->group_runnable += cpu_runnable_without(rq, p); 10232 local = task_running_on_cpu(i, p); 10233 sgs->sum_h_nr_running += rq->cfs.h_nr_running - local; 10234 10235 nr_running = rq->nr_running - local; 10236 sgs->sum_nr_running += nr_running; 10237 10238 /* 10239 * No need to call idle_cpu_without() if nr_running is not 0 10240 */ 10241 if (!nr_running && idle_cpu_without(i, p)) 10242 sgs->idle_cpus++; 10243 10244 /* Check if task fits in the CPU */ 10245 if (sd->flags & SD_ASYM_CPUCAPACITY && 10246 sgs->group_misfit_task_load && 10247 task_fits_cpu(p, i)) 10248 sgs->group_misfit_task_load = 0; 10249 10250 } 10251 10252 sgs->group_capacity = group->sgc->capacity; 10253 10254 sgs->group_weight = group->group_weight; 10255 10256 sgs->group_type = group_classify(sd->imbalance_pct, group, sgs); 10257 10258 /* 10259 * Computing avg_load makes sense only when group is fully busy or 10260 * overloaded 10261 */ 10262 if (sgs->group_type == group_fully_busy || 10263 sgs->group_type == group_overloaded) 10264 sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) / 10265 sgs->group_capacity; 10266 } 10267 10268 static bool update_pick_idlest(struct sched_group *idlest, 10269 struct sg_lb_stats *idlest_sgs, 10270 struct sched_group *group, 10271 struct sg_lb_stats *sgs) 10272 { 10273 if (sgs->group_type < idlest_sgs->group_type) 10274 return true; 10275 10276 if (sgs->group_type > idlest_sgs->group_type) 10277 return false; 10278 10279 /* 10280 * The candidate and the current idlest group are the same type of 10281 * group. Let check which one is the idlest according to the type. 10282 */ 10283 10284 switch (sgs->group_type) { 10285 case group_overloaded: 10286 case group_fully_busy: 10287 /* Select the group with lowest avg_load. */ 10288 if (idlest_sgs->avg_load <= sgs->avg_load) 10289 return false; 10290 break; 10291 10292 case group_imbalanced: 10293 case group_asym_packing: 10294 case group_smt_balance: 10295 /* Those types are not used in the slow wakeup path */ 10296 return false; 10297 10298 case group_misfit_task: 10299 /* Select group with the highest max capacity */ 10300 if (idlest->sgc->max_capacity >= group->sgc->max_capacity) 10301 return false; 10302 break; 10303 10304 case group_has_spare: 10305 /* Select group with most idle CPUs */ 10306 if (idlest_sgs->idle_cpus > sgs->idle_cpus) 10307 return false; 10308 10309 /* Select group with lowest group_util */ 10310 if (idlest_sgs->idle_cpus == sgs->idle_cpus && 10311 idlest_sgs->group_util <= sgs->group_util) 10312 return false; 10313 10314 break; 10315 } 10316 10317 return true; 10318 } 10319 10320 /* 10321 * find_idlest_group() finds and returns the least busy CPU group within the 10322 * domain. 10323 * 10324 * Assumes p is allowed on at least one CPU in sd. 10325 */ 10326 static struct sched_group * 10327 find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu) 10328 { 10329 struct sched_group *idlest = NULL, *local = NULL, *group = sd->groups; 10330 struct sg_lb_stats local_sgs, tmp_sgs; 10331 struct sg_lb_stats *sgs; 10332 unsigned long imbalance; 10333 struct sg_lb_stats idlest_sgs = { 10334 .avg_load = UINT_MAX, 10335 .group_type = group_overloaded, 10336 }; 10337 10338 do { 10339 int local_group; 10340 10341 /* Skip over this group if it has no CPUs allowed */ 10342 if (!cpumask_intersects(sched_group_span(group), 10343 p->cpus_ptr)) 10344 continue; 10345 10346 /* Skip over this group if no cookie matched */ 10347 if (!sched_group_cookie_match(cpu_rq(this_cpu), p, group)) 10348 continue; 10349 10350 local_group = cpumask_test_cpu(this_cpu, 10351 sched_group_span(group)); 10352 10353 if (local_group) { 10354 sgs = &local_sgs; 10355 local = group; 10356 } else { 10357 sgs = &tmp_sgs; 10358 } 10359 10360 update_sg_wakeup_stats(sd, group, sgs, p); 10361 10362 if (!local_group && update_pick_idlest(idlest, &idlest_sgs, group, sgs)) { 10363 idlest = group; 10364 idlest_sgs = *sgs; 10365 } 10366 10367 } while (group = group->next, group != sd->groups); 10368 10369 10370 /* There is no idlest group to push tasks to */ 10371 if (!idlest) 10372 return NULL; 10373 10374 /* The local group has been skipped because of CPU affinity */ 10375 if (!local) 10376 return idlest; 10377 10378 /* 10379 * If the local group is idler than the selected idlest group 10380 * don't try and push the task. 10381 */ 10382 if (local_sgs.group_type < idlest_sgs.group_type) 10383 return NULL; 10384 10385 /* 10386 * If the local group is busier than the selected idlest group 10387 * try and push the task. 10388 */ 10389 if (local_sgs.group_type > idlest_sgs.group_type) 10390 return idlest; 10391 10392 switch (local_sgs.group_type) { 10393 case group_overloaded: 10394 case group_fully_busy: 10395 10396 /* Calculate allowed imbalance based on load */ 10397 imbalance = scale_load_down(NICE_0_LOAD) * 10398 (sd->imbalance_pct-100) / 100; 10399 10400 /* 10401 * When comparing groups across NUMA domains, it's possible for 10402 * the local domain to be very lightly loaded relative to the 10403 * remote domains but "imbalance" skews the comparison making 10404 * remote CPUs look much more favourable. When considering 10405 * cross-domain, add imbalance to the load on the remote node 10406 * and consider staying local. 10407 */ 10408 10409 if ((sd->flags & SD_NUMA) && 10410 ((idlest_sgs.avg_load + imbalance) >= local_sgs.avg_load)) 10411 return NULL; 10412 10413 /* 10414 * If the local group is less loaded than the selected 10415 * idlest group don't try and push any tasks. 10416 */ 10417 if (idlest_sgs.avg_load >= (local_sgs.avg_load + imbalance)) 10418 return NULL; 10419 10420 if (100 * local_sgs.avg_load <= sd->imbalance_pct * idlest_sgs.avg_load) 10421 return NULL; 10422 break; 10423 10424 case group_imbalanced: 10425 case group_asym_packing: 10426 case group_smt_balance: 10427 /* Those type are not used in the slow wakeup path */ 10428 return NULL; 10429 10430 case group_misfit_task: 10431 /* Select group with the highest max capacity */ 10432 if (local->sgc->max_capacity >= idlest->sgc->max_capacity) 10433 return NULL; 10434 break; 10435 10436 case group_has_spare: 10437 #ifdef CONFIG_NUMA 10438 if (sd->flags & SD_NUMA) { 10439 int imb_numa_nr = sd->imb_numa_nr; 10440 #ifdef CONFIG_NUMA_BALANCING 10441 int idlest_cpu; 10442 /* 10443 * If there is spare capacity at NUMA, try to select 10444 * the preferred node 10445 */ 10446 if (cpu_to_node(this_cpu) == p->numa_preferred_nid) 10447 return NULL; 10448 10449 idlest_cpu = cpumask_first(sched_group_span(idlest)); 10450 if (cpu_to_node(idlest_cpu) == p->numa_preferred_nid) 10451 return idlest; 10452 #endif /* CONFIG_NUMA_BALANCING */ 10453 /* 10454 * Otherwise, keep the task close to the wakeup source 10455 * and improve locality if the number of running tasks 10456 * would remain below threshold where an imbalance is 10457 * allowed while accounting for the possibility the 10458 * task is pinned to a subset of CPUs. If there is a 10459 * real need of migration, periodic load balance will 10460 * take care of it. 10461 */ 10462 if (p->nr_cpus_allowed != NR_CPUS) { 10463 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 10464 10465 cpumask_and(cpus, sched_group_span(local), p->cpus_ptr); 10466 imb_numa_nr = min(cpumask_weight(cpus), sd->imb_numa_nr); 10467 } 10468 10469 imbalance = abs(local_sgs.idle_cpus - idlest_sgs.idle_cpus); 10470 if (!adjust_numa_imbalance(imbalance, 10471 local_sgs.sum_nr_running + 1, 10472 imb_numa_nr)) { 10473 return NULL; 10474 } 10475 } 10476 #endif /* CONFIG_NUMA */ 10477 10478 /* 10479 * Select group with highest number of idle CPUs. We could also 10480 * compare the utilization which is more stable but it can end 10481 * up that the group has less spare capacity but finally more 10482 * idle CPUs which means more opportunity to run task. 10483 */ 10484 if (local_sgs.idle_cpus >= idlest_sgs.idle_cpus) 10485 return NULL; 10486 break; 10487 } 10488 10489 return idlest; 10490 } 10491 10492 static void update_idle_cpu_scan(struct lb_env *env, 10493 unsigned long sum_util) 10494 { 10495 struct sched_domain_shared *sd_share; 10496 int llc_weight, pct; 10497 u64 x, y, tmp; 10498 /* 10499 * Update the number of CPUs to scan in LLC domain, which could 10500 * be used as a hint in select_idle_cpu(). The update of sd_share 10501 * could be expensive because it is within a shared cache line. 10502 * So the write of this hint only occurs during periodic load 10503 * balancing, rather than CPU_NEWLY_IDLE, because the latter 10504 * can fire way more frequently than the former. 10505 */ 10506 if (!sched_feat(SIS_UTIL) || env->idle == CPU_NEWLY_IDLE) 10507 return; 10508 10509 llc_weight = per_cpu(sd_llc_size, env->dst_cpu); 10510 if (env->sd->span_weight != llc_weight) 10511 return; 10512 10513 sd_share = rcu_dereference(per_cpu(sd_llc_shared, env->dst_cpu)); 10514 if (!sd_share) 10515 return; 10516 10517 /* 10518 * The number of CPUs to search drops as sum_util increases, when 10519 * sum_util hits 85% or above, the scan stops. 10520 * The reason to choose 85% as the threshold is because this is the 10521 * imbalance_pct(117) when a LLC sched group is overloaded. 10522 * 10523 * let y = SCHED_CAPACITY_SCALE - p * x^2 [1] 10524 * and y'= y / SCHED_CAPACITY_SCALE 10525 * 10526 * x is the ratio of sum_util compared to the CPU capacity: 10527 * x = sum_util / (llc_weight * SCHED_CAPACITY_SCALE) 10528 * y' is the ratio of CPUs to be scanned in the LLC domain, 10529 * and the number of CPUs to scan is calculated by: 10530 * 10531 * nr_scan = llc_weight * y' [2] 10532 * 10533 * When x hits the threshold of overloaded, AKA, when 10534 * x = 100 / pct, y drops to 0. According to [1], 10535 * p should be SCHED_CAPACITY_SCALE * pct^2 / 10000 10536 * 10537 * Scale x by SCHED_CAPACITY_SCALE: 10538 * x' = sum_util / llc_weight; [3] 10539 * 10540 * and finally [1] becomes: 10541 * y = SCHED_CAPACITY_SCALE - 10542 * x'^2 * pct^2 / (10000 * SCHED_CAPACITY_SCALE) [4] 10543 * 10544 */ 10545 /* equation [3] */ 10546 x = sum_util; 10547 do_div(x, llc_weight); 10548 10549 /* equation [4] */ 10550 pct = env->sd->imbalance_pct; 10551 tmp = x * x * pct * pct; 10552 do_div(tmp, 10000 * SCHED_CAPACITY_SCALE); 10553 tmp = min_t(long, tmp, SCHED_CAPACITY_SCALE); 10554 y = SCHED_CAPACITY_SCALE - tmp; 10555 10556 /* equation [2] */ 10557 y *= llc_weight; 10558 do_div(y, SCHED_CAPACITY_SCALE); 10559 if ((int)y != sd_share->nr_idle_scan) 10560 WRITE_ONCE(sd_share->nr_idle_scan, (int)y); 10561 } 10562 10563 /** 10564 * update_sd_lb_stats - Update sched_domain's statistics for load balancing. 10565 * @env: The load balancing environment. 10566 * @sds: variable to hold the statistics for this sched_domain. 10567 */ 10568 10569 static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sds) 10570 { 10571 struct sched_group *sg = env->sd->groups; 10572 struct sg_lb_stats *local = &sds->local_stat; 10573 struct sg_lb_stats tmp_sgs; 10574 unsigned long sum_util = 0; 10575 int sg_status = 0; 10576 10577 do { 10578 struct sg_lb_stats *sgs = &tmp_sgs; 10579 int local_group; 10580 10581 local_group = cpumask_test_cpu(env->dst_cpu, sched_group_span(sg)); 10582 if (local_group) { 10583 sds->local = sg; 10584 sgs = local; 10585 10586 if (env->idle != CPU_NEWLY_IDLE || 10587 time_after_eq(jiffies, sg->sgc->next_update)) 10588 update_group_capacity(env->sd, env->dst_cpu); 10589 } 10590 10591 update_sg_lb_stats(env, sds, sg, sgs, &sg_status); 10592 10593 if (local_group) 10594 goto next_group; 10595 10596 10597 if (update_sd_pick_busiest(env, sds, sg, sgs)) { 10598 sds->busiest = sg; 10599 sds->busiest_stat = *sgs; 10600 } 10601 10602 next_group: 10603 /* Now, start updating sd_lb_stats */ 10604 sds->total_load += sgs->group_load; 10605 sds->total_capacity += sgs->group_capacity; 10606 10607 sum_util += sgs->group_util; 10608 sg = sg->next; 10609 } while (sg != env->sd->groups); 10610 10611 /* 10612 * Indicate that the child domain of the busiest group prefers tasks 10613 * go to a child's sibling domains first. NB the flags of a sched group 10614 * are those of the child domain. 10615 */ 10616 if (sds->busiest) 10617 sds->prefer_sibling = !!(sds->busiest->flags & SD_PREFER_SIBLING); 10618 10619 10620 if (env->sd->flags & SD_NUMA) 10621 env->fbq_type = fbq_classify_group(&sds->busiest_stat); 10622 10623 if (!env->sd->parent) { 10624 /* update overload indicator if we are at root domain */ 10625 WRITE_ONCE(env->dst_rq->rd->overload, sg_status & SG_OVERLOAD); 10626 10627 /* Update over-utilization (tipping point, U >= 0) indicator */ 10628 set_rd_overutilized_status(env->dst_rq->rd, 10629 sg_status & SG_OVERUTILIZED); 10630 } else if (sg_status & SG_OVERUTILIZED) { 10631 set_rd_overutilized_status(env->dst_rq->rd, SG_OVERUTILIZED); 10632 } 10633 10634 update_idle_cpu_scan(env, sum_util); 10635 } 10636 10637 /** 10638 * calculate_imbalance - Calculate the amount of imbalance present within the 10639 * groups of a given sched_domain during load balance. 10640 * @env: load balance environment 10641 * @sds: statistics of the sched_domain whose imbalance is to be calculated. 10642 */ 10643 static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds) 10644 { 10645 struct sg_lb_stats *local, *busiest; 10646 10647 local = &sds->local_stat; 10648 busiest = &sds->busiest_stat; 10649 10650 if (busiest->group_type == group_misfit_task) { 10651 if (env->sd->flags & SD_ASYM_CPUCAPACITY) { 10652 /* Set imbalance to allow misfit tasks to be balanced. */ 10653 env->migration_type = migrate_misfit; 10654 env->imbalance = 1; 10655 } else { 10656 /* 10657 * Set load imbalance to allow moving task from cpu 10658 * with reduced capacity. 10659 */ 10660 env->migration_type = migrate_load; 10661 env->imbalance = busiest->group_misfit_task_load; 10662 } 10663 return; 10664 } 10665 10666 if (busiest->group_type == group_asym_packing) { 10667 /* 10668 * In case of asym capacity, we will try to migrate all load to 10669 * the preferred CPU. 10670 */ 10671 env->migration_type = migrate_task; 10672 env->imbalance = busiest->sum_h_nr_running; 10673 return; 10674 } 10675 10676 if (busiest->group_type == group_smt_balance) { 10677 /* Reduce number of tasks sharing CPU capacity */ 10678 env->migration_type = migrate_task; 10679 env->imbalance = 1; 10680 return; 10681 } 10682 10683 if (busiest->group_type == group_imbalanced) { 10684 /* 10685 * In the group_imb case we cannot rely on group-wide averages 10686 * to ensure CPU-load equilibrium, try to move any task to fix 10687 * the imbalance. The next load balance will take care of 10688 * balancing back the system. 10689 */ 10690 env->migration_type = migrate_task; 10691 env->imbalance = 1; 10692 return; 10693 } 10694 10695 /* 10696 * Try to use spare capacity of local group without overloading it or 10697 * emptying busiest. 10698 */ 10699 if (local->group_type == group_has_spare) { 10700 if ((busiest->group_type > group_fully_busy) && 10701 !(env->sd->flags & SD_SHARE_PKG_RESOURCES)) { 10702 /* 10703 * If busiest is overloaded, try to fill spare 10704 * capacity. This might end up creating spare capacity 10705 * in busiest or busiest still being overloaded but 10706 * there is no simple way to directly compute the 10707 * amount of load to migrate in order to balance the 10708 * system. 10709 */ 10710 env->migration_type = migrate_util; 10711 env->imbalance = max(local->group_capacity, local->group_util) - 10712 local->group_util; 10713 10714 /* 10715 * In some cases, the group's utilization is max or even 10716 * higher than capacity because of migrations but the 10717 * local CPU is (newly) idle. There is at least one 10718 * waiting task in this overloaded busiest group. Let's 10719 * try to pull it. 10720 */ 10721 if (env->idle != CPU_NOT_IDLE && env->imbalance == 0) { 10722 env->migration_type = migrate_task; 10723 env->imbalance = 1; 10724 } 10725 10726 return; 10727 } 10728 10729 if (busiest->group_weight == 1 || sds->prefer_sibling) { 10730 /* 10731 * When prefer sibling, evenly spread running tasks on 10732 * groups. 10733 */ 10734 env->migration_type = migrate_task; 10735 env->imbalance = sibling_imbalance(env, sds, busiest, local); 10736 } else { 10737 10738 /* 10739 * If there is no overload, we just want to even the number of 10740 * idle cpus. 10741 */ 10742 env->migration_type = migrate_task; 10743 env->imbalance = max_t(long, 0, 10744 (local->idle_cpus - busiest->idle_cpus)); 10745 } 10746 10747 #ifdef CONFIG_NUMA 10748 /* Consider allowing a small imbalance between NUMA groups */ 10749 if (env->sd->flags & SD_NUMA) { 10750 env->imbalance = adjust_numa_imbalance(env->imbalance, 10751 local->sum_nr_running + 1, 10752 env->sd->imb_numa_nr); 10753 } 10754 #endif 10755 10756 /* Number of tasks to move to restore balance */ 10757 env->imbalance >>= 1; 10758 10759 return; 10760 } 10761 10762 /* 10763 * Local is fully busy but has to take more load to relieve the 10764 * busiest group 10765 */ 10766 if (local->group_type < group_overloaded) { 10767 /* 10768 * Local will become overloaded so the avg_load metrics are 10769 * finally needed. 10770 */ 10771 10772 local->avg_load = (local->group_load * SCHED_CAPACITY_SCALE) / 10773 local->group_capacity; 10774 10775 /* 10776 * If the local group is more loaded than the selected 10777 * busiest group don't try to pull any tasks. 10778 */ 10779 if (local->avg_load >= busiest->avg_load) { 10780 env->imbalance = 0; 10781 return; 10782 } 10783 10784 sds->avg_load = (sds->total_load * SCHED_CAPACITY_SCALE) / 10785 sds->total_capacity; 10786 10787 /* 10788 * If the local group is more loaded than the average system 10789 * load, don't try to pull any tasks. 10790 */ 10791 if (local->avg_load >= sds->avg_load) { 10792 env->imbalance = 0; 10793 return; 10794 } 10795 10796 } 10797 10798 /* 10799 * Both group are or will become overloaded and we're trying to get all 10800 * the CPUs to the average_load, so we don't want to push ourselves 10801 * above the average load, nor do we wish to reduce the max loaded CPU 10802 * below the average load. At the same time, we also don't want to 10803 * reduce the group load below the group capacity. Thus we look for 10804 * the minimum possible imbalance. 10805 */ 10806 env->migration_type = migrate_load; 10807 env->imbalance = min( 10808 (busiest->avg_load - sds->avg_load) * busiest->group_capacity, 10809 (sds->avg_load - local->avg_load) * local->group_capacity 10810 ) / SCHED_CAPACITY_SCALE; 10811 } 10812 10813 /******* find_busiest_group() helpers end here *********************/ 10814 10815 /* 10816 * Decision matrix according to the local and busiest group type: 10817 * 10818 * busiest \ local has_spare fully_busy misfit asym imbalanced overloaded 10819 * has_spare nr_idle balanced N/A N/A balanced balanced 10820 * fully_busy nr_idle nr_idle N/A N/A balanced balanced 10821 * misfit_task force N/A N/A N/A N/A N/A 10822 * asym_packing force force N/A N/A force force 10823 * imbalanced force force N/A N/A force force 10824 * overloaded force force N/A N/A force avg_load 10825 * 10826 * N/A : Not Applicable because already filtered while updating 10827 * statistics. 10828 * balanced : The system is balanced for these 2 groups. 10829 * force : Calculate the imbalance as load migration is probably needed. 10830 * avg_load : Only if imbalance is significant enough. 10831 * nr_idle : dst_cpu is not busy and the number of idle CPUs is quite 10832 * different in groups. 10833 */ 10834 10835 /** 10836 * find_busiest_group - Returns the busiest group within the sched_domain 10837 * if there is an imbalance. 10838 * @env: The load balancing environment. 10839 * 10840 * Also calculates the amount of runnable load which should be moved 10841 * to restore balance. 10842 * 10843 * Return: - The busiest group if imbalance exists. 10844 */ 10845 static struct sched_group *find_busiest_group(struct lb_env *env) 10846 { 10847 struct sg_lb_stats *local, *busiest; 10848 struct sd_lb_stats sds; 10849 10850 init_sd_lb_stats(&sds); 10851 10852 /* 10853 * Compute the various statistics relevant for load balancing at 10854 * this level. 10855 */ 10856 update_sd_lb_stats(env, &sds); 10857 10858 /* There is no busy sibling group to pull tasks from */ 10859 if (!sds.busiest) 10860 goto out_balanced; 10861 10862 busiest = &sds.busiest_stat; 10863 10864 /* Misfit tasks should be dealt with regardless of the avg load */ 10865 if (busiest->group_type == group_misfit_task) 10866 goto force_balance; 10867 10868 if (sched_energy_enabled()) { 10869 struct root_domain *rd = env->dst_rq->rd; 10870 10871 if (rcu_dereference(rd->pd) && !READ_ONCE(rd->overutilized)) 10872 goto out_balanced; 10873 } 10874 10875 /* ASYM feature bypasses nice load balance check */ 10876 if (busiest->group_type == group_asym_packing) 10877 goto force_balance; 10878 10879 /* 10880 * If the busiest group is imbalanced the below checks don't 10881 * work because they assume all things are equal, which typically 10882 * isn't true due to cpus_ptr constraints and the like. 10883 */ 10884 if (busiest->group_type == group_imbalanced) 10885 goto force_balance; 10886 10887 local = &sds.local_stat; 10888 /* 10889 * If the local group is busier than the selected busiest group 10890 * don't try and pull any tasks. 10891 */ 10892 if (local->group_type > busiest->group_type) 10893 goto out_balanced; 10894 10895 /* 10896 * When groups are overloaded, use the avg_load to ensure fairness 10897 * between tasks. 10898 */ 10899 if (local->group_type == group_overloaded) { 10900 /* 10901 * If the local group is more loaded than the selected 10902 * busiest group don't try to pull any tasks. 10903 */ 10904 if (local->avg_load >= busiest->avg_load) 10905 goto out_balanced; 10906 10907 /* XXX broken for overlapping NUMA groups */ 10908 sds.avg_load = (sds.total_load * SCHED_CAPACITY_SCALE) / 10909 sds.total_capacity; 10910 10911 /* 10912 * Don't pull any tasks if this group is already above the 10913 * domain average load. 10914 */ 10915 if (local->avg_load >= sds.avg_load) 10916 goto out_balanced; 10917 10918 /* 10919 * If the busiest group is more loaded, use imbalance_pct to be 10920 * conservative. 10921 */ 10922 if (100 * busiest->avg_load <= 10923 env->sd->imbalance_pct * local->avg_load) 10924 goto out_balanced; 10925 } 10926 10927 /* 10928 * Try to move all excess tasks to a sibling domain of the busiest 10929 * group's child domain. 10930 */ 10931 if (sds.prefer_sibling && local->group_type == group_has_spare && 10932 sibling_imbalance(env, &sds, busiest, local) > 1) 10933 goto force_balance; 10934 10935 if (busiest->group_type != group_overloaded) { 10936 if (env->idle == CPU_NOT_IDLE) { 10937 /* 10938 * If the busiest group is not overloaded (and as a 10939 * result the local one too) but this CPU is already 10940 * busy, let another idle CPU try to pull task. 10941 */ 10942 goto out_balanced; 10943 } 10944 10945 if (busiest->group_type == group_smt_balance && 10946 smt_vs_nonsmt_groups(sds.local, sds.busiest)) { 10947 /* Let non SMT CPU pull from SMT CPU sharing with sibling */ 10948 goto force_balance; 10949 } 10950 10951 if (busiest->group_weight > 1 && 10952 local->idle_cpus <= (busiest->idle_cpus + 1)) { 10953 /* 10954 * If the busiest group is not overloaded 10955 * and there is no imbalance between this and busiest 10956 * group wrt idle CPUs, it is balanced. The imbalance 10957 * becomes significant if the diff is greater than 1 10958 * otherwise we might end up to just move the imbalance 10959 * on another group. Of course this applies only if 10960 * there is more than 1 CPU per group. 10961 */ 10962 goto out_balanced; 10963 } 10964 10965 if (busiest->sum_h_nr_running == 1) { 10966 /* 10967 * busiest doesn't have any tasks waiting to run 10968 */ 10969 goto out_balanced; 10970 } 10971 } 10972 10973 force_balance: 10974 /* Looks like there is an imbalance. Compute it */ 10975 calculate_imbalance(env, &sds); 10976 return env->imbalance ? sds.busiest : NULL; 10977 10978 out_balanced: 10979 env->imbalance = 0; 10980 return NULL; 10981 } 10982 10983 /* 10984 * find_busiest_queue - find the busiest runqueue among the CPUs in the group. 10985 */ 10986 static struct rq *find_busiest_queue(struct lb_env *env, 10987 struct sched_group *group) 10988 { 10989 struct rq *busiest = NULL, *rq; 10990 unsigned long busiest_util = 0, busiest_load = 0, busiest_capacity = 1; 10991 unsigned int busiest_nr = 0; 10992 int i; 10993 10994 for_each_cpu_and(i, sched_group_span(group), env->cpus) { 10995 unsigned long capacity, load, util; 10996 unsigned int nr_running; 10997 enum fbq_type rt; 10998 10999 rq = cpu_rq(i); 11000 rt = fbq_classify_rq(rq); 11001 11002 /* 11003 * We classify groups/runqueues into three groups: 11004 * - regular: there are !numa tasks 11005 * - remote: there are numa tasks that run on the 'wrong' node 11006 * - all: there is no distinction 11007 * 11008 * In order to avoid migrating ideally placed numa tasks, 11009 * ignore those when there's better options. 11010 * 11011 * If we ignore the actual busiest queue to migrate another 11012 * task, the next balance pass can still reduce the busiest 11013 * queue by moving tasks around inside the node. 11014 * 11015 * If we cannot move enough load due to this classification 11016 * the next pass will adjust the group classification and 11017 * allow migration of more tasks. 11018 * 11019 * Both cases only affect the total convergence complexity. 11020 */ 11021 if (rt > env->fbq_type) 11022 continue; 11023 11024 nr_running = rq->cfs.h_nr_running; 11025 if (!nr_running) 11026 continue; 11027 11028 capacity = capacity_of(i); 11029 11030 /* 11031 * For ASYM_CPUCAPACITY domains, don't pick a CPU that could 11032 * eventually lead to active_balancing high->low capacity. 11033 * Higher per-CPU capacity is considered better than balancing 11034 * average load. 11035 */ 11036 if (env->sd->flags & SD_ASYM_CPUCAPACITY && 11037 !capacity_greater(capacity_of(env->dst_cpu), capacity) && 11038 nr_running == 1) 11039 continue; 11040 11041 /* 11042 * Make sure we only pull tasks from a CPU of lower priority 11043 * when balancing between SMT siblings. 11044 * 11045 * If balancing between cores, let lower priority CPUs help 11046 * SMT cores with more than one busy sibling. 11047 */ 11048 if ((env->sd->flags & SD_ASYM_PACKING) && 11049 sched_use_asym_prio(env->sd, i) && 11050 sched_asym_prefer(i, env->dst_cpu) && 11051 nr_running == 1) 11052 continue; 11053 11054 switch (env->migration_type) { 11055 case migrate_load: 11056 /* 11057 * When comparing with load imbalance, use cpu_load() 11058 * which is not scaled with the CPU capacity. 11059 */ 11060 load = cpu_load(rq); 11061 11062 if (nr_running == 1 && load > env->imbalance && 11063 !check_cpu_capacity(rq, env->sd)) 11064 break; 11065 11066 /* 11067 * For the load comparisons with the other CPUs, 11068 * consider the cpu_load() scaled with the CPU 11069 * capacity, so that the load can be moved away 11070 * from the CPU that is potentially running at a 11071 * lower capacity. 11072 * 11073 * Thus we're looking for max(load_i / capacity_i), 11074 * crosswise multiplication to rid ourselves of the 11075 * division works out to: 11076 * load_i * capacity_j > load_j * capacity_i; 11077 * where j is our previous maximum. 11078 */ 11079 if (load * busiest_capacity > busiest_load * capacity) { 11080 busiest_load = load; 11081 busiest_capacity = capacity; 11082 busiest = rq; 11083 } 11084 break; 11085 11086 case migrate_util: 11087 util = cpu_util_cfs_boost(i); 11088 11089 /* 11090 * Don't try to pull utilization from a CPU with one 11091 * running task. Whatever its utilization, we will fail 11092 * detach the task. 11093 */ 11094 if (nr_running <= 1) 11095 continue; 11096 11097 if (busiest_util < util) { 11098 busiest_util = util; 11099 busiest = rq; 11100 } 11101 break; 11102 11103 case migrate_task: 11104 if (busiest_nr < nr_running) { 11105 busiest_nr = nr_running; 11106 busiest = rq; 11107 } 11108 break; 11109 11110 case migrate_misfit: 11111 /* 11112 * For ASYM_CPUCAPACITY domains with misfit tasks we 11113 * simply seek the "biggest" misfit task. 11114 */ 11115 if (rq->misfit_task_load > busiest_load) { 11116 busiest_load = rq->misfit_task_load; 11117 busiest = rq; 11118 } 11119 11120 break; 11121 11122 } 11123 } 11124 11125 return busiest; 11126 } 11127 11128 /* 11129 * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but 11130 * so long as it is large enough. 11131 */ 11132 #define MAX_PINNED_INTERVAL 512 11133 11134 static inline bool 11135 asym_active_balance(struct lb_env *env) 11136 { 11137 /* 11138 * ASYM_PACKING needs to force migrate tasks from busy but lower 11139 * priority CPUs in order to pack all tasks in the highest priority 11140 * CPUs. When done between cores, do it only if the whole core if the 11141 * whole core is idle. 11142 * 11143 * If @env::src_cpu is an SMT core with busy siblings, let 11144 * the lower priority @env::dst_cpu help it. Do not follow 11145 * CPU priority. 11146 */ 11147 return env->idle != CPU_NOT_IDLE && (env->sd->flags & SD_ASYM_PACKING) && 11148 sched_use_asym_prio(env->sd, env->dst_cpu) && 11149 (sched_asym_prefer(env->dst_cpu, env->src_cpu) || 11150 !sched_use_asym_prio(env->sd, env->src_cpu)); 11151 } 11152 11153 static inline bool 11154 imbalanced_active_balance(struct lb_env *env) 11155 { 11156 struct sched_domain *sd = env->sd; 11157 11158 /* 11159 * The imbalanced case includes the case of pinned tasks preventing a fair 11160 * distribution of the load on the system but also the even distribution of the 11161 * threads on a system with spare capacity 11162 */ 11163 if ((env->migration_type == migrate_task) && 11164 (sd->nr_balance_failed > sd->cache_nice_tries+2)) 11165 return 1; 11166 11167 return 0; 11168 } 11169 11170 static int need_active_balance(struct lb_env *env) 11171 { 11172 struct sched_domain *sd = env->sd; 11173 11174 if (asym_active_balance(env)) 11175 return 1; 11176 11177 if (imbalanced_active_balance(env)) 11178 return 1; 11179 11180 /* 11181 * The dst_cpu is idle and the src_cpu CPU has only 1 CFS task. 11182 * It's worth migrating the task if the src_cpu's capacity is reduced 11183 * because of other sched_class or IRQs if more capacity stays 11184 * available on dst_cpu. 11185 */ 11186 if ((env->idle != CPU_NOT_IDLE) && 11187 (env->src_rq->cfs.h_nr_running == 1)) { 11188 if ((check_cpu_capacity(env->src_rq, sd)) && 11189 (capacity_of(env->src_cpu)*sd->imbalance_pct < capacity_of(env->dst_cpu)*100)) 11190 return 1; 11191 } 11192 11193 if (env->migration_type == migrate_misfit) 11194 return 1; 11195 11196 return 0; 11197 } 11198 11199 static int active_load_balance_cpu_stop(void *data); 11200 11201 static int should_we_balance(struct lb_env *env) 11202 { 11203 struct cpumask *swb_cpus = this_cpu_cpumask_var_ptr(should_we_balance_tmpmask); 11204 struct sched_group *sg = env->sd->groups; 11205 int cpu, idle_smt = -1; 11206 11207 /* 11208 * Ensure the balancing environment is consistent; can happen 11209 * when the softirq triggers 'during' hotplug. 11210 */ 11211 if (!cpumask_test_cpu(env->dst_cpu, env->cpus)) 11212 return 0; 11213 11214 /* 11215 * In the newly idle case, we will allow all the CPUs 11216 * to do the newly idle load balance. 11217 * 11218 * However, we bail out if we already have tasks or a wakeup pending, 11219 * to optimize wakeup latency. 11220 */ 11221 if (env->idle == CPU_NEWLY_IDLE) { 11222 if (env->dst_rq->nr_running > 0 || env->dst_rq->ttwu_pending) 11223 return 0; 11224 return 1; 11225 } 11226 11227 cpumask_copy(swb_cpus, group_balance_mask(sg)); 11228 /* Try to find first idle CPU */ 11229 for_each_cpu_and(cpu, swb_cpus, env->cpus) { 11230 if (!idle_cpu(cpu)) 11231 continue; 11232 11233 /* 11234 * Don't balance to idle SMT in busy core right away when 11235 * balancing cores, but remember the first idle SMT CPU for 11236 * later consideration. Find CPU on an idle core first. 11237 */ 11238 if (!(env->sd->flags & SD_SHARE_CPUCAPACITY) && !is_core_idle(cpu)) { 11239 if (idle_smt == -1) 11240 idle_smt = cpu; 11241 /* 11242 * If the core is not idle, and first SMT sibling which is 11243 * idle has been found, then its not needed to check other 11244 * SMT siblings for idleness: 11245 */ 11246 #ifdef CONFIG_SCHED_SMT 11247 cpumask_andnot(swb_cpus, swb_cpus, cpu_smt_mask(cpu)); 11248 #endif 11249 continue; 11250 } 11251 11252 /* 11253 * Are we the first idle core in a non-SMT domain or higher, 11254 * or the first idle CPU in a SMT domain? 11255 */ 11256 return cpu == env->dst_cpu; 11257 } 11258 11259 /* Are we the first idle CPU with busy siblings? */ 11260 if (idle_smt != -1) 11261 return idle_smt == env->dst_cpu; 11262 11263 /* Are we the first CPU of this group ? */ 11264 return group_balance_cpu(sg) == env->dst_cpu; 11265 } 11266 11267 /* 11268 * Check this_cpu to ensure it is balanced within domain. Attempt to move 11269 * tasks if there is an imbalance. 11270 */ 11271 static int load_balance(int this_cpu, struct rq *this_rq, 11272 struct sched_domain *sd, enum cpu_idle_type idle, 11273 int *continue_balancing) 11274 { 11275 int ld_moved, cur_ld_moved, active_balance = 0; 11276 struct sched_domain *sd_parent = sd->parent; 11277 struct sched_group *group; 11278 struct rq *busiest; 11279 struct rq_flags rf; 11280 struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask); 11281 struct lb_env env = { 11282 .sd = sd, 11283 .dst_cpu = this_cpu, 11284 .dst_rq = this_rq, 11285 .dst_grpmask = group_balance_mask(sd->groups), 11286 .idle = idle, 11287 .loop_break = SCHED_NR_MIGRATE_BREAK, 11288 .cpus = cpus, 11289 .fbq_type = all, 11290 .tasks = LIST_HEAD_INIT(env.tasks), 11291 }; 11292 11293 cpumask_and(cpus, sched_domain_span(sd), cpu_active_mask); 11294 11295 schedstat_inc(sd->lb_count[idle]); 11296 11297 redo: 11298 if (!should_we_balance(&env)) { 11299 *continue_balancing = 0; 11300 goto out_balanced; 11301 } 11302 11303 group = find_busiest_group(&env); 11304 if (!group) { 11305 schedstat_inc(sd->lb_nobusyg[idle]); 11306 goto out_balanced; 11307 } 11308 11309 busiest = find_busiest_queue(&env, group); 11310 if (!busiest) { 11311 schedstat_inc(sd->lb_nobusyq[idle]); 11312 goto out_balanced; 11313 } 11314 11315 WARN_ON_ONCE(busiest == env.dst_rq); 11316 11317 schedstat_add(sd->lb_imbalance[idle], env.imbalance); 11318 11319 env.src_cpu = busiest->cpu; 11320 env.src_rq = busiest; 11321 11322 ld_moved = 0; 11323 /* Clear this flag as soon as we find a pullable task */ 11324 env.flags |= LBF_ALL_PINNED; 11325 if (busiest->nr_running > 1) { 11326 /* 11327 * Attempt to move tasks. If find_busiest_group has found 11328 * an imbalance but busiest->nr_running <= 1, the group is 11329 * still unbalanced. ld_moved simply stays zero, so it is 11330 * correctly treated as an imbalance. 11331 */ 11332 env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running); 11333 11334 more_balance: 11335 rq_lock_irqsave(busiest, &rf); 11336 update_rq_clock(busiest); 11337 11338 /* 11339 * cur_ld_moved - load moved in current iteration 11340 * ld_moved - cumulative load moved across iterations 11341 */ 11342 cur_ld_moved = detach_tasks(&env); 11343 11344 /* 11345 * We've detached some tasks from busiest_rq. Every 11346 * task is masked "TASK_ON_RQ_MIGRATING", so we can safely 11347 * unlock busiest->lock, and we are able to be sure 11348 * that nobody can manipulate the tasks in parallel. 11349 * See task_rq_lock() family for the details. 11350 */ 11351 11352 rq_unlock(busiest, &rf); 11353 11354 if (cur_ld_moved) { 11355 attach_tasks(&env); 11356 ld_moved += cur_ld_moved; 11357 } 11358 11359 local_irq_restore(rf.flags); 11360 11361 if (env.flags & LBF_NEED_BREAK) { 11362 env.flags &= ~LBF_NEED_BREAK; 11363 goto more_balance; 11364 } 11365 11366 /* 11367 * Revisit (affine) tasks on src_cpu that couldn't be moved to 11368 * us and move them to an alternate dst_cpu in our sched_group 11369 * where they can run. The upper limit on how many times we 11370 * iterate on same src_cpu is dependent on number of CPUs in our 11371 * sched_group. 11372 * 11373 * This changes load balance semantics a bit on who can move 11374 * load to a given_cpu. In addition to the given_cpu itself 11375 * (or a ilb_cpu acting on its behalf where given_cpu is 11376 * nohz-idle), we now have balance_cpu in a position to move 11377 * load to given_cpu. In rare situations, this may cause 11378 * conflicts (balance_cpu and given_cpu/ilb_cpu deciding 11379 * _independently_ and at _same_ time to move some load to 11380 * given_cpu) causing excess load to be moved to given_cpu. 11381 * This however should not happen so much in practice and 11382 * moreover subsequent load balance cycles should correct the 11383 * excess load moved. 11384 */ 11385 if ((env.flags & LBF_DST_PINNED) && env.imbalance > 0) { 11386 11387 /* Prevent to re-select dst_cpu via env's CPUs */ 11388 __cpumask_clear_cpu(env.dst_cpu, env.cpus); 11389 11390 env.dst_rq = cpu_rq(env.new_dst_cpu); 11391 env.dst_cpu = env.new_dst_cpu; 11392 env.flags &= ~LBF_DST_PINNED; 11393 env.loop = 0; 11394 env.loop_break = SCHED_NR_MIGRATE_BREAK; 11395 11396 /* 11397 * Go back to "more_balance" rather than "redo" since we 11398 * need to continue with same src_cpu. 11399 */ 11400 goto more_balance; 11401 } 11402 11403 /* 11404 * We failed to reach balance because of affinity. 11405 */ 11406 if (sd_parent) { 11407 int *group_imbalance = &sd_parent->groups->sgc->imbalance; 11408 11409 if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0) 11410 *group_imbalance = 1; 11411 } 11412 11413 /* All tasks on this runqueue were pinned by CPU affinity */ 11414 if (unlikely(env.flags & LBF_ALL_PINNED)) { 11415 __cpumask_clear_cpu(cpu_of(busiest), cpus); 11416 /* 11417 * Attempting to continue load balancing at the current 11418 * sched_domain level only makes sense if there are 11419 * active CPUs remaining as possible busiest CPUs to 11420 * pull load from which are not contained within the 11421 * destination group that is receiving any migrated 11422 * load. 11423 */ 11424 if (!cpumask_subset(cpus, env.dst_grpmask)) { 11425 env.loop = 0; 11426 env.loop_break = SCHED_NR_MIGRATE_BREAK; 11427 goto redo; 11428 } 11429 goto out_all_pinned; 11430 } 11431 } 11432 11433 if (!ld_moved) { 11434 schedstat_inc(sd->lb_failed[idle]); 11435 /* 11436 * Increment the failure counter only on periodic balance. 11437 * We do not want newidle balance, which can be very 11438 * frequent, pollute the failure counter causing 11439 * excessive cache_hot migrations and active balances. 11440 */ 11441 if (idle != CPU_NEWLY_IDLE) 11442 sd->nr_balance_failed++; 11443 11444 if (need_active_balance(&env)) { 11445 unsigned long flags; 11446 11447 raw_spin_rq_lock_irqsave(busiest, flags); 11448 11449 /* 11450 * Don't kick the active_load_balance_cpu_stop, 11451 * if the curr task on busiest CPU can't be 11452 * moved to this_cpu: 11453 */ 11454 if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr)) { 11455 raw_spin_rq_unlock_irqrestore(busiest, flags); 11456 goto out_one_pinned; 11457 } 11458 11459 /* Record that we found at least one task that could run on this_cpu */ 11460 env.flags &= ~LBF_ALL_PINNED; 11461 11462 /* 11463 * ->active_balance synchronizes accesses to 11464 * ->active_balance_work. Once set, it's cleared 11465 * only after active load balance is finished. 11466 */ 11467 if (!busiest->active_balance) { 11468 busiest->active_balance = 1; 11469 busiest->push_cpu = this_cpu; 11470 active_balance = 1; 11471 } 11472 11473 preempt_disable(); 11474 raw_spin_rq_unlock_irqrestore(busiest, flags); 11475 if (active_balance) { 11476 stop_one_cpu_nowait(cpu_of(busiest), 11477 active_load_balance_cpu_stop, busiest, 11478 &busiest->active_balance_work); 11479 } 11480 preempt_enable(); 11481 } 11482 } else { 11483 sd->nr_balance_failed = 0; 11484 } 11485 11486 if (likely(!active_balance) || need_active_balance(&env)) { 11487 /* We were unbalanced, so reset the balancing interval */ 11488 sd->balance_interval = sd->min_interval; 11489 } 11490 11491 goto out; 11492 11493 out_balanced: 11494 /* 11495 * We reach balance although we may have faced some affinity 11496 * constraints. Clear the imbalance flag only if other tasks got 11497 * a chance to move and fix the imbalance. 11498 */ 11499 if (sd_parent && !(env.flags & LBF_ALL_PINNED)) { 11500 int *group_imbalance = &sd_parent->groups->sgc->imbalance; 11501 11502 if (*group_imbalance) 11503 *group_imbalance = 0; 11504 } 11505 11506 out_all_pinned: 11507 /* 11508 * We reach balance because all tasks are pinned at this level so 11509 * we can't migrate them. Let the imbalance flag set so parent level 11510 * can try to migrate them. 11511 */ 11512 schedstat_inc(sd->lb_balanced[idle]); 11513 11514 sd->nr_balance_failed = 0; 11515 11516 out_one_pinned: 11517 ld_moved = 0; 11518 11519 /* 11520 * newidle_balance() disregards balance intervals, so we could 11521 * repeatedly reach this code, which would lead to balance_interval 11522 * skyrocketing in a short amount of time. Skip the balance_interval 11523 * increase logic to avoid that. 11524 */ 11525 if (env.idle == CPU_NEWLY_IDLE) 11526 goto out; 11527 11528 /* tune up the balancing interval */ 11529 if ((env.flags & LBF_ALL_PINNED && 11530 sd->balance_interval < MAX_PINNED_INTERVAL) || 11531 sd->balance_interval < sd->max_interval) 11532 sd->balance_interval *= 2; 11533 out: 11534 return ld_moved; 11535 } 11536 11537 static inline unsigned long 11538 get_sd_balance_interval(struct sched_domain *sd, int cpu_busy) 11539 { 11540 unsigned long interval = sd->balance_interval; 11541 11542 if (cpu_busy) 11543 interval *= sd->busy_factor; 11544 11545 /* scale ms to jiffies */ 11546 interval = msecs_to_jiffies(interval); 11547 11548 /* 11549 * Reduce likelihood of busy balancing at higher domains racing with 11550 * balancing at lower domains by preventing their balancing periods 11551 * from being multiples of each other. 11552 */ 11553 if (cpu_busy) 11554 interval -= 1; 11555 11556 interval = clamp(interval, 1UL, max_load_balance_interval); 11557 11558 return interval; 11559 } 11560 11561 static inline void 11562 update_next_balance(struct sched_domain *sd, unsigned long *next_balance) 11563 { 11564 unsigned long interval, next; 11565 11566 /* used by idle balance, so cpu_busy = 0 */ 11567 interval = get_sd_balance_interval(sd, 0); 11568 next = sd->last_balance + interval; 11569 11570 if (time_after(*next_balance, next)) 11571 *next_balance = next; 11572 } 11573 11574 /* 11575 * active_load_balance_cpu_stop is run by the CPU stopper. It pushes 11576 * running tasks off the busiest CPU onto idle CPUs. It requires at 11577 * least 1 task to be running on each physical CPU where possible, and 11578 * avoids physical / logical imbalances. 11579 */ 11580 static int active_load_balance_cpu_stop(void *data) 11581 { 11582 struct rq *busiest_rq = data; 11583 int busiest_cpu = cpu_of(busiest_rq); 11584 int target_cpu = busiest_rq->push_cpu; 11585 struct rq *target_rq = cpu_rq(target_cpu); 11586 struct sched_domain *sd; 11587 struct task_struct *p = NULL; 11588 struct rq_flags rf; 11589 11590 rq_lock_irq(busiest_rq, &rf); 11591 /* 11592 * Between queueing the stop-work and running it is a hole in which 11593 * CPUs can become inactive. We should not move tasks from or to 11594 * inactive CPUs. 11595 */ 11596 if (!cpu_active(busiest_cpu) || !cpu_active(target_cpu)) 11597 goto out_unlock; 11598 11599 /* Make sure the requested CPU hasn't gone down in the meantime: */ 11600 if (unlikely(busiest_cpu != smp_processor_id() || 11601 !busiest_rq->active_balance)) 11602 goto out_unlock; 11603 11604 /* Is there any task to move? */ 11605 if (busiest_rq->nr_running <= 1) 11606 goto out_unlock; 11607 11608 /* 11609 * This condition is "impossible", if it occurs 11610 * we need to fix it. Originally reported by 11611 * Bjorn Helgaas on a 128-CPU setup. 11612 */ 11613 WARN_ON_ONCE(busiest_rq == target_rq); 11614 11615 /* Search for an sd spanning us and the target CPU. */ 11616 rcu_read_lock(); 11617 for_each_domain(target_cpu, sd) { 11618 if (cpumask_test_cpu(busiest_cpu, sched_domain_span(sd))) 11619 break; 11620 } 11621 11622 if (likely(sd)) { 11623 struct lb_env env = { 11624 .sd = sd, 11625 .dst_cpu = target_cpu, 11626 .dst_rq = target_rq, 11627 .src_cpu = busiest_rq->cpu, 11628 .src_rq = busiest_rq, 11629 .idle = CPU_IDLE, 11630 .flags = LBF_ACTIVE_LB, 11631 }; 11632 11633 schedstat_inc(sd->alb_count); 11634 update_rq_clock(busiest_rq); 11635 11636 p = detach_one_task(&env); 11637 if (p) { 11638 schedstat_inc(sd->alb_pushed); 11639 /* Active balancing done, reset the failure counter. */ 11640 sd->nr_balance_failed = 0; 11641 } else { 11642 schedstat_inc(sd->alb_failed); 11643 } 11644 } 11645 rcu_read_unlock(); 11646 out_unlock: 11647 busiest_rq->active_balance = 0; 11648 rq_unlock(busiest_rq, &rf); 11649 11650 if (p) 11651 attach_one_task(target_rq, p); 11652 11653 local_irq_enable(); 11654 11655 return 0; 11656 } 11657 11658 static DEFINE_SPINLOCK(balancing); 11659 11660 /* 11661 * Scale the max load_balance interval with the number of CPUs in the system. 11662 * This trades load-balance latency on larger machines for less cross talk. 11663 */ 11664 void update_max_interval(void) 11665 { 11666 max_load_balance_interval = HZ*num_online_cpus()/10; 11667 } 11668 11669 static inline bool update_newidle_cost(struct sched_domain *sd, u64 cost) 11670 { 11671 if (cost > sd->max_newidle_lb_cost) { 11672 /* 11673 * Track max cost of a domain to make sure to not delay the 11674 * next wakeup on the CPU. 11675 */ 11676 sd->max_newidle_lb_cost = cost; 11677 sd->last_decay_max_lb_cost = jiffies; 11678 } else if (time_after(jiffies, sd->last_decay_max_lb_cost + HZ)) { 11679 /* 11680 * Decay the newidle max times by ~1% per second to ensure that 11681 * it is not outdated and the current max cost is actually 11682 * shorter. 11683 */ 11684 sd->max_newidle_lb_cost = (sd->max_newidle_lb_cost * 253) / 256; 11685 sd->last_decay_max_lb_cost = jiffies; 11686 11687 return true; 11688 } 11689 11690 return false; 11691 } 11692 11693 /* 11694 * It checks each scheduling domain to see if it is due to be balanced, 11695 * and initiates a balancing operation if so. 11696 * 11697 * Balancing parameters are set up in init_sched_domains. 11698 */ 11699 static void rebalance_domains(struct rq *rq, enum cpu_idle_type idle) 11700 { 11701 int continue_balancing = 1; 11702 int cpu = rq->cpu; 11703 int busy = idle != CPU_IDLE && !sched_idle_cpu(cpu); 11704 unsigned long interval; 11705 struct sched_domain *sd; 11706 /* Earliest time when we have to do rebalance again */ 11707 unsigned long next_balance = jiffies + 60*HZ; 11708 int update_next_balance = 0; 11709 int need_serialize, need_decay = 0; 11710 u64 max_cost = 0; 11711 11712 rcu_read_lock(); 11713 for_each_domain(cpu, sd) { 11714 /* 11715 * Decay the newidle max times here because this is a regular 11716 * visit to all the domains. 11717 */ 11718 need_decay = update_newidle_cost(sd, 0); 11719 max_cost += sd->max_newidle_lb_cost; 11720 11721 /* 11722 * Stop the load balance at this level. There is another 11723 * CPU in our sched group which is doing load balancing more 11724 * actively. 11725 */ 11726 if (!continue_balancing) { 11727 if (need_decay) 11728 continue; 11729 break; 11730 } 11731 11732 interval = get_sd_balance_interval(sd, busy); 11733 11734 need_serialize = sd->flags & SD_SERIALIZE; 11735 if (need_serialize) { 11736 if (!spin_trylock(&balancing)) 11737 goto out; 11738 } 11739 11740 if (time_after_eq(jiffies, sd->last_balance + interval)) { 11741 if (load_balance(cpu, rq, sd, idle, &continue_balancing)) { 11742 /* 11743 * The LBF_DST_PINNED logic could have changed 11744 * env->dst_cpu, so we can't know our idle 11745 * state even if we migrated tasks. Update it. 11746 */ 11747 idle = idle_cpu(cpu) ? CPU_IDLE : CPU_NOT_IDLE; 11748 busy = idle != CPU_IDLE && !sched_idle_cpu(cpu); 11749 } 11750 sd->last_balance = jiffies; 11751 interval = get_sd_balance_interval(sd, busy); 11752 } 11753 if (need_serialize) 11754 spin_unlock(&balancing); 11755 out: 11756 if (time_after(next_balance, sd->last_balance + interval)) { 11757 next_balance = sd->last_balance + interval; 11758 update_next_balance = 1; 11759 } 11760 } 11761 if (need_decay) { 11762 /* 11763 * Ensure the rq-wide value also decays but keep it at a 11764 * reasonable floor to avoid funnies with rq->avg_idle. 11765 */ 11766 rq->max_idle_balance_cost = 11767 max((u64)sysctl_sched_migration_cost, max_cost); 11768 } 11769 rcu_read_unlock(); 11770 11771 /* 11772 * next_balance will be updated only when there is a need. 11773 * When the cpu is attached to null domain for ex, it will not be 11774 * updated. 11775 */ 11776 if (likely(update_next_balance)) 11777 rq->next_balance = next_balance; 11778 11779 } 11780 11781 static inline int on_null_domain(struct rq *rq) 11782 { 11783 return unlikely(!rcu_dereference_sched(rq->sd)); 11784 } 11785 11786 #ifdef CONFIG_NO_HZ_COMMON 11787 /* 11788 * idle load balancing details 11789 * - When one of the busy CPUs notice that there may be an idle rebalancing 11790 * needed, they will kick the idle load balancer, which then does idle 11791 * load balancing for all the idle CPUs. 11792 * - HK_TYPE_MISC CPUs are used for this task, because HK_TYPE_SCHED not set 11793 * anywhere yet. 11794 */ 11795 11796 static inline int find_new_ilb(void) 11797 { 11798 int ilb; 11799 const struct cpumask *hk_mask; 11800 11801 hk_mask = housekeeping_cpumask(HK_TYPE_MISC); 11802 11803 for_each_cpu_and(ilb, nohz.idle_cpus_mask, hk_mask) { 11804 11805 if (ilb == smp_processor_id()) 11806 continue; 11807 11808 if (idle_cpu(ilb)) 11809 return ilb; 11810 } 11811 11812 return nr_cpu_ids; 11813 } 11814 11815 /* 11816 * Kick a CPU to do the nohz balancing, if it is time for it. We pick any 11817 * idle CPU in the HK_TYPE_MISC housekeeping set (if there is one). 11818 */ 11819 static void kick_ilb(unsigned int flags) 11820 { 11821 int ilb_cpu; 11822 11823 /* 11824 * Increase nohz.next_balance only when if full ilb is triggered but 11825 * not if we only update stats. 11826 */ 11827 if (flags & NOHZ_BALANCE_KICK) 11828 nohz.next_balance = jiffies+1; 11829 11830 ilb_cpu = find_new_ilb(); 11831 11832 if (ilb_cpu >= nr_cpu_ids) 11833 return; 11834 11835 /* 11836 * Access to rq::nohz_csd is serialized by NOHZ_KICK_MASK; he who sets 11837 * the first flag owns it; cleared by nohz_csd_func(). 11838 */ 11839 flags = atomic_fetch_or(flags, nohz_flags(ilb_cpu)); 11840 if (flags & NOHZ_KICK_MASK) 11841 return; 11842 11843 /* 11844 * This way we generate an IPI on the target CPU which 11845 * is idle. And the softirq performing nohz idle load balance 11846 * will be run before returning from the IPI. 11847 */ 11848 smp_call_function_single_async(ilb_cpu, &cpu_rq(ilb_cpu)->nohz_csd); 11849 } 11850 11851 /* 11852 * Current decision point for kicking the idle load balancer in the presence 11853 * of idle CPUs in the system. 11854 */ 11855 static void nohz_balancer_kick(struct rq *rq) 11856 { 11857 unsigned long now = jiffies; 11858 struct sched_domain_shared *sds; 11859 struct sched_domain *sd; 11860 int nr_busy, i, cpu = rq->cpu; 11861 unsigned int flags = 0; 11862 11863 if (unlikely(rq->idle_balance)) 11864 return; 11865 11866 /* 11867 * We may be recently in ticked or tickless idle mode. At the first 11868 * busy tick after returning from idle, we will update the busy stats. 11869 */ 11870 nohz_balance_exit_idle(rq); 11871 11872 /* 11873 * None are in tickless mode and hence no need for NOHZ idle load 11874 * balancing. 11875 */ 11876 if (likely(!atomic_read(&nohz.nr_cpus))) 11877 return; 11878 11879 if (READ_ONCE(nohz.has_blocked) && 11880 time_after(now, READ_ONCE(nohz.next_blocked))) 11881 flags = NOHZ_STATS_KICK; 11882 11883 if (time_before(now, nohz.next_balance)) 11884 goto out; 11885 11886 if (rq->nr_running >= 2) { 11887 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 11888 goto out; 11889 } 11890 11891 rcu_read_lock(); 11892 11893 sd = rcu_dereference(rq->sd); 11894 if (sd) { 11895 /* 11896 * If there's a CFS task and the current CPU has reduced 11897 * capacity; kick the ILB to see if there's a better CPU to run 11898 * on. 11899 */ 11900 if (rq->cfs.h_nr_running >= 1 && check_cpu_capacity(rq, sd)) { 11901 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 11902 goto unlock; 11903 } 11904 } 11905 11906 sd = rcu_dereference(per_cpu(sd_asym_packing, cpu)); 11907 if (sd) { 11908 /* 11909 * When ASYM_PACKING; see if there's a more preferred CPU 11910 * currently idle; in which case, kick the ILB to move tasks 11911 * around. 11912 * 11913 * When balancing betwen cores, all the SMT siblings of the 11914 * preferred CPU must be idle. 11915 */ 11916 for_each_cpu_and(i, sched_domain_span(sd), nohz.idle_cpus_mask) { 11917 if (sched_use_asym_prio(sd, i) && 11918 sched_asym_prefer(i, cpu)) { 11919 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 11920 goto unlock; 11921 } 11922 } 11923 } 11924 11925 sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, cpu)); 11926 if (sd) { 11927 /* 11928 * When ASYM_CPUCAPACITY; see if there's a higher capacity CPU 11929 * to run the misfit task on. 11930 */ 11931 if (check_misfit_status(rq, sd)) { 11932 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 11933 goto unlock; 11934 } 11935 11936 /* 11937 * For asymmetric systems, we do not want to nicely balance 11938 * cache use, instead we want to embrace asymmetry and only 11939 * ensure tasks have enough CPU capacity. 11940 * 11941 * Skip the LLC logic because it's not relevant in that case. 11942 */ 11943 goto unlock; 11944 } 11945 11946 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); 11947 if (sds) { 11948 /* 11949 * If there is an imbalance between LLC domains (IOW we could 11950 * increase the overall cache use), we need some less-loaded LLC 11951 * domain to pull some load. Likewise, we may need to spread 11952 * load within the current LLC domain (e.g. packed SMT cores but 11953 * other CPUs are idle). We can't really know from here how busy 11954 * the others are - so just get a nohz balance going if it looks 11955 * like this LLC domain has tasks we could move. 11956 */ 11957 nr_busy = atomic_read(&sds->nr_busy_cpus); 11958 if (nr_busy > 1) { 11959 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 11960 goto unlock; 11961 } 11962 } 11963 unlock: 11964 rcu_read_unlock(); 11965 out: 11966 if (READ_ONCE(nohz.needs_update)) 11967 flags |= NOHZ_NEXT_KICK; 11968 11969 if (flags) 11970 kick_ilb(flags); 11971 } 11972 11973 static void set_cpu_sd_state_busy(int cpu) 11974 { 11975 struct sched_domain *sd; 11976 11977 rcu_read_lock(); 11978 sd = rcu_dereference(per_cpu(sd_llc, cpu)); 11979 11980 if (!sd || !sd->nohz_idle) 11981 goto unlock; 11982 sd->nohz_idle = 0; 11983 11984 atomic_inc(&sd->shared->nr_busy_cpus); 11985 unlock: 11986 rcu_read_unlock(); 11987 } 11988 11989 void nohz_balance_exit_idle(struct rq *rq) 11990 { 11991 SCHED_WARN_ON(rq != this_rq()); 11992 11993 if (likely(!rq->nohz_tick_stopped)) 11994 return; 11995 11996 rq->nohz_tick_stopped = 0; 11997 cpumask_clear_cpu(rq->cpu, nohz.idle_cpus_mask); 11998 atomic_dec(&nohz.nr_cpus); 11999 12000 set_cpu_sd_state_busy(rq->cpu); 12001 } 12002 12003 static void set_cpu_sd_state_idle(int cpu) 12004 { 12005 struct sched_domain *sd; 12006 12007 rcu_read_lock(); 12008 sd = rcu_dereference(per_cpu(sd_llc, cpu)); 12009 12010 if (!sd || sd->nohz_idle) 12011 goto unlock; 12012 sd->nohz_idle = 1; 12013 12014 atomic_dec(&sd->shared->nr_busy_cpus); 12015 unlock: 12016 rcu_read_unlock(); 12017 } 12018 12019 /* 12020 * This routine will record that the CPU is going idle with tick stopped. 12021 * This info will be used in performing idle load balancing in the future. 12022 */ 12023 void nohz_balance_enter_idle(int cpu) 12024 { 12025 struct rq *rq = cpu_rq(cpu); 12026 12027 SCHED_WARN_ON(cpu != smp_processor_id()); 12028 12029 /* If this CPU is going down, then nothing needs to be done: */ 12030 if (!cpu_active(cpu)) 12031 return; 12032 12033 /* Spare idle load balancing on CPUs that don't want to be disturbed: */ 12034 if (!housekeeping_cpu(cpu, HK_TYPE_SCHED)) 12035 return; 12036 12037 /* 12038 * Can be set safely without rq->lock held 12039 * If a clear happens, it will have evaluated last additions because 12040 * rq->lock is held during the check and the clear 12041 */ 12042 rq->has_blocked_load = 1; 12043 12044 /* 12045 * The tick is still stopped but load could have been added in the 12046 * meantime. We set the nohz.has_blocked flag to trig a check of the 12047 * *_avg. The CPU is already part of nohz.idle_cpus_mask so the clear 12048 * of nohz.has_blocked can only happen after checking the new load 12049 */ 12050 if (rq->nohz_tick_stopped) 12051 goto out; 12052 12053 /* If we're a completely isolated CPU, we don't play: */ 12054 if (on_null_domain(rq)) 12055 return; 12056 12057 rq->nohz_tick_stopped = 1; 12058 12059 cpumask_set_cpu(cpu, nohz.idle_cpus_mask); 12060 atomic_inc(&nohz.nr_cpus); 12061 12062 /* 12063 * Ensures that if nohz_idle_balance() fails to observe our 12064 * @idle_cpus_mask store, it must observe the @has_blocked 12065 * and @needs_update stores. 12066 */ 12067 smp_mb__after_atomic(); 12068 12069 set_cpu_sd_state_idle(cpu); 12070 12071 WRITE_ONCE(nohz.needs_update, 1); 12072 out: 12073 /* 12074 * Each time a cpu enter idle, we assume that it has blocked load and 12075 * enable the periodic update of the load of idle cpus 12076 */ 12077 WRITE_ONCE(nohz.has_blocked, 1); 12078 } 12079 12080 static bool update_nohz_stats(struct rq *rq) 12081 { 12082 unsigned int cpu = rq->cpu; 12083 12084 if (!rq->has_blocked_load) 12085 return false; 12086 12087 if (!cpumask_test_cpu(cpu, nohz.idle_cpus_mask)) 12088 return false; 12089 12090 if (!time_after(jiffies, READ_ONCE(rq->last_blocked_load_update_tick))) 12091 return true; 12092 12093 update_blocked_averages(cpu); 12094 12095 return rq->has_blocked_load; 12096 } 12097 12098 /* 12099 * Internal function that runs load balance for all idle cpus. The load balance 12100 * can be a simple update of blocked load or a complete load balance with 12101 * tasks movement depending of flags. 12102 */ 12103 static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags) 12104 { 12105 /* Earliest time when we have to do rebalance again */ 12106 unsigned long now = jiffies; 12107 unsigned long next_balance = now + 60*HZ; 12108 bool has_blocked_load = false; 12109 int update_next_balance = 0; 12110 int this_cpu = this_rq->cpu; 12111 int balance_cpu; 12112 struct rq *rq; 12113 12114 SCHED_WARN_ON((flags & NOHZ_KICK_MASK) == NOHZ_BALANCE_KICK); 12115 12116 /* 12117 * We assume there will be no idle load after this update and clear 12118 * the has_blocked flag. If a cpu enters idle in the mean time, it will 12119 * set the has_blocked flag and trigger another update of idle load. 12120 * Because a cpu that becomes idle, is added to idle_cpus_mask before 12121 * setting the flag, we are sure to not clear the state and not 12122 * check the load of an idle cpu. 12123 * 12124 * Same applies to idle_cpus_mask vs needs_update. 12125 */ 12126 if (flags & NOHZ_STATS_KICK) 12127 WRITE_ONCE(nohz.has_blocked, 0); 12128 if (flags & NOHZ_NEXT_KICK) 12129 WRITE_ONCE(nohz.needs_update, 0); 12130 12131 /* 12132 * Ensures that if we miss the CPU, we must see the has_blocked 12133 * store from nohz_balance_enter_idle(). 12134 */ 12135 smp_mb(); 12136 12137 /* 12138 * Start with the next CPU after this_cpu so we will end with this_cpu and let a 12139 * chance for other idle cpu to pull load. 12140 */ 12141 for_each_cpu_wrap(balance_cpu, nohz.idle_cpus_mask, this_cpu+1) { 12142 if (!idle_cpu(balance_cpu)) 12143 continue; 12144 12145 /* 12146 * If this CPU gets work to do, stop the load balancing 12147 * work being done for other CPUs. Next load 12148 * balancing owner will pick it up. 12149 */ 12150 if (!idle_cpu(this_cpu) && need_resched()) { 12151 if (flags & NOHZ_STATS_KICK) 12152 has_blocked_load = true; 12153 if (flags & NOHZ_NEXT_KICK) 12154 WRITE_ONCE(nohz.needs_update, 1); 12155 goto abort; 12156 } 12157 12158 rq = cpu_rq(balance_cpu); 12159 12160 if (flags & NOHZ_STATS_KICK) 12161 has_blocked_load |= update_nohz_stats(rq); 12162 12163 /* 12164 * If time for next balance is due, 12165 * do the balance. 12166 */ 12167 if (time_after_eq(jiffies, rq->next_balance)) { 12168 struct rq_flags rf; 12169 12170 rq_lock_irqsave(rq, &rf); 12171 update_rq_clock(rq); 12172 rq_unlock_irqrestore(rq, &rf); 12173 12174 if (flags & NOHZ_BALANCE_KICK) 12175 rebalance_domains(rq, CPU_IDLE); 12176 } 12177 12178 if (time_after(next_balance, rq->next_balance)) { 12179 next_balance = rq->next_balance; 12180 update_next_balance = 1; 12181 } 12182 } 12183 12184 /* 12185 * next_balance will be updated only when there is a need. 12186 * When the CPU is attached to null domain for ex, it will not be 12187 * updated. 12188 */ 12189 if (likely(update_next_balance)) 12190 nohz.next_balance = next_balance; 12191 12192 if (flags & NOHZ_STATS_KICK) 12193 WRITE_ONCE(nohz.next_blocked, 12194 now + msecs_to_jiffies(LOAD_AVG_PERIOD)); 12195 12196 abort: 12197 /* There is still blocked load, enable periodic update */ 12198 if (has_blocked_load) 12199 WRITE_ONCE(nohz.has_blocked, 1); 12200 } 12201 12202 /* 12203 * In CONFIG_NO_HZ_COMMON case, the idle balance kickee will do the 12204 * rebalancing for all the cpus for whom scheduler ticks are stopped. 12205 */ 12206 static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) 12207 { 12208 unsigned int flags = this_rq->nohz_idle_balance; 12209 12210 if (!flags) 12211 return false; 12212 12213 this_rq->nohz_idle_balance = 0; 12214 12215 if (idle != CPU_IDLE) 12216 return false; 12217 12218 _nohz_idle_balance(this_rq, flags); 12219 12220 return true; 12221 } 12222 12223 /* 12224 * Check if we need to run the ILB for updating blocked load before entering 12225 * idle state. 12226 */ 12227 void nohz_run_idle_balance(int cpu) 12228 { 12229 unsigned int flags; 12230 12231 flags = atomic_fetch_andnot(NOHZ_NEWILB_KICK, nohz_flags(cpu)); 12232 12233 /* 12234 * Update the blocked load only if no SCHED_SOFTIRQ is about to happen 12235 * (ie NOHZ_STATS_KICK set) and will do the same. 12236 */ 12237 if ((flags == NOHZ_NEWILB_KICK) && !need_resched()) 12238 _nohz_idle_balance(cpu_rq(cpu), NOHZ_STATS_KICK); 12239 } 12240 12241 static void nohz_newidle_balance(struct rq *this_rq) 12242 { 12243 int this_cpu = this_rq->cpu; 12244 12245 /* 12246 * This CPU doesn't want to be disturbed by scheduler 12247 * housekeeping 12248 */ 12249 if (!housekeeping_cpu(this_cpu, HK_TYPE_SCHED)) 12250 return; 12251 12252 /* Will wake up very soon. No time for doing anything else*/ 12253 if (this_rq->avg_idle < sysctl_sched_migration_cost) 12254 return; 12255 12256 /* Don't need to update blocked load of idle CPUs*/ 12257 if (!READ_ONCE(nohz.has_blocked) || 12258 time_before(jiffies, READ_ONCE(nohz.next_blocked))) 12259 return; 12260 12261 /* 12262 * Set the need to trigger ILB in order to update blocked load 12263 * before entering idle state. 12264 */ 12265 atomic_or(NOHZ_NEWILB_KICK, nohz_flags(this_cpu)); 12266 } 12267 12268 #else /* !CONFIG_NO_HZ_COMMON */ 12269 static inline void nohz_balancer_kick(struct rq *rq) { } 12270 12271 static inline bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) 12272 { 12273 return false; 12274 } 12275 12276 static inline void nohz_newidle_balance(struct rq *this_rq) { } 12277 #endif /* CONFIG_NO_HZ_COMMON */ 12278 12279 /* 12280 * newidle_balance is called by schedule() if this_cpu is about to become 12281 * idle. Attempts to pull tasks from other CPUs. 12282 * 12283 * Returns: 12284 * < 0 - we released the lock and there are !fair tasks present 12285 * 0 - failed, no new tasks 12286 * > 0 - success, new (fair) tasks present 12287 */ 12288 static int newidle_balance(struct rq *this_rq, struct rq_flags *rf) 12289 { 12290 unsigned long next_balance = jiffies + HZ; 12291 int this_cpu = this_rq->cpu; 12292 u64 t0, t1, curr_cost = 0; 12293 struct sched_domain *sd; 12294 int pulled_task = 0; 12295 12296 update_misfit_status(NULL, this_rq); 12297 12298 /* 12299 * There is a task waiting to run. No need to search for one. 12300 * Return 0; the task will be enqueued when switching to idle. 12301 */ 12302 if (this_rq->ttwu_pending) 12303 return 0; 12304 12305 /* 12306 * We must set idle_stamp _before_ calling idle_balance(), such that we 12307 * measure the duration of idle_balance() as idle time. 12308 */ 12309 this_rq->idle_stamp = rq_clock(this_rq); 12310 12311 /* 12312 * Do not pull tasks towards !active CPUs... 12313 */ 12314 if (!cpu_active(this_cpu)) 12315 return 0; 12316 12317 /* 12318 * This is OK, because current is on_cpu, which avoids it being picked 12319 * for load-balance and preemption/IRQs are still disabled avoiding 12320 * further scheduler activity on it and we're being very careful to 12321 * re-start the picking loop. 12322 */ 12323 rq_unpin_lock(this_rq, rf); 12324 12325 rcu_read_lock(); 12326 sd = rcu_dereference_check_sched_domain(this_rq->sd); 12327 12328 if (!READ_ONCE(this_rq->rd->overload) || 12329 (sd && this_rq->avg_idle < sd->max_newidle_lb_cost)) { 12330 12331 if (sd) 12332 update_next_balance(sd, &next_balance); 12333 rcu_read_unlock(); 12334 12335 goto out; 12336 } 12337 rcu_read_unlock(); 12338 12339 raw_spin_rq_unlock(this_rq); 12340 12341 t0 = sched_clock_cpu(this_cpu); 12342 update_blocked_averages(this_cpu); 12343 12344 rcu_read_lock(); 12345 for_each_domain(this_cpu, sd) { 12346 int continue_balancing = 1; 12347 u64 domain_cost; 12348 12349 update_next_balance(sd, &next_balance); 12350 12351 if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost) 12352 break; 12353 12354 if (sd->flags & SD_BALANCE_NEWIDLE) { 12355 12356 pulled_task = load_balance(this_cpu, this_rq, 12357 sd, CPU_NEWLY_IDLE, 12358 &continue_balancing); 12359 12360 t1 = sched_clock_cpu(this_cpu); 12361 domain_cost = t1 - t0; 12362 update_newidle_cost(sd, domain_cost); 12363 12364 curr_cost += domain_cost; 12365 t0 = t1; 12366 } 12367 12368 /* 12369 * Stop searching for tasks to pull if there are 12370 * now runnable tasks on this rq. 12371 */ 12372 if (pulled_task || this_rq->nr_running > 0 || 12373 this_rq->ttwu_pending) 12374 break; 12375 } 12376 rcu_read_unlock(); 12377 12378 raw_spin_rq_lock(this_rq); 12379 12380 if (curr_cost > this_rq->max_idle_balance_cost) 12381 this_rq->max_idle_balance_cost = curr_cost; 12382 12383 /* 12384 * While browsing the domains, we released the rq lock, a task could 12385 * have been enqueued in the meantime. Since we're not going idle, 12386 * pretend we pulled a task. 12387 */ 12388 if (this_rq->cfs.h_nr_running && !pulled_task) 12389 pulled_task = 1; 12390 12391 /* Is there a task of a high priority class? */ 12392 if (this_rq->nr_running != this_rq->cfs.h_nr_running) 12393 pulled_task = -1; 12394 12395 out: 12396 /* Move the next balance forward */ 12397 if (time_after(this_rq->next_balance, next_balance)) 12398 this_rq->next_balance = next_balance; 12399 12400 if (pulled_task) 12401 this_rq->idle_stamp = 0; 12402 else 12403 nohz_newidle_balance(this_rq); 12404 12405 rq_repin_lock(this_rq, rf); 12406 12407 return pulled_task; 12408 } 12409 12410 /* 12411 * run_rebalance_domains is triggered when needed from the scheduler tick. 12412 * Also triggered for nohz idle balancing (with nohz_balancing_kick set). 12413 */ 12414 static __latent_entropy void run_rebalance_domains(struct softirq_action *h) 12415 { 12416 struct rq *this_rq = this_rq(); 12417 enum cpu_idle_type idle = this_rq->idle_balance ? 12418 CPU_IDLE : CPU_NOT_IDLE; 12419 12420 /* 12421 * If this CPU has a pending nohz_balance_kick, then do the 12422 * balancing on behalf of the other idle CPUs whose ticks are 12423 * stopped. Do nohz_idle_balance *before* rebalance_domains to 12424 * give the idle CPUs a chance to load balance. Else we may 12425 * load balance only within the local sched_domain hierarchy 12426 * and abort nohz_idle_balance altogether if we pull some load. 12427 */ 12428 if (nohz_idle_balance(this_rq, idle)) 12429 return; 12430 12431 /* normal load balance */ 12432 update_blocked_averages(this_rq->cpu); 12433 rebalance_domains(this_rq, idle); 12434 } 12435 12436 /* 12437 * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing. 12438 */ 12439 void trigger_load_balance(struct rq *rq) 12440 { 12441 /* 12442 * Don't need to rebalance while attached to NULL domain or 12443 * runqueue CPU is not active 12444 */ 12445 if (unlikely(on_null_domain(rq) || !cpu_active(cpu_of(rq)))) 12446 return; 12447 12448 if (time_after_eq(jiffies, rq->next_balance)) 12449 raise_softirq(SCHED_SOFTIRQ); 12450 12451 nohz_balancer_kick(rq); 12452 } 12453 12454 static void rq_online_fair(struct rq *rq) 12455 { 12456 update_sysctl(); 12457 12458 update_runtime_enabled(rq); 12459 } 12460 12461 static void rq_offline_fair(struct rq *rq) 12462 { 12463 update_sysctl(); 12464 12465 /* Ensure any throttled groups are reachable by pick_next_task */ 12466 unthrottle_offline_cfs_rqs(rq); 12467 } 12468 12469 #endif /* CONFIG_SMP */ 12470 12471 #ifdef CONFIG_SCHED_CORE 12472 static inline bool 12473 __entity_slice_used(struct sched_entity *se, int min_nr_tasks) 12474 { 12475 u64 rtime = se->sum_exec_runtime - se->prev_sum_exec_runtime; 12476 u64 slice = se->slice; 12477 12478 return (rtime * min_nr_tasks > slice); 12479 } 12480 12481 #define MIN_NR_TASKS_DURING_FORCEIDLE 2 12482 static inline void task_tick_core(struct rq *rq, struct task_struct *curr) 12483 { 12484 if (!sched_core_enabled(rq)) 12485 return; 12486 12487 /* 12488 * If runqueue has only one task which used up its slice and 12489 * if the sibling is forced idle, then trigger schedule to 12490 * give forced idle task a chance. 12491 * 12492 * sched_slice() considers only this active rq and it gets the 12493 * whole slice. But during force idle, we have siblings acting 12494 * like a single runqueue and hence we need to consider runnable 12495 * tasks on this CPU and the forced idle CPU. Ideally, we should 12496 * go through the forced idle rq, but that would be a perf hit. 12497 * We can assume that the forced idle CPU has at least 12498 * MIN_NR_TASKS_DURING_FORCEIDLE - 1 tasks and use that to check 12499 * if we need to give up the CPU. 12500 */ 12501 if (rq->core->core_forceidle_count && rq->cfs.nr_running == 1 && 12502 __entity_slice_used(&curr->se, MIN_NR_TASKS_DURING_FORCEIDLE)) 12503 resched_curr(rq); 12504 } 12505 12506 /* 12507 * se_fi_update - Update the cfs_rq->min_vruntime_fi in a CFS hierarchy if needed. 12508 */ 12509 static void se_fi_update(const struct sched_entity *se, unsigned int fi_seq, 12510 bool forceidle) 12511 { 12512 for_each_sched_entity(se) { 12513 struct cfs_rq *cfs_rq = cfs_rq_of(se); 12514 12515 if (forceidle) { 12516 if (cfs_rq->forceidle_seq == fi_seq) 12517 break; 12518 cfs_rq->forceidle_seq = fi_seq; 12519 } 12520 12521 cfs_rq->min_vruntime_fi = cfs_rq->min_vruntime; 12522 } 12523 } 12524 12525 void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi) 12526 { 12527 struct sched_entity *se = &p->se; 12528 12529 if (p->sched_class != &fair_sched_class) 12530 return; 12531 12532 se_fi_update(se, rq->core->core_forceidle_seq, in_fi); 12533 } 12534 12535 bool cfs_prio_less(const struct task_struct *a, const struct task_struct *b, 12536 bool in_fi) 12537 { 12538 struct rq *rq = task_rq(a); 12539 const struct sched_entity *sea = &a->se; 12540 const struct sched_entity *seb = &b->se; 12541 struct cfs_rq *cfs_rqa; 12542 struct cfs_rq *cfs_rqb; 12543 s64 delta; 12544 12545 SCHED_WARN_ON(task_rq(b)->core != rq->core); 12546 12547 #ifdef CONFIG_FAIR_GROUP_SCHED 12548 /* 12549 * Find an se in the hierarchy for tasks a and b, such that the se's 12550 * are immediate siblings. 12551 */ 12552 while (sea->cfs_rq->tg != seb->cfs_rq->tg) { 12553 int sea_depth = sea->depth; 12554 int seb_depth = seb->depth; 12555 12556 if (sea_depth >= seb_depth) 12557 sea = parent_entity(sea); 12558 if (sea_depth <= seb_depth) 12559 seb = parent_entity(seb); 12560 } 12561 12562 se_fi_update(sea, rq->core->core_forceidle_seq, in_fi); 12563 se_fi_update(seb, rq->core->core_forceidle_seq, in_fi); 12564 12565 cfs_rqa = sea->cfs_rq; 12566 cfs_rqb = seb->cfs_rq; 12567 #else 12568 cfs_rqa = &task_rq(a)->cfs; 12569 cfs_rqb = &task_rq(b)->cfs; 12570 #endif 12571 12572 /* 12573 * Find delta after normalizing se's vruntime with its cfs_rq's 12574 * min_vruntime_fi, which would have been updated in prior calls 12575 * to se_fi_update(). 12576 */ 12577 delta = (s64)(sea->vruntime - seb->vruntime) + 12578 (s64)(cfs_rqb->min_vruntime_fi - cfs_rqa->min_vruntime_fi); 12579 12580 return delta > 0; 12581 } 12582 12583 static int task_is_throttled_fair(struct task_struct *p, int cpu) 12584 { 12585 struct cfs_rq *cfs_rq; 12586 12587 #ifdef CONFIG_FAIR_GROUP_SCHED 12588 cfs_rq = task_group(p)->cfs_rq[cpu]; 12589 #else 12590 cfs_rq = &cpu_rq(cpu)->cfs; 12591 #endif 12592 return throttled_hierarchy(cfs_rq); 12593 } 12594 #else 12595 static inline void task_tick_core(struct rq *rq, struct task_struct *curr) {} 12596 #endif 12597 12598 /* 12599 * scheduler tick hitting a task of our scheduling class. 12600 * 12601 * NOTE: This function can be called remotely by the tick offload that 12602 * goes along full dynticks. Therefore no local assumption can be made 12603 * and everything must be accessed through the @rq and @curr passed in 12604 * parameters. 12605 */ 12606 static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) 12607 { 12608 struct cfs_rq *cfs_rq; 12609 struct sched_entity *se = &curr->se; 12610 12611 for_each_sched_entity(se) { 12612 cfs_rq = cfs_rq_of(se); 12613 entity_tick(cfs_rq, se, queued); 12614 } 12615 12616 if (static_branch_unlikely(&sched_numa_balancing)) 12617 task_tick_numa(rq, curr); 12618 12619 update_misfit_status(curr, rq); 12620 check_update_overutilized_status(task_rq(curr)); 12621 12622 task_tick_core(rq, curr); 12623 } 12624 12625 /* 12626 * called on fork with the child task as argument from the parent's context 12627 * - child not yet on the tasklist 12628 * - preemption disabled 12629 */ 12630 static void task_fork_fair(struct task_struct *p) 12631 { 12632 struct sched_entity *se = &p->se, *curr; 12633 struct cfs_rq *cfs_rq; 12634 struct rq *rq = this_rq(); 12635 struct rq_flags rf; 12636 12637 rq_lock(rq, &rf); 12638 update_rq_clock(rq); 12639 12640 cfs_rq = task_cfs_rq(current); 12641 curr = cfs_rq->curr; 12642 if (curr) 12643 update_curr(cfs_rq); 12644 place_entity(cfs_rq, se, ENQUEUE_INITIAL); 12645 rq_unlock(rq, &rf); 12646 } 12647 12648 /* 12649 * Priority of the task has changed. Check to see if we preempt 12650 * the current task. 12651 */ 12652 static void 12653 prio_changed_fair(struct rq *rq, struct task_struct *p, int oldprio) 12654 { 12655 if (!task_on_rq_queued(p)) 12656 return; 12657 12658 if (rq->cfs.nr_running == 1) 12659 return; 12660 12661 /* 12662 * Reschedule if we are currently running on this runqueue and 12663 * our priority decreased, or if we are not currently running on 12664 * this runqueue and our priority is higher than the current's 12665 */ 12666 if (task_current(rq, p)) { 12667 if (p->prio > oldprio) 12668 resched_curr(rq); 12669 } else 12670 wakeup_preempt(rq, p, 0); 12671 } 12672 12673 #ifdef CONFIG_FAIR_GROUP_SCHED 12674 /* 12675 * Propagate the changes of the sched_entity across the tg tree to make it 12676 * visible to the root 12677 */ 12678 static void propagate_entity_cfs_rq(struct sched_entity *se) 12679 { 12680 struct cfs_rq *cfs_rq = cfs_rq_of(se); 12681 12682 if (cfs_rq_throttled(cfs_rq)) 12683 return; 12684 12685 if (!throttled_hierarchy(cfs_rq)) 12686 list_add_leaf_cfs_rq(cfs_rq); 12687 12688 /* Start to propagate at parent */ 12689 se = se->parent; 12690 12691 for_each_sched_entity(se) { 12692 cfs_rq = cfs_rq_of(se); 12693 12694 update_load_avg(cfs_rq, se, UPDATE_TG); 12695 12696 if (cfs_rq_throttled(cfs_rq)) 12697 break; 12698 12699 if (!throttled_hierarchy(cfs_rq)) 12700 list_add_leaf_cfs_rq(cfs_rq); 12701 } 12702 } 12703 #else 12704 static void propagate_entity_cfs_rq(struct sched_entity *se) { } 12705 #endif 12706 12707 static void detach_entity_cfs_rq(struct sched_entity *se) 12708 { 12709 struct cfs_rq *cfs_rq = cfs_rq_of(se); 12710 12711 #ifdef CONFIG_SMP 12712 /* 12713 * In case the task sched_avg hasn't been attached: 12714 * - A forked task which hasn't been woken up by wake_up_new_task(). 12715 * - A task which has been woken up by try_to_wake_up() but is 12716 * waiting for actually being woken up by sched_ttwu_pending(). 12717 */ 12718 if (!se->avg.last_update_time) 12719 return; 12720 #endif 12721 12722 /* Catch up with the cfs_rq and remove our load when we leave */ 12723 update_load_avg(cfs_rq, se, 0); 12724 detach_entity_load_avg(cfs_rq, se); 12725 update_tg_load_avg(cfs_rq); 12726 propagate_entity_cfs_rq(se); 12727 } 12728 12729 static void attach_entity_cfs_rq(struct sched_entity *se) 12730 { 12731 struct cfs_rq *cfs_rq = cfs_rq_of(se); 12732 12733 /* Synchronize entity with its cfs_rq */ 12734 update_load_avg(cfs_rq, se, sched_feat(ATTACH_AGE_LOAD) ? 0 : SKIP_AGE_LOAD); 12735 attach_entity_load_avg(cfs_rq, se); 12736 update_tg_load_avg(cfs_rq); 12737 propagate_entity_cfs_rq(se); 12738 } 12739 12740 static void detach_task_cfs_rq(struct task_struct *p) 12741 { 12742 struct sched_entity *se = &p->se; 12743 12744 detach_entity_cfs_rq(se); 12745 } 12746 12747 static void attach_task_cfs_rq(struct task_struct *p) 12748 { 12749 struct sched_entity *se = &p->se; 12750 12751 attach_entity_cfs_rq(se); 12752 } 12753 12754 static void switched_from_fair(struct rq *rq, struct task_struct *p) 12755 { 12756 detach_task_cfs_rq(p); 12757 } 12758 12759 static void switched_to_fair(struct rq *rq, struct task_struct *p) 12760 { 12761 attach_task_cfs_rq(p); 12762 12763 if (task_on_rq_queued(p)) { 12764 /* 12765 * We were most likely switched from sched_rt, so 12766 * kick off the schedule if running, otherwise just see 12767 * if we can still preempt the current task. 12768 */ 12769 if (task_current(rq, p)) 12770 resched_curr(rq); 12771 else 12772 wakeup_preempt(rq, p, 0); 12773 } 12774 } 12775 12776 /* Account for a task changing its policy or group. 12777 * 12778 * This routine is mostly called to set cfs_rq->curr field when a task 12779 * migrates between groups/classes. 12780 */ 12781 static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) 12782 { 12783 struct sched_entity *se = &p->se; 12784 12785 #ifdef CONFIG_SMP 12786 if (task_on_rq_queued(p)) { 12787 /* 12788 * Move the next running task to the front of the list, so our 12789 * cfs_tasks list becomes MRU one. 12790 */ 12791 list_move(&se->group_node, &rq->cfs_tasks); 12792 } 12793 #endif 12794 12795 for_each_sched_entity(se) { 12796 struct cfs_rq *cfs_rq = cfs_rq_of(se); 12797 12798 set_next_entity(cfs_rq, se); 12799 /* ensure bandwidth has been allocated on our new cfs_rq */ 12800 account_cfs_rq_runtime(cfs_rq, 0); 12801 } 12802 } 12803 12804 void init_cfs_rq(struct cfs_rq *cfs_rq) 12805 { 12806 cfs_rq->tasks_timeline = RB_ROOT_CACHED; 12807 u64_u32_store(cfs_rq->min_vruntime, (u64)(-(1LL << 20))); 12808 #ifdef CONFIG_SMP 12809 raw_spin_lock_init(&cfs_rq->removed.lock); 12810 #endif 12811 } 12812 12813 #ifdef CONFIG_FAIR_GROUP_SCHED 12814 static void task_change_group_fair(struct task_struct *p) 12815 { 12816 /* 12817 * We couldn't detach or attach a forked task which 12818 * hasn't been woken up by wake_up_new_task(). 12819 */ 12820 if (READ_ONCE(p->__state) == TASK_NEW) 12821 return; 12822 12823 detach_task_cfs_rq(p); 12824 12825 #ifdef CONFIG_SMP 12826 /* Tell se's cfs_rq has been changed -- migrated */ 12827 p->se.avg.last_update_time = 0; 12828 #endif 12829 set_task_rq(p, task_cpu(p)); 12830 attach_task_cfs_rq(p); 12831 } 12832 12833 void free_fair_sched_group(struct task_group *tg) 12834 { 12835 int i; 12836 12837 for_each_possible_cpu(i) { 12838 if (tg->cfs_rq) 12839 kfree(tg->cfs_rq[i]); 12840 if (tg->se) 12841 kfree(tg->se[i]); 12842 } 12843 12844 kfree(tg->cfs_rq); 12845 kfree(tg->se); 12846 } 12847 12848 int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) 12849 { 12850 struct sched_entity *se; 12851 struct cfs_rq *cfs_rq; 12852 int i; 12853 12854 tg->cfs_rq = kcalloc(nr_cpu_ids, sizeof(cfs_rq), GFP_KERNEL); 12855 if (!tg->cfs_rq) 12856 goto err; 12857 tg->se = kcalloc(nr_cpu_ids, sizeof(se), GFP_KERNEL); 12858 if (!tg->se) 12859 goto err; 12860 12861 tg->shares = NICE_0_LOAD; 12862 12863 init_cfs_bandwidth(tg_cfs_bandwidth(tg), tg_cfs_bandwidth(parent)); 12864 12865 for_each_possible_cpu(i) { 12866 cfs_rq = kzalloc_node(sizeof(struct cfs_rq), 12867 GFP_KERNEL, cpu_to_node(i)); 12868 if (!cfs_rq) 12869 goto err; 12870 12871 se = kzalloc_node(sizeof(struct sched_entity_stats), 12872 GFP_KERNEL, cpu_to_node(i)); 12873 if (!se) 12874 goto err_free_rq; 12875 12876 init_cfs_rq(cfs_rq); 12877 init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]); 12878 init_entity_runnable_average(se); 12879 } 12880 12881 return 1; 12882 12883 err_free_rq: 12884 kfree(cfs_rq); 12885 err: 12886 return 0; 12887 } 12888 12889 void online_fair_sched_group(struct task_group *tg) 12890 { 12891 struct sched_entity *se; 12892 struct rq_flags rf; 12893 struct rq *rq; 12894 int i; 12895 12896 for_each_possible_cpu(i) { 12897 rq = cpu_rq(i); 12898 se = tg->se[i]; 12899 rq_lock_irq(rq, &rf); 12900 update_rq_clock(rq); 12901 attach_entity_cfs_rq(se); 12902 sync_throttle(tg, i); 12903 rq_unlock_irq(rq, &rf); 12904 } 12905 } 12906 12907 void unregister_fair_sched_group(struct task_group *tg) 12908 { 12909 unsigned long flags; 12910 struct rq *rq; 12911 int cpu; 12912 12913 destroy_cfs_bandwidth(tg_cfs_bandwidth(tg)); 12914 12915 for_each_possible_cpu(cpu) { 12916 if (tg->se[cpu]) 12917 remove_entity_load_avg(tg->se[cpu]); 12918 12919 /* 12920 * Only empty task groups can be destroyed; so we can speculatively 12921 * check on_list without danger of it being re-added. 12922 */ 12923 if (!tg->cfs_rq[cpu]->on_list) 12924 continue; 12925 12926 rq = cpu_rq(cpu); 12927 12928 raw_spin_rq_lock_irqsave(rq, flags); 12929 list_del_leaf_cfs_rq(tg->cfs_rq[cpu]); 12930 raw_spin_rq_unlock_irqrestore(rq, flags); 12931 } 12932 } 12933 12934 void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq, 12935 struct sched_entity *se, int cpu, 12936 struct sched_entity *parent) 12937 { 12938 struct rq *rq = cpu_rq(cpu); 12939 12940 cfs_rq->tg = tg; 12941 cfs_rq->rq = rq; 12942 init_cfs_rq_runtime(cfs_rq); 12943 12944 tg->cfs_rq[cpu] = cfs_rq; 12945 tg->se[cpu] = se; 12946 12947 /* se could be NULL for root_task_group */ 12948 if (!se) 12949 return; 12950 12951 if (!parent) { 12952 se->cfs_rq = &rq->cfs; 12953 se->depth = 0; 12954 } else { 12955 se->cfs_rq = parent->my_q; 12956 se->depth = parent->depth + 1; 12957 } 12958 12959 se->my_q = cfs_rq; 12960 /* guarantee group entities always have weight */ 12961 update_load_set(&se->load, NICE_0_LOAD); 12962 se->parent = parent; 12963 } 12964 12965 static DEFINE_MUTEX(shares_mutex); 12966 12967 static int __sched_group_set_shares(struct task_group *tg, unsigned long shares) 12968 { 12969 int i; 12970 12971 lockdep_assert_held(&shares_mutex); 12972 12973 /* 12974 * We can't change the weight of the root cgroup. 12975 */ 12976 if (!tg->se[0]) 12977 return -EINVAL; 12978 12979 shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES)); 12980 12981 if (tg->shares == shares) 12982 return 0; 12983 12984 tg->shares = shares; 12985 for_each_possible_cpu(i) { 12986 struct rq *rq = cpu_rq(i); 12987 struct sched_entity *se = tg->se[i]; 12988 struct rq_flags rf; 12989 12990 /* Propagate contribution to hierarchy */ 12991 rq_lock_irqsave(rq, &rf); 12992 update_rq_clock(rq); 12993 for_each_sched_entity(se) { 12994 update_load_avg(cfs_rq_of(se), se, UPDATE_TG); 12995 update_cfs_group(se); 12996 } 12997 rq_unlock_irqrestore(rq, &rf); 12998 } 12999 13000 return 0; 13001 } 13002 13003 int sched_group_set_shares(struct task_group *tg, unsigned long shares) 13004 { 13005 int ret; 13006 13007 mutex_lock(&shares_mutex); 13008 if (tg_is_idle(tg)) 13009 ret = -EINVAL; 13010 else 13011 ret = __sched_group_set_shares(tg, shares); 13012 mutex_unlock(&shares_mutex); 13013 13014 return ret; 13015 } 13016 13017 int sched_group_set_idle(struct task_group *tg, long idle) 13018 { 13019 int i; 13020 13021 if (tg == &root_task_group) 13022 return -EINVAL; 13023 13024 if (idle < 0 || idle > 1) 13025 return -EINVAL; 13026 13027 mutex_lock(&shares_mutex); 13028 13029 if (tg->idle == idle) { 13030 mutex_unlock(&shares_mutex); 13031 return 0; 13032 } 13033 13034 tg->idle = idle; 13035 13036 for_each_possible_cpu(i) { 13037 struct rq *rq = cpu_rq(i); 13038 struct sched_entity *se = tg->se[i]; 13039 struct cfs_rq *parent_cfs_rq, *grp_cfs_rq = tg->cfs_rq[i]; 13040 bool was_idle = cfs_rq_is_idle(grp_cfs_rq); 13041 long idle_task_delta; 13042 struct rq_flags rf; 13043 13044 rq_lock_irqsave(rq, &rf); 13045 13046 grp_cfs_rq->idle = idle; 13047 if (WARN_ON_ONCE(was_idle == cfs_rq_is_idle(grp_cfs_rq))) 13048 goto next_cpu; 13049 13050 if (se->on_rq) { 13051 parent_cfs_rq = cfs_rq_of(se); 13052 if (cfs_rq_is_idle(grp_cfs_rq)) 13053 parent_cfs_rq->idle_nr_running++; 13054 else 13055 parent_cfs_rq->idle_nr_running--; 13056 } 13057 13058 idle_task_delta = grp_cfs_rq->h_nr_running - 13059 grp_cfs_rq->idle_h_nr_running; 13060 if (!cfs_rq_is_idle(grp_cfs_rq)) 13061 idle_task_delta *= -1; 13062 13063 for_each_sched_entity(se) { 13064 struct cfs_rq *cfs_rq = cfs_rq_of(se); 13065 13066 if (!se->on_rq) 13067 break; 13068 13069 cfs_rq->idle_h_nr_running += idle_task_delta; 13070 13071 /* Already accounted at parent level and above. */ 13072 if (cfs_rq_is_idle(cfs_rq)) 13073 break; 13074 } 13075 13076 next_cpu: 13077 rq_unlock_irqrestore(rq, &rf); 13078 } 13079 13080 /* Idle groups have minimum weight. */ 13081 if (tg_is_idle(tg)) 13082 __sched_group_set_shares(tg, scale_load(WEIGHT_IDLEPRIO)); 13083 else 13084 __sched_group_set_shares(tg, NICE_0_LOAD); 13085 13086 mutex_unlock(&shares_mutex); 13087 return 0; 13088 } 13089 13090 #else /* CONFIG_FAIR_GROUP_SCHED */ 13091 13092 void free_fair_sched_group(struct task_group *tg) { } 13093 13094 int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) 13095 { 13096 return 1; 13097 } 13098 13099 void online_fair_sched_group(struct task_group *tg) { } 13100 13101 void unregister_fair_sched_group(struct task_group *tg) { } 13102 13103 #endif /* CONFIG_FAIR_GROUP_SCHED */ 13104 13105 13106 static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task) 13107 { 13108 struct sched_entity *se = &task->se; 13109 unsigned int rr_interval = 0; 13110 13111 /* 13112 * Time slice is 0 for SCHED_OTHER tasks that are on an otherwise 13113 * idle runqueue: 13114 */ 13115 if (rq->cfs.load.weight) 13116 rr_interval = NS_TO_JIFFIES(se->slice); 13117 13118 return rr_interval; 13119 } 13120 13121 /* 13122 * All the scheduling class methods: 13123 */ 13124 DEFINE_SCHED_CLASS(fair) = { 13125 13126 .enqueue_task = enqueue_task_fair, 13127 .dequeue_task = dequeue_task_fair, 13128 .yield_task = yield_task_fair, 13129 .yield_to_task = yield_to_task_fair, 13130 13131 .wakeup_preempt = check_preempt_wakeup_fair, 13132 13133 .pick_next_task = __pick_next_task_fair, 13134 .put_prev_task = put_prev_task_fair, 13135 .set_next_task = set_next_task_fair, 13136 13137 #ifdef CONFIG_SMP 13138 .balance = balance_fair, 13139 .pick_task = pick_task_fair, 13140 .select_task_rq = select_task_rq_fair, 13141 .migrate_task_rq = migrate_task_rq_fair, 13142 13143 .rq_online = rq_online_fair, 13144 .rq_offline = rq_offline_fair, 13145 13146 .task_dead = task_dead_fair, 13147 .set_cpus_allowed = set_cpus_allowed_common, 13148 #endif 13149 13150 .task_tick = task_tick_fair, 13151 .task_fork = task_fork_fair, 13152 13153 .prio_changed = prio_changed_fair, 13154 .switched_from = switched_from_fair, 13155 .switched_to = switched_to_fair, 13156 13157 .get_rr_interval = get_rr_interval_fair, 13158 13159 .update_curr = update_curr_fair, 13160 13161 #ifdef CONFIG_FAIR_GROUP_SCHED 13162 .task_change_group = task_change_group_fair, 13163 #endif 13164 13165 #ifdef CONFIG_SCHED_CORE 13166 .task_is_throttled = task_is_throttled_fair, 13167 #endif 13168 13169 #ifdef CONFIG_UCLAMP_TASK 13170 .uclamp_enabled = 1, 13171 #endif 13172 }; 13173 13174 #ifdef CONFIG_SCHED_DEBUG 13175 void print_cfs_stats(struct seq_file *m, int cpu) 13176 { 13177 struct cfs_rq *cfs_rq, *pos; 13178 13179 rcu_read_lock(); 13180 for_each_leaf_cfs_rq_safe(cpu_rq(cpu), cfs_rq, pos) 13181 print_cfs_rq(m, cpu, cfs_rq); 13182 rcu_read_unlock(); 13183 } 13184 13185 #ifdef CONFIG_NUMA_BALANCING 13186 void show_numa_stats(struct task_struct *p, struct seq_file *m) 13187 { 13188 int node; 13189 unsigned long tsf = 0, tpf = 0, gsf = 0, gpf = 0; 13190 struct numa_group *ng; 13191 13192 rcu_read_lock(); 13193 ng = rcu_dereference(p->numa_group); 13194 for_each_online_node(node) { 13195 if (p->numa_faults) { 13196 tsf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 0)]; 13197 tpf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 1)]; 13198 } 13199 if (ng) { 13200 gsf = ng->faults[task_faults_idx(NUMA_MEM, node, 0)], 13201 gpf = ng->faults[task_faults_idx(NUMA_MEM, node, 1)]; 13202 } 13203 print_numa_stats(m, node, tsf, tpf, gsf, gpf); 13204 } 13205 rcu_read_unlock(); 13206 } 13207 #endif /* CONFIG_NUMA_BALANCING */ 13208 #endif /* CONFIG_SCHED_DEBUG */ 13209 13210 __init void init_sched_fair_class(void) 13211 { 13212 #ifdef CONFIG_SMP 13213 int i; 13214 13215 for_each_possible_cpu(i) { 13216 zalloc_cpumask_var_node(&per_cpu(load_balance_mask, i), GFP_KERNEL, cpu_to_node(i)); 13217 zalloc_cpumask_var_node(&per_cpu(select_rq_mask, i), GFP_KERNEL, cpu_to_node(i)); 13218 zalloc_cpumask_var_node(&per_cpu(should_we_balance_tmpmask, i), 13219 GFP_KERNEL, cpu_to_node(i)); 13220 13221 #ifdef CONFIG_CFS_BANDWIDTH 13222 INIT_CSD(&cpu_rq(i)->cfsb_csd, __cfsb_csd_unthrottle, cpu_rq(i)); 13223 INIT_LIST_HEAD(&cpu_rq(i)->cfsb_csd_list); 13224 #endif 13225 } 13226 13227 open_softirq(SCHED_SOFTIRQ, run_rebalance_domains); 13228 13229 #ifdef CONFIG_NO_HZ_COMMON 13230 nohz.next_balance = jiffies; 13231 nohz.next_blocked = jiffies; 13232 zalloc_cpumask_var(&nohz.idle_cpus_mask, GFP_NOWAIT); 13233 #endif 13234 #endif /* SMP */ 13235 13236 } 13237