1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Completely Fair Scheduling (CFS) Class (SCHED_NORMAL/SCHED_BATCH) 4 * 5 * Copyright (C) 2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> 6 * 7 * Interactivity improvements by Mike Galbraith 8 * (C) 2007 Mike Galbraith <efault@gmx.de> 9 * 10 * Various enhancements by Dmitry Adamushko. 11 * (C) 2007 Dmitry Adamushko <dmitry.adamushko@gmail.com> 12 * 13 * Group scheduling enhancements by Srivatsa Vaddagiri 14 * Copyright IBM Corporation, 2007 15 * Author: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com> 16 * 17 * Scaled math optimizations by Thomas Gleixner 18 * Copyright (C) 2007, Thomas Gleixner <tglx@linutronix.de> 19 * 20 * Adaptive scheduling granularity, math enhancements by Peter Zijlstra 21 * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra 22 */ 23 #include "sched.h" 24 25 #include <trace/events/sched.h> 26 27 /* 28 * Targeted preemption latency for CPU-bound tasks: 29 * 30 * NOTE: this latency value is not the same as the concept of 31 * 'timeslice length' - timeslices in CFS are of variable length 32 * and have no persistent notion like in traditional, time-slice 33 * based scheduling concepts. 34 * 35 * (to see the precise effective timeslice length of your workload, 36 * run vmstat and monitor the context-switches (cs) field) 37 * 38 * (default: 6ms * (1 + ilog(ncpus)), units: nanoseconds) 39 */ 40 unsigned int sysctl_sched_latency = 6000000ULL; 41 static unsigned int normalized_sysctl_sched_latency = 6000000ULL; 42 43 /* 44 * The initial- and re-scaling of tunables is configurable 45 * 46 * Options are: 47 * 48 * SCHED_TUNABLESCALING_NONE - unscaled, always *1 49 * SCHED_TUNABLESCALING_LOG - scaled logarithmical, *1+ilog(ncpus) 50 * SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus 51 * 52 * (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus)) 53 */ 54 enum sched_tunable_scaling sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG; 55 56 /* 57 * Minimal preemption granularity for CPU-bound tasks: 58 * 59 * (default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds) 60 */ 61 unsigned int sysctl_sched_min_granularity = 750000ULL; 62 static unsigned int normalized_sysctl_sched_min_granularity = 750000ULL; 63 64 /* 65 * This value is kept at sysctl_sched_latency/sysctl_sched_min_granularity 66 */ 67 static unsigned int sched_nr_latency = 8; 68 69 /* 70 * After fork, child runs first. If set to 0 (default) then 71 * parent will (try to) run first. 72 */ 73 unsigned int sysctl_sched_child_runs_first __read_mostly; 74 75 /* 76 * SCHED_OTHER wake-up granularity. 77 * 78 * This option delays the preemption effects of decoupled workloads 79 * and reduces their over-scheduling. Synchronous workloads will still 80 * have immediate wakeup/sleep latencies. 81 * 82 * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds) 83 */ 84 unsigned int sysctl_sched_wakeup_granularity = 1000000UL; 85 static unsigned int normalized_sysctl_sched_wakeup_granularity = 1000000UL; 86 87 const_debug unsigned int sysctl_sched_migration_cost = 500000UL; 88 89 #ifdef CONFIG_SMP 90 /* 91 * For asym packing, by default the lower numbered CPU has higher priority. 92 */ 93 int __weak arch_asym_cpu_priority(int cpu) 94 { 95 return -cpu; 96 } 97 98 /* 99 * The margin used when comparing utilization with CPU capacity: 100 * util * margin < capacity * 1024 101 * 102 * (default: ~20%) 103 */ 104 static unsigned int capacity_margin = 1280; 105 #endif 106 107 #ifdef CONFIG_CFS_BANDWIDTH 108 /* 109 * Amount of runtime to allocate from global (tg) to local (per-cfs_rq) pool 110 * each time a cfs_rq requests quota. 111 * 112 * Note: in the case that the slice exceeds the runtime remaining (either due 113 * to consumption or the quota being specified to be smaller than the slice) 114 * we will always only issue the remaining available time. 115 * 116 * (default: 5 msec, units: microseconds) 117 */ 118 unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; 119 #endif 120 121 static inline void update_load_add(struct load_weight *lw, unsigned long inc) 122 { 123 lw->weight += inc; 124 lw->inv_weight = 0; 125 } 126 127 static inline void update_load_sub(struct load_weight *lw, unsigned long dec) 128 { 129 lw->weight -= dec; 130 lw->inv_weight = 0; 131 } 132 133 static inline void update_load_set(struct load_weight *lw, unsigned long w) 134 { 135 lw->weight = w; 136 lw->inv_weight = 0; 137 } 138 139 /* 140 * Increase the granularity value when there are more CPUs, 141 * because with more CPUs the 'effective latency' as visible 142 * to users decreases. But the relationship is not linear, 143 * so pick a second-best guess by going with the log2 of the 144 * number of CPUs. 145 * 146 * This idea comes from the SD scheduler of Con Kolivas: 147 */ 148 static unsigned int get_update_sysctl_factor(void) 149 { 150 unsigned int cpus = min_t(unsigned int, num_online_cpus(), 8); 151 unsigned int factor; 152 153 switch (sysctl_sched_tunable_scaling) { 154 case SCHED_TUNABLESCALING_NONE: 155 factor = 1; 156 break; 157 case SCHED_TUNABLESCALING_LINEAR: 158 factor = cpus; 159 break; 160 case SCHED_TUNABLESCALING_LOG: 161 default: 162 factor = 1 + ilog2(cpus); 163 break; 164 } 165 166 return factor; 167 } 168 169 static void update_sysctl(void) 170 { 171 unsigned int factor = get_update_sysctl_factor(); 172 173 #define SET_SYSCTL(name) \ 174 (sysctl_##name = (factor) * normalized_sysctl_##name) 175 SET_SYSCTL(sched_min_granularity); 176 SET_SYSCTL(sched_latency); 177 SET_SYSCTL(sched_wakeup_granularity); 178 #undef SET_SYSCTL 179 } 180 181 void sched_init_granularity(void) 182 { 183 update_sysctl(); 184 } 185 186 #define WMULT_CONST (~0U) 187 #define WMULT_SHIFT 32 188 189 static void __update_inv_weight(struct load_weight *lw) 190 { 191 unsigned long w; 192 193 if (likely(lw->inv_weight)) 194 return; 195 196 w = scale_load_down(lw->weight); 197 198 if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST)) 199 lw->inv_weight = 1; 200 else if (unlikely(!w)) 201 lw->inv_weight = WMULT_CONST; 202 else 203 lw->inv_weight = WMULT_CONST / w; 204 } 205 206 /* 207 * delta_exec * weight / lw.weight 208 * OR 209 * (delta_exec * (weight * lw->inv_weight)) >> WMULT_SHIFT 210 * 211 * Either weight := NICE_0_LOAD and lw \e sched_prio_to_wmult[], in which case 212 * we're guaranteed shift stays positive because inv_weight is guaranteed to 213 * fit 32 bits, and NICE_0_LOAD gives another 10 bits; therefore shift >= 22. 214 * 215 * Or, weight =< lw.weight (because lw.weight is the runqueue weight), thus 216 * weight/lw.weight <= 1, and therefore our shift will also be positive. 217 */ 218 static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw) 219 { 220 u64 fact = scale_load_down(weight); 221 int shift = WMULT_SHIFT; 222 223 __update_inv_weight(lw); 224 225 if (unlikely(fact >> 32)) { 226 while (fact >> 32) { 227 fact >>= 1; 228 shift--; 229 } 230 } 231 232 /* hint to use a 32x32->64 mul */ 233 fact = (u64)(u32)fact * lw->inv_weight; 234 235 while (fact >> 32) { 236 fact >>= 1; 237 shift--; 238 } 239 240 return mul_u64_u32_shr(delta_exec, fact, shift); 241 } 242 243 244 const struct sched_class fair_sched_class; 245 246 /************************************************************** 247 * CFS operations on generic schedulable entities: 248 */ 249 250 #ifdef CONFIG_FAIR_GROUP_SCHED 251 252 /* cpu runqueue to which this cfs_rq is attached */ 253 static inline struct rq *rq_of(struct cfs_rq *cfs_rq) 254 { 255 return cfs_rq->rq; 256 } 257 258 static inline struct task_struct *task_of(struct sched_entity *se) 259 { 260 SCHED_WARN_ON(!entity_is_task(se)); 261 return container_of(se, struct task_struct, se); 262 } 263 264 /* Walk up scheduling entities hierarchy */ 265 #define for_each_sched_entity(se) \ 266 for (; se; se = se->parent) 267 268 static inline struct cfs_rq *task_cfs_rq(struct task_struct *p) 269 { 270 return p->se.cfs_rq; 271 } 272 273 /* runqueue on which this entity is (to be) queued */ 274 static inline struct cfs_rq *cfs_rq_of(struct sched_entity *se) 275 { 276 return se->cfs_rq; 277 } 278 279 /* runqueue "owned" by this group */ 280 static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp) 281 { 282 return grp->my_q; 283 } 284 285 static inline void list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) 286 { 287 if (!cfs_rq->on_list) { 288 struct rq *rq = rq_of(cfs_rq); 289 int cpu = cpu_of(rq); 290 /* 291 * Ensure we either appear before our parent (if already 292 * enqueued) or force our parent to appear after us when it is 293 * enqueued. The fact that we always enqueue bottom-up 294 * reduces this to two cases and a special case for the root 295 * cfs_rq. Furthermore, it also means that we will always reset 296 * tmp_alone_branch either when the branch is connected 297 * to a tree or when we reach the beg of the tree 298 */ 299 if (cfs_rq->tg->parent && 300 cfs_rq->tg->parent->cfs_rq[cpu]->on_list) { 301 /* 302 * If parent is already on the list, we add the child 303 * just before. Thanks to circular linked property of 304 * the list, this means to put the child at the tail 305 * of the list that starts by parent. 306 */ 307 list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, 308 &(cfs_rq->tg->parent->cfs_rq[cpu]->leaf_cfs_rq_list)); 309 /* 310 * The branch is now connected to its tree so we can 311 * reset tmp_alone_branch to the beginning of the 312 * list. 313 */ 314 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 315 } else if (!cfs_rq->tg->parent) { 316 /* 317 * cfs rq without parent should be put 318 * at the tail of the list. 319 */ 320 list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, 321 &rq->leaf_cfs_rq_list); 322 /* 323 * We have reach the beg of a tree so we can reset 324 * tmp_alone_branch to the beginning of the list. 325 */ 326 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 327 } else { 328 /* 329 * The parent has not already been added so we want to 330 * make sure that it will be put after us. 331 * tmp_alone_branch points to the beg of the branch 332 * where we will add parent. 333 */ 334 list_add_rcu(&cfs_rq->leaf_cfs_rq_list, 335 rq->tmp_alone_branch); 336 /* 337 * update tmp_alone_branch to points to the new beg 338 * of the branch 339 */ 340 rq->tmp_alone_branch = &cfs_rq->leaf_cfs_rq_list; 341 } 342 343 cfs_rq->on_list = 1; 344 } 345 } 346 347 static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) 348 { 349 if (cfs_rq->on_list) { 350 list_del_rcu(&cfs_rq->leaf_cfs_rq_list); 351 cfs_rq->on_list = 0; 352 } 353 } 354 355 /* Iterate through all leaf cfs_rq's on a runqueue: */ 356 #define for_each_leaf_cfs_rq(rq, cfs_rq) \ 357 list_for_each_entry_rcu(cfs_rq, &rq->leaf_cfs_rq_list, leaf_cfs_rq_list) 358 359 /* Do the two (enqueued) entities belong to the same group ? */ 360 static inline struct cfs_rq * 361 is_same_group(struct sched_entity *se, struct sched_entity *pse) 362 { 363 if (se->cfs_rq == pse->cfs_rq) 364 return se->cfs_rq; 365 366 return NULL; 367 } 368 369 static inline struct sched_entity *parent_entity(struct sched_entity *se) 370 { 371 return se->parent; 372 } 373 374 static void 375 find_matching_se(struct sched_entity **se, struct sched_entity **pse) 376 { 377 int se_depth, pse_depth; 378 379 /* 380 * preemption test can be made between sibling entities who are in the 381 * same cfs_rq i.e who have a common parent. Walk up the hierarchy of 382 * both tasks until we find their ancestors who are siblings of common 383 * parent. 384 */ 385 386 /* First walk up until both entities are at same depth */ 387 se_depth = (*se)->depth; 388 pse_depth = (*pse)->depth; 389 390 while (se_depth > pse_depth) { 391 se_depth--; 392 *se = parent_entity(*se); 393 } 394 395 while (pse_depth > se_depth) { 396 pse_depth--; 397 *pse = parent_entity(*pse); 398 } 399 400 while (!is_same_group(*se, *pse)) { 401 *se = parent_entity(*se); 402 *pse = parent_entity(*pse); 403 } 404 } 405 406 #else /* !CONFIG_FAIR_GROUP_SCHED */ 407 408 static inline struct task_struct *task_of(struct sched_entity *se) 409 { 410 return container_of(se, struct task_struct, se); 411 } 412 413 static inline struct rq *rq_of(struct cfs_rq *cfs_rq) 414 { 415 return container_of(cfs_rq, struct rq, cfs); 416 } 417 418 419 #define for_each_sched_entity(se) \ 420 for (; se; se = NULL) 421 422 static inline struct cfs_rq *task_cfs_rq(struct task_struct *p) 423 { 424 return &task_rq(p)->cfs; 425 } 426 427 static inline struct cfs_rq *cfs_rq_of(struct sched_entity *se) 428 { 429 struct task_struct *p = task_of(se); 430 struct rq *rq = task_rq(p); 431 432 return &rq->cfs; 433 } 434 435 /* runqueue "owned" by this group */ 436 static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp) 437 { 438 return NULL; 439 } 440 441 static inline void list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) 442 { 443 } 444 445 static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) 446 { 447 } 448 449 #define for_each_leaf_cfs_rq(rq, cfs_rq) \ 450 for (cfs_rq = &rq->cfs; cfs_rq; cfs_rq = NULL) 451 452 static inline struct sched_entity *parent_entity(struct sched_entity *se) 453 { 454 return NULL; 455 } 456 457 static inline void 458 find_matching_se(struct sched_entity **se, struct sched_entity **pse) 459 { 460 } 461 462 #endif /* CONFIG_FAIR_GROUP_SCHED */ 463 464 static __always_inline 465 void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec); 466 467 /************************************************************** 468 * Scheduling class tree data structure manipulation methods: 469 */ 470 471 static inline u64 max_vruntime(u64 max_vruntime, u64 vruntime) 472 { 473 s64 delta = (s64)(vruntime - max_vruntime); 474 if (delta > 0) 475 max_vruntime = vruntime; 476 477 return max_vruntime; 478 } 479 480 static inline u64 min_vruntime(u64 min_vruntime, u64 vruntime) 481 { 482 s64 delta = (s64)(vruntime - min_vruntime); 483 if (delta < 0) 484 min_vruntime = vruntime; 485 486 return min_vruntime; 487 } 488 489 static inline int entity_before(struct sched_entity *a, 490 struct sched_entity *b) 491 { 492 return (s64)(a->vruntime - b->vruntime) < 0; 493 } 494 495 static void update_min_vruntime(struct cfs_rq *cfs_rq) 496 { 497 struct sched_entity *curr = cfs_rq->curr; 498 struct rb_node *leftmost = rb_first_cached(&cfs_rq->tasks_timeline); 499 500 u64 vruntime = cfs_rq->min_vruntime; 501 502 if (curr) { 503 if (curr->on_rq) 504 vruntime = curr->vruntime; 505 else 506 curr = NULL; 507 } 508 509 if (leftmost) { /* non-empty tree */ 510 struct sched_entity *se; 511 se = rb_entry(leftmost, struct sched_entity, run_node); 512 513 if (!curr) 514 vruntime = se->vruntime; 515 else 516 vruntime = min_vruntime(vruntime, se->vruntime); 517 } 518 519 /* ensure we never gain time by being placed backwards. */ 520 cfs_rq->min_vruntime = max_vruntime(cfs_rq->min_vruntime, vruntime); 521 #ifndef CONFIG_64BIT 522 smp_wmb(); 523 cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime; 524 #endif 525 } 526 527 /* 528 * Enqueue an entity into the rb-tree: 529 */ 530 static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 531 { 532 struct rb_node **link = &cfs_rq->tasks_timeline.rb_root.rb_node; 533 struct rb_node *parent = NULL; 534 struct sched_entity *entry; 535 bool leftmost = true; 536 537 /* 538 * Find the right place in the rbtree: 539 */ 540 while (*link) { 541 parent = *link; 542 entry = rb_entry(parent, struct sched_entity, run_node); 543 /* 544 * We dont care about collisions. Nodes with 545 * the same key stay together. 546 */ 547 if (entity_before(se, entry)) { 548 link = &parent->rb_left; 549 } else { 550 link = &parent->rb_right; 551 leftmost = false; 552 } 553 } 554 555 rb_link_node(&se->run_node, parent, link); 556 rb_insert_color_cached(&se->run_node, 557 &cfs_rq->tasks_timeline, leftmost); 558 } 559 560 static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 561 { 562 rb_erase_cached(&se->run_node, &cfs_rq->tasks_timeline); 563 } 564 565 struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq) 566 { 567 struct rb_node *left = rb_first_cached(&cfs_rq->tasks_timeline); 568 569 if (!left) 570 return NULL; 571 572 return rb_entry(left, struct sched_entity, run_node); 573 } 574 575 static struct sched_entity *__pick_next_entity(struct sched_entity *se) 576 { 577 struct rb_node *next = rb_next(&se->run_node); 578 579 if (!next) 580 return NULL; 581 582 return rb_entry(next, struct sched_entity, run_node); 583 } 584 585 #ifdef CONFIG_SCHED_DEBUG 586 struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq) 587 { 588 struct rb_node *last = rb_last(&cfs_rq->tasks_timeline.rb_root); 589 590 if (!last) 591 return NULL; 592 593 return rb_entry(last, struct sched_entity, run_node); 594 } 595 596 /************************************************************** 597 * Scheduling class statistics methods: 598 */ 599 600 int sched_proc_update_handler(struct ctl_table *table, int write, 601 void __user *buffer, size_t *lenp, 602 loff_t *ppos) 603 { 604 int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 605 unsigned int factor = get_update_sysctl_factor(); 606 607 if (ret || !write) 608 return ret; 609 610 sched_nr_latency = DIV_ROUND_UP(sysctl_sched_latency, 611 sysctl_sched_min_granularity); 612 613 #define WRT_SYSCTL(name) \ 614 (normalized_sysctl_##name = sysctl_##name / (factor)) 615 WRT_SYSCTL(sched_min_granularity); 616 WRT_SYSCTL(sched_latency); 617 WRT_SYSCTL(sched_wakeup_granularity); 618 #undef WRT_SYSCTL 619 620 return 0; 621 } 622 #endif 623 624 /* 625 * delta /= w 626 */ 627 static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se) 628 { 629 if (unlikely(se->load.weight != NICE_0_LOAD)) 630 delta = __calc_delta(delta, NICE_0_LOAD, &se->load); 631 632 return delta; 633 } 634 635 /* 636 * The idea is to set a period in which each task runs once. 637 * 638 * When there are too many tasks (sched_nr_latency) we have to stretch 639 * this period because otherwise the slices get too small. 640 * 641 * p = (nr <= nl) ? l : l*nr/nl 642 */ 643 static u64 __sched_period(unsigned long nr_running) 644 { 645 if (unlikely(nr_running > sched_nr_latency)) 646 return nr_running * sysctl_sched_min_granularity; 647 else 648 return sysctl_sched_latency; 649 } 650 651 /* 652 * We calculate the wall-time slice from the period by taking a part 653 * proportional to the weight. 654 * 655 * s = p*P[w/rw] 656 */ 657 static u64 sched_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) 658 { 659 u64 slice = __sched_period(cfs_rq->nr_running + !se->on_rq); 660 661 for_each_sched_entity(se) { 662 struct load_weight *load; 663 struct load_weight lw; 664 665 cfs_rq = cfs_rq_of(se); 666 load = &cfs_rq->load; 667 668 if (unlikely(!se->on_rq)) { 669 lw = cfs_rq->load; 670 671 update_load_add(&lw, se->load.weight); 672 load = &lw; 673 } 674 slice = __calc_delta(slice, se->load.weight, load); 675 } 676 return slice; 677 } 678 679 /* 680 * We calculate the vruntime slice of a to-be-inserted task. 681 * 682 * vs = s/w 683 */ 684 static u64 sched_vslice(struct cfs_rq *cfs_rq, struct sched_entity *se) 685 { 686 return calc_delta_fair(sched_slice(cfs_rq, se), se); 687 } 688 689 #ifdef CONFIG_SMP 690 #include "pelt.h" 691 #include "sched-pelt.h" 692 693 static int select_idle_sibling(struct task_struct *p, int prev_cpu, int cpu); 694 static unsigned long task_h_load(struct task_struct *p); 695 static unsigned long capacity_of(int cpu); 696 697 /* Give new sched_entity start runnable values to heavy its load in infant time */ 698 void init_entity_runnable_average(struct sched_entity *se) 699 { 700 struct sched_avg *sa = &se->avg; 701 702 memset(sa, 0, sizeof(*sa)); 703 704 /* 705 * Tasks are initialized with full load to be seen as heavy tasks until 706 * they get a chance to stabilize to their real load level. 707 * Group entities are initialized with zero load to reflect the fact that 708 * nothing has been attached to the task group yet. 709 */ 710 if (entity_is_task(se)) 711 sa->runnable_load_avg = sa->load_avg = scale_load_down(se->load.weight); 712 713 se->runnable_weight = se->load.weight; 714 715 /* when this task enqueue'ed, it will contribute to its cfs_rq's load_avg */ 716 } 717 718 static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq); 719 static void attach_entity_cfs_rq(struct sched_entity *se); 720 721 /* 722 * With new tasks being created, their initial util_avgs are extrapolated 723 * based on the cfs_rq's current util_avg: 724 * 725 * util_avg = cfs_rq->util_avg / (cfs_rq->load_avg + 1) * se.load.weight 726 * 727 * However, in many cases, the above util_avg does not give a desired 728 * value. Moreover, the sum of the util_avgs may be divergent, such 729 * as when the series is a harmonic series. 730 * 731 * To solve this problem, we also cap the util_avg of successive tasks to 732 * only 1/2 of the left utilization budget: 733 * 734 * util_avg_cap = (cpu_scale - cfs_rq->avg.util_avg) / 2^n 735 * 736 * where n denotes the nth task and cpu_scale the CPU capacity. 737 * 738 * For example, for a CPU with 1024 of capacity, a simplest series from 739 * the beginning would be like: 740 * 741 * task util_avg: 512, 256, 128, 64, 32, 16, 8, ... 742 * cfs_rq util_avg: 512, 768, 896, 960, 992, 1008, 1016, ... 743 * 744 * Finally, that extrapolated util_avg is clamped to the cap (util_avg_cap) 745 * if util_avg > util_avg_cap. 746 */ 747 void post_init_entity_util_avg(struct sched_entity *se) 748 { 749 struct cfs_rq *cfs_rq = cfs_rq_of(se); 750 struct sched_avg *sa = &se->avg; 751 long cpu_scale = arch_scale_cpu_capacity(NULL, cpu_of(rq_of(cfs_rq))); 752 long cap = (long)(cpu_scale - cfs_rq->avg.util_avg) / 2; 753 754 if (cap > 0) { 755 if (cfs_rq->avg.util_avg != 0) { 756 sa->util_avg = cfs_rq->avg.util_avg * se->load.weight; 757 sa->util_avg /= (cfs_rq->avg.load_avg + 1); 758 759 if (sa->util_avg > cap) 760 sa->util_avg = cap; 761 } else { 762 sa->util_avg = cap; 763 } 764 } 765 766 if (entity_is_task(se)) { 767 struct task_struct *p = task_of(se); 768 if (p->sched_class != &fair_sched_class) { 769 /* 770 * For !fair tasks do: 771 * 772 update_cfs_rq_load_avg(now, cfs_rq); 773 attach_entity_load_avg(cfs_rq, se, 0); 774 switched_from_fair(rq, p); 775 * 776 * such that the next switched_to_fair() has the 777 * expected state. 778 */ 779 se->avg.last_update_time = cfs_rq_clock_task(cfs_rq); 780 return; 781 } 782 } 783 784 attach_entity_cfs_rq(se); 785 } 786 787 #else /* !CONFIG_SMP */ 788 void init_entity_runnable_average(struct sched_entity *se) 789 { 790 } 791 void post_init_entity_util_avg(struct sched_entity *se) 792 { 793 } 794 static void update_tg_load_avg(struct cfs_rq *cfs_rq, int force) 795 { 796 } 797 #endif /* CONFIG_SMP */ 798 799 /* 800 * Update the current task's runtime statistics. 801 */ 802 static void update_curr(struct cfs_rq *cfs_rq) 803 { 804 struct sched_entity *curr = cfs_rq->curr; 805 u64 now = rq_clock_task(rq_of(cfs_rq)); 806 u64 delta_exec; 807 808 if (unlikely(!curr)) 809 return; 810 811 delta_exec = now - curr->exec_start; 812 if (unlikely((s64)delta_exec <= 0)) 813 return; 814 815 curr->exec_start = now; 816 817 schedstat_set(curr->statistics.exec_max, 818 max(delta_exec, curr->statistics.exec_max)); 819 820 curr->sum_exec_runtime += delta_exec; 821 schedstat_add(cfs_rq->exec_clock, delta_exec); 822 823 curr->vruntime += calc_delta_fair(delta_exec, curr); 824 update_min_vruntime(cfs_rq); 825 826 if (entity_is_task(curr)) { 827 struct task_struct *curtask = task_of(curr); 828 829 trace_sched_stat_runtime(curtask, delta_exec, curr->vruntime); 830 cgroup_account_cputime(curtask, delta_exec); 831 account_group_exec_runtime(curtask, delta_exec); 832 } 833 834 account_cfs_rq_runtime(cfs_rq, delta_exec); 835 } 836 837 static void update_curr_fair(struct rq *rq) 838 { 839 update_curr(cfs_rq_of(&rq->curr->se)); 840 } 841 842 static inline void 843 update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se) 844 { 845 u64 wait_start, prev_wait_start; 846 847 if (!schedstat_enabled()) 848 return; 849 850 wait_start = rq_clock(rq_of(cfs_rq)); 851 prev_wait_start = schedstat_val(se->statistics.wait_start); 852 853 if (entity_is_task(se) && task_on_rq_migrating(task_of(se)) && 854 likely(wait_start > prev_wait_start)) 855 wait_start -= prev_wait_start; 856 857 __schedstat_set(se->statistics.wait_start, wait_start); 858 } 859 860 static inline void 861 update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se) 862 { 863 struct task_struct *p; 864 u64 delta; 865 866 if (!schedstat_enabled()) 867 return; 868 869 delta = rq_clock(rq_of(cfs_rq)) - schedstat_val(se->statistics.wait_start); 870 871 if (entity_is_task(se)) { 872 p = task_of(se); 873 if (task_on_rq_migrating(p)) { 874 /* 875 * Preserve migrating task's wait time so wait_start 876 * time stamp can be adjusted to accumulate wait time 877 * prior to migration. 878 */ 879 __schedstat_set(se->statistics.wait_start, delta); 880 return; 881 } 882 trace_sched_stat_wait(p, delta); 883 } 884 885 __schedstat_set(se->statistics.wait_max, 886 max(schedstat_val(se->statistics.wait_max), delta)); 887 __schedstat_inc(se->statistics.wait_count); 888 __schedstat_add(se->statistics.wait_sum, delta); 889 __schedstat_set(se->statistics.wait_start, 0); 890 } 891 892 static inline void 893 update_stats_enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se) 894 { 895 struct task_struct *tsk = NULL; 896 u64 sleep_start, block_start; 897 898 if (!schedstat_enabled()) 899 return; 900 901 sleep_start = schedstat_val(se->statistics.sleep_start); 902 block_start = schedstat_val(se->statistics.block_start); 903 904 if (entity_is_task(se)) 905 tsk = task_of(se); 906 907 if (sleep_start) { 908 u64 delta = rq_clock(rq_of(cfs_rq)) - sleep_start; 909 910 if ((s64)delta < 0) 911 delta = 0; 912 913 if (unlikely(delta > schedstat_val(se->statistics.sleep_max))) 914 __schedstat_set(se->statistics.sleep_max, delta); 915 916 __schedstat_set(se->statistics.sleep_start, 0); 917 __schedstat_add(se->statistics.sum_sleep_runtime, delta); 918 919 if (tsk) { 920 account_scheduler_latency(tsk, delta >> 10, 1); 921 trace_sched_stat_sleep(tsk, delta); 922 } 923 } 924 if (block_start) { 925 u64 delta = rq_clock(rq_of(cfs_rq)) - block_start; 926 927 if ((s64)delta < 0) 928 delta = 0; 929 930 if (unlikely(delta > schedstat_val(se->statistics.block_max))) 931 __schedstat_set(se->statistics.block_max, delta); 932 933 __schedstat_set(se->statistics.block_start, 0); 934 __schedstat_add(se->statistics.sum_sleep_runtime, delta); 935 936 if (tsk) { 937 if (tsk->in_iowait) { 938 __schedstat_add(se->statistics.iowait_sum, delta); 939 __schedstat_inc(se->statistics.iowait_count); 940 trace_sched_stat_iowait(tsk, delta); 941 } 942 943 trace_sched_stat_blocked(tsk, delta); 944 945 /* 946 * Blocking time is in units of nanosecs, so shift by 947 * 20 to get a milliseconds-range estimation of the 948 * amount of time that the task spent sleeping: 949 */ 950 if (unlikely(prof_on == SLEEP_PROFILING)) { 951 profile_hits(SLEEP_PROFILING, 952 (void *)get_wchan(tsk), 953 delta >> 20); 954 } 955 account_scheduler_latency(tsk, delta >> 10, 0); 956 } 957 } 958 } 959 960 /* 961 * Task is being enqueued - update stats: 962 */ 963 static inline void 964 update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 965 { 966 if (!schedstat_enabled()) 967 return; 968 969 /* 970 * Are we enqueueing a waiting task? (for current tasks 971 * a dequeue/enqueue event is a NOP) 972 */ 973 if (se != cfs_rq->curr) 974 update_stats_wait_start(cfs_rq, se); 975 976 if (flags & ENQUEUE_WAKEUP) 977 update_stats_enqueue_sleeper(cfs_rq, se); 978 } 979 980 static inline void 981 update_stats_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 982 { 983 984 if (!schedstat_enabled()) 985 return; 986 987 /* 988 * Mark the end of the wait period if dequeueing a 989 * waiting task: 990 */ 991 if (se != cfs_rq->curr) 992 update_stats_wait_end(cfs_rq, se); 993 994 if ((flags & DEQUEUE_SLEEP) && entity_is_task(se)) { 995 struct task_struct *tsk = task_of(se); 996 997 if (tsk->state & TASK_INTERRUPTIBLE) 998 __schedstat_set(se->statistics.sleep_start, 999 rq_clock(rq_of(cfs_rq))); 1000 if (tsk->state & TASK_UNINTERRUPTIBLE) 1001 __schedstat_set(se->statistics.block_start, 1002 rq_clock(rq_of(cfs_rq))); 1003 } 1004 } 1005 1006 /* 1007 * We are picking a new current task - update its stats: 1008 */ 1009 static inline void 1010 update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se) 1011 { 1012 /* 1013 * We are starting a new run period: 1014 */ 1015 se->exec_start = rq_clock_task(rq_of(cfs_rq)); 1016 } 1017 1018 /************************************************** 1019 * Scheduling class queueing methods: 1020 */ 1021 1022 #ifdef CONFIG_NUMA_BALANCING 1023 /* 1024 * Approximate time to scan a full NUMA task in ms. The task scan period is 1025 * calculated based on the tasks virtual memory size and 1026 * numa_balancing_scan_size. 1027 */ 1028 unsigned int sysctl_numa_balancing_scan_period_min = 1000; 1029 unsigned int sysctl_numa_balancing_scan_period_max = 60000; 1030 1031 /* Portion of address space to scan in MB */ 1032 unsigned int sysctl_numa_balancing_scan_size = 256; 1033 1034 /* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */ 1035 unsigned int sysctl_numa_balancing_scan_delay = 1000; 1036 1037 struct numa_group { 1038 atomic_t refcount; 1039 1040 spinlock_t lock; /* nr_tasks, tasks */ 1041 int nr_tasks; 1042 pid_t gid; 1043 int active_nodes; 1044 1045 struct rcu_head rcu; 1046 unsigned long total_faults; 1047 unsigned long max_faults_cpu; 1048 /* 1049 * Faults_cpu is used to decide whether memory should move 1050 * towards the CPU. As a consequence, these stats are weighted 1051 * more by CPU use than by memory faults. 1052 */ 1053 unsigned long *faults_cpu; 1054 unsigned long faults[0]; 1055 }; 1056 1057 static inline unsigned long group_faults_priv(struct numa_group *ng); 1058 static inline unsigned long group_faults_shared(struct numa_group *ng); 1059 1060 static unsigned int task_nr_scan_windows(struct task_struct *p) 1061 { 1062 unsigned long rss = 0; 1063 unsigned long nr_scan_pages; 1064 1065 /* 1066 * Calculations based on RSS as non-present and empty pages are skipped 1067 * by the PTE scanner and NUMA hinting faults should be trapped based 1068 * on resident pages 1069 */ 1070 nr_scan_pages = sysctl_numa_balancing_scan_size << (20 - PAGE_SHIFT); 1071 rss = get_mm_rss(p->mm); 1072 if (!rss) 1073 rss = nr_scan_pages; 1074 1075 rss = round_up(rss, nr_scan_pages); 1076 return rss / nr_scan_pages; 1077 } 1078 1079 /* For sanitys sake, never scan more PTEs than MAX_SCAN_WINDOW MB/sec. */ 1080 #define MAX_SCAN_WINDOW 2560 1081 1082 static unsigned int task_scan_min(struct task_struct *p) 1083 { 1084 unsigned int scan_size = READ_ONCE(sysctl_numa_balancing_scan_size); 1085 unsigned int scan, floor; 1086 unsigned int windows = 1; 1087 1088 if (scan_size < MAX_SCAN_WINDOW) 1089 windows = MAX_SCAN_WINDOW / scan_size; 1090 floor = 1000 / windows; 1091 1092 scan = sysctl_numa_balancing_scan_period_min / task_nr_scan_windows(p); 1093 return max_t(unsigned int, floor, scan); 1094 } 1095 1096 static unsigned int task_scan_start(struct task_struct *p) 1097 { 1098 unsigned long smin = task_scan_min(p); 1099 unsigned long period = smin; 1100 1101 /* Scale the maximum scan period with the amount of shared memory. */ 1102 if (p->numa_group) { 1103 struct numa_group *ng = p->numa_group; 1104 unsigned long shared = group_faults_shared(ng); 1105 unsigned long private = group_faults_priv(ng); 1106 1107 period *= atomic_read(&ng->refcount); 1108 period *= shared + 1; 1109 period /= private + shared + 1; 1110 } 1111 1112 return max(smin, period); 1113 } 1114 1115 static unsigned int task_scan_max(struct task_struct *p) 1116 { 1117 unsigned long smin = task_scan_min(p); 1118 unsigned long smax; 1119 1120 /* Watch for min being lower than max due to floor calculations */ 1121 smax = sysctl_numa_balancing_scan_period_max / task_nr_scan_windows(p); 1122 1123 /* Scale the maximum scan period with the amount of shared memory. */ 1124 if (p->numa_group) { 1125 struct numa_group *ng = p->numa_group; 1126 unsigned long shared = group_faults_shared(ng); 1127 unsigned long private = group_faults_priv(ng); 1128 unsigned long period = smax; 1129 1130 period *= atomic_read(&ng->refcount); 1131 period *= shared + 1; 1132 period /= private + shared + 1; 1133 1134 smax = max(smax, period); 1135 } 1136 1137 return max(smin, smax); 1138 } 1139 1140 void init_numa_balancing(unsigned long clone_flags, struct task_struct *p) 1141 { 1142 int mm_users = 0; 1143 struct mm_struct *mm = p->mm; 1144 1145 if (mm) { 1146 mm_users = atomic_read(&mm->mm_users); 1147 if (mm_users == 1) { 1148 mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 1149 mm->numa_scan_seq = 0; 1150 } 1151 } 1152 p->node_stamp = 0; 1153 p->numa_scan_seq = mm ? mm->numa_scan_seq : 0; 1154 p->numa_scan_period = sysctl_numa_balancing_scan_delay; 1155 p->numa_work.next = &p->numa_work; 1156 p->numa_faults = NULL; 1157 p->numa_group = NULL; 1158 p->last_task_numa_placement = 0; 1159 p->last_sum_exec_runtime = 0; 1160 1161 /* New address space, reset the preferred nid */ 1162 if (!(clone_flags & CLONE_VM)) { 1163 p->numa_preferred_nid = -1; 1164 return; 1165 } 1166 1167 /* 1168 * New thread, keep existing numa_preferred_nid which should be copied 1169 * already by arch_dup_task_struct but stagger when scans start. 1170 */ 1171 if (mm) { 1172 unsigned int delay; 1173 1174 delay = min_t(unsigned int, task_scan_max(current), 1175 current->numa_scan_period * mm_users * NSEC_PER_MSEC); 1176 delay += 2 * TICK_NSEC; 1177 p->node_stamp = delay; 1178 } 1179 } 1180 1181 static void account_numa_enqueue(struct rq *rq, struct task_struct *p) 1182 { 1183 rq->nr_numa_running += (p->numa_preferred_nid != -1); 1184 rq->nr_preferred_running += (p->numa_preferred_nid == task_node(p)); 1185 } 1186 1187 static void account_numa_dequeue(struct rq *rq, struct task_struct *p) 1188 { 1189 rq->nr_numa_running -= (p->numa_preferred_nid != -1); 1190 rq->nr_preferred_running -= (p->numa_preferred_nid == task_node(p)); 1191 } 1192 1193 /* Shared or private faults. */ 1194 #define NR_NUMA_HINT_FAULT_TYPES 2 1195 1196 /* Memory and CPU locality */ 1197 #define NR_NUMA_HINT_FAULT_STATS (NR_NUMA_HINT_FAULT_TYPES * 2) 1198 1199 /* Averaged statistics, and temporary buffers. */ 1200 #define NR_NUMA_HINT_FAULT_BUCKETS (NR_NUMA_HINT_FAULT_STATS * 2) 1201 1202 pid_t task_numa_group_id(struct task_struct *p) 1203 { 1204 return p->numa_group ? p->numa_group->gid : 0; 1205 } 1206 1207 /* 1208 * The averaged statistics, shared & private, memory & CPU, 1209 * occupy the first half of the array. The second half of the 1210 * array is for current counters, which are averaged into the 1211 * first set by task_numa_placement. 1212 */ 1213 static inline int task_faults_idx(enum numa_faults_stats s, int nid, int priv) 1214 { 1215 return NR_NUMA_HINT_FAULT_TYPES * (s * nr_node_ids + nid) + priv; 1216 } 1217 1218 static inline unsigned long task_faults(struct task_struct *p, int nid) 1219 { 1220 if (!p->numa_faults) 1221 return 0; 1222 1223 return p->numa_faults[task_faults_idx(NUMA_MEM, nid, 0)] + 1224 p->numa_faults[task_faults_idx(NUMA_MEM, nid, 1)]; 1225 } 1226 1227 static inline unsigned long group_faults(struct task_struct *p, int nid) 1228 { 1229 if (!p->numa_group) 1230 return 0; 1231 1232 return p->numa_group->faults[task_faults_idx(NUMA_MEM, nid, 0)] + 1233 p->numa_group->faults[task_faults_idx(NUMA_MEM, nid, 1)]; 1234 } 1235 1236 static inline unsigned long group_faults_cpu(struct numa_group *group, int nid) 1237 { 1238 return group->faults_cpu[task_faults_idx(NUMA_MEM, nid, 0)] + 1239 group->faults_cpu[task_faults_idx(NUMA_MEM, nid, 1)]; 1240 } 1241 1242 static inline unsigned long group_faults_priv(struct numa_group *ng) 1243 { 1244 unsigned long faults = 0; 1245 int node; 1246 1247 for_each_online_node(node) { 1248 faults += ng->faults[task_faults_idx(NUMA_MEM, node, 1)]; 1249 } 1250 1251 return faults; 1252 } 1253 1254 static inline unsigned long group_faults_shared(struct numa_group *ng) 1255 { 1256 unsigned long faults = 0; 1257 int node; 1258 1259 for_each_online_node(node) { 1260 faults += ng->faults[task_faults_idx(NUMA_MEM, node, 0)]; 1261 } 1262 1263 return faults; 1264 } 1265 1266 /* 1267 * A node triggering more than 1/3 as many NUMA faults as the maximum is 1268 * considered part of a numa group's pseudo-interleaving set. Migrations 1269 * between these nodes are slowed down, to allow things to settle down. 1270 */ 1271 #define ACTIVE_NODE_FRACTION 3 1272 1273 static bool numa_is_active_node(int nid, struct numa_group *ng) 1274 { 1275 return group_faults_cpu(ng, nid) * ACTIVE_NODE_FRACTION > ng->max_faults_cpu; 1276 } 1277 1278 /* Handle placement on systems where not all nodes are directly connected. */ 1279 static unsigned long score_nearby_nodes(struct task_struct *p, int nid, 1280 int maxdist, bool task) 1281 { 1282 unsigned long score = 0; 1283 int node; 1284 1285 /* 1286 * All nodes are directly connected, and the same distance 1287 * from each other. No need for fancy placement algorithms. 1288 */ 1289 if (sched_numa_topology_type == NUMA_DIRECT) 1290 return 0; 1291 1292 /* 1293 * This code is called for each node, introducing N^2 complexity, 1294 * which should be ok given the number of nodes rarely exceeds 8. 1295 */ 1296 for_each_online_node(node) { 1297 unsigned long faults; 1298 int dist = node_distance(nid, node); 1299 1300 /* 1301 * The furthest away nodes in the system are not interesting 1302 * for placement; nid was already counted. 1303 */ 1304 if (dist == sched_max_numa_distance || node == nid) 1305 continue; 1306 1307 /* 1308 * On systems with a backplane NUMA topology, compare groups 1309 * of nodes, and move tasks towards the group with the most 1310 * memory accesses. When comparing two nodes at distance 1311 * "hoplimit", only nodes closer by than "hoplimit" are part 1312 * of each group. Skip other nodes. 1313 */ 1314 if (sched_numa_topology_type == NUMA_BACKPLANE && 1315 dist >= maxdist) 1316 continue; 1317 1318 /* Add up the faults from nearby nodes. */ 1319 if (task) 1320 faults = task_faults(p, node); 1321 else 1322 faults = group_faults(p, node); 1323 1324 /* 1325 * On systems with a glueless mesh NUMA topology, there are 1326 * no fixed "groups of nodes". Instead, nodes that are not 1327 * directly connected bounce traffic through intermediate 1328 * nodes; a numa_group can occupy any set of nodes. 1329 * The further away a node is, the less the faults count. 1330 * This seems to result in good task placement. 1331 */ 1332 if (sched_numa_topology_type == NUMA_GLUELESS_MESH) { 1333 faults *= (sched_max_numa_distance - dist); 1334 faults /= (sched_max_numa_distance - LOCAL_DISTANCE); 1335 } 1336 1337 score += faults; 1338 } 1339 1340 return score; 1341 } 1342 1343 /* 1344 * These return the fraction of accesses done by a particular task, or 1345 * task group, on a particular numa node. The group weight is given a 1346 * larger multiplier, in order to group tasks together that are almost 1347 * evenly spread out between numa nodes. 1348 */ 1349 static inline unsigned long task_weight(struct task_struct *p, int nid, 1350 int dist) 1351 { 1352 unsigned long faults, total_faults; 1353 1354 if (!p->numa_faults) 1355 return 0; 1356 1357 total_faults = p->total_numa_faults; 1358 1359 if (!total_faults) 1360 return 0; 1361 1362 faults = task_faults(p, nid); 1363 faults += score_nearby_nodes(p, nid, dist, true); 1364 1365 return 1000 * faults / total_faults; 1366 } 1367 1368 static inline unsigned long group_weight(struct task_struct *p, int nid, 1369 int dist) 1370 { 1371 unsigned long faults, total_faults; 1372 1373 if (!p->numa_group) 1374 return 0; 1375 1376 total_faults = p->numa_group->total_faults; 1377 1378 if (!total_faults) 1379 return 0; 1380 1381 faults = group_faults(p, nid); 1382 faults += score_nearby_nodes(p, nid, dist, false); 1383 1384 return 1000 * faults / total_faults; 1385 } 1386 1387 bool should_numa_migrate_memory(struct task_struct *p, struct page * page, 1388 int src_nid, int dst_cpu) 1389 { 1390 struct numa_group *ng = p->numa_group; 1391 int dst_nid = cpu_to_node(dst_cpu); 1392 int last_cpupid, this_cpupid; 1393 1394 this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid); 1395 last_cpupid = page_cpupid_xchg_last(page, this_cpupid); 1396 1397 /* 1398 * Allow first faults or private faults to migrate immediately early in 1399 * the lifetime of a task. The magic number 4 is based on waiting for 1400 * two full passes of the "multi-stage node selection" test that is 1401 * executed below. 1402 */ 1403 if ((p->numa_preferred_nid == -1 || p->numa_scan_seq <= 4) && 1404 (cpupid_pid_unset(last_cpupid) || cpupid_match_pid(p, last_cpupid))) 1405 return true; 1406 1407 /* 1408 * Multi-stage node selection is used in conjunction with a periodic 1409 * migration fault to build a temporal task<->page relation. By using 1410 * a two-stage filter we remove short/unlikely relations. 1411 * 1412 * Using P(p) ~ n_p / n_t as per frequentist probability, we can equate 1413 * a task's usage of a particular page (n_p) per total usage of this 1414 * page (n_t) (in a given time-span) to a probability. 1415 * 1416 * Our periodic faults will sample this probability and getting the 1417 * same result twice in a row, given these samples are fully 1418 * independent, is then given by P(n)^2, provided our sample period 1419 * is sufficiently short compared to the usage pattern. 1420 * 1421 * This quadric squishes small probabilities, making it less likely we 1422 * act on an unlikely task<->page relation. 1423 */ 1424 if (!cpupid_pid_unset(last_cpupid) && 1425 cpupid_to_nid(last_cpupid) != dst_nid) 1426 return false; 1427 1428 /* Always allow migrate on private faults */ 1429 if (cpupid_match_pid(p, last_cpupid)) 1430 return true; 1431 1432 /* A shared fault, but p->numa_group has not been set up yet. */ 1433 if (!ng) 1434 return true; 1435 1436 /* 1437 * Destination node is much more heavily used than the source 1438 * node? Allow migration. 1439 */ 1440 if (group_faults_cpu(ng, dst_nid) > group_faults_cpu(ng, src_nid) * 1441 ACTIVE_NODE_FRACTION) 1442 return true; 1443 1444 /* 1445 * Distribute memory according to CPU & memory use on each node, 1446 * with 3/4 hysteresis to avoid unnecessary memory migrations: 1447 * 1448 * faults_cpu(dst) 3 faults_cpu(src) 1449 * --------------- * - > --------------- 1450 * faults_mem(dst) 4 faults_mem(src) 1451 */ 1452 return group_faults_cpu(ng, dst_nid) * group_faults(p, src_nid) * 3 > 1453 group_faults_cpu(ng, src_nid) * group_faults(p, dst_nid) * 4; 1454 } 1455 1456 static unsigned long weighted_cpuload(struct rq *rq); 1457 static unsigned long source_load(int cpu, int type); 1458 static unsigned long target_load(int cpu, int type); 1459 1460 /* Cached statistics for all CPUs within a node */ 1461 struct numa_stats { 1462 unsigned long load; 1463 1464 /* Total compute capacity of CPUs on a node */ 1465 unsigned long compute_capacity; 1466 }; 1467 1468 /* 1469 * XXX borrowed from update_sg_lb_stats 1470 */ 1471 static void update_numa_stats(struct numa_stats *ns, int nid) 1472 { 1473 int cpu; 1474 1475 memset(ns, 0, sizeof(*ns)); 1476 for_each_cpu(cpu, cpumask_of_node(nid)) { 1477 struct rq *rq = cpu_rq(cpu); 1478 1479 ns->load += weighted_cpuload(rq); 1480 ns->compute_capacity += capacity_of(cpu); 1481 } 1482 1483 } 1484 1485 struct task_numa_env { 1486 struct task_struct *p; 1487 1488 int src_cpu, src_nid; 1489 int dst_cpu, dst_nid; 1490 1491 struct numa_stats src_stats, dst_stats; 1492 1493 int imbalance_pct; 1494 int dist; 1495 1496 struct task_struct *best_task; 1497 long best_imp; 1498 int best_cpu; 1499 }; 1500 1501 static void task_numa_assign(struct task_numa_env *env, 1502 struct task_struct *p, long imp) 1503 { 1504 struct rq *rq = cpu_rq(env->dst_cpu); 1505 1506 /* Bail out if run-queue part of active NUMA balance. */ 1507 if (xchg(&rq->numa_migrate_on, 1)) 1508 return; 1509 1510 /* 1511 * Clear previous best_cpu/rq numa-migrate flag, since task now 1512 * found a better CPU to move/swap. 1513 */ 1514 if (env->best_cpu != -1) { 1515 rq = cpu_rq(env->best_cpu); 1516 WRITE_ONCE(rq->numa_migrate_on, 0); 1517 } 1518 1519 if (env->best_task) 1520 put_task_struct(env->best_task); 1521 if (p) 1522 get_task_struct(p); 1523 1524 env->best_task = p; 1525 env->best_imp = imp; 1526 env->best_cpu = env->dst_cpu; 1527 } 1528 1529 static bool load_too_imbalanced(long src_load, long dst_load, 1530 struct task_numa_env *env) 1531 { 1532 long imb, old_imb; 1533 long orig_src_load, orig_dst_load; 1534 long src_capacity, dst_capacity; 1535 1536 /* 1537 * The load is corrected for the CPU capacity available on each node. 1538 * 1539 * src_load dst_load 1540 * ------------ vs --------- 1541 * src_capacity dst_capacity 1542 */ 1543 src_capacity = env->src_stats.compute_capacity; 1544 dst_capacity = env->dst_stats.compute_capacity; 1545 1546 imb = abs(dst_load * src_capacity - src_load * dst_capacity); 1547 1548 orig_src_load = env->src_stats.load; 1549 orig_dst_load = env->dst_stats.load; 1550 1551 old_imb = abs(orig_dst_load * src_capacity - orig_src_load * dst_capacity); 1552 1553 /* Would this change make things worse? */ 1554 return (imb > old_imb); 1555 } 1556 1557 /* 1558 * Maximum NUMA importance can be 1998 (2*999); 1559 * SMALLIMP @ 30 would be close to 1998/64. 1560 * Used to deter task migration. 1561 */ 1562 #define SMALLIMP 30 1563 1564 /* 1565 * This checks if the overall compute and NUMA accesses of the system would 1566 * be improved if the source tasks was migrated to the target dst_cpu taking 1567 * into account that it might be best if task running on the dst_cpu should 1568 * be exchanged with the source task 1569 */ 1570 static void task_numa_compare(struct task_numa_env *env, 1571 long taskimp, long groupimp, bool maymove) 1572 { 1573 struct rq *dst_rq = cpu_rq(env->dst_cpu); 1574 struct task_struct *cur; 1575 long src_load, dst_load; 1576 long load; 1577 long imp = env->p->numa_group ? groupimp : taskimp; 1578 long moveimp = imp; 1579 int dist = env->dist; 1580 1581 if (READ_ONCE(dst_rq->numa_migrate_on)) 1582 return; 1583 1584 rcu_read_lock(); 1585 cur = task_rcu_dereference(&dst_rq->curr); 1586 if (cur && ((cur->flags & PF_EXITING) || is_idle_task(cur))) 1587 cur = NULL; 1588 1589 /* 1590 * Because we have preemption enabled we can get migrated around and 1591 * end try selecting ourselves (current == env->p) as a swap candidate. 1592 */ 1593 if (cur == env->p) 1594 goto unlock; 1595 1596 if (!cur) { 1597 if (maymove && moveimp >= env->best_imp) 1598 goto assign; 1599 else 1600 goto unlock; 1601 } 1602 1603 /* 1604 * "imp" is the fault differential for the source task between the 1605 * source and destination node. Calculate the total differential for 1606 * the source task and potential destination task. The more negative 1607 * the value is, the more remote accesses that would be expected to 1608 * be incurred if the tasks were swapped. 1609 */ 1610 /* Skip this swap candidate if cannot move to the source cpu */ 1611 if (!cpumask_test_cpu(env->src_cpu, &cur->cpus_allowed)) 1612 goto unlock; 1613 1614 /* 1615 * If dst and source tasks are in the same NUMA group, or not 1616 * in any group then look only at task weights. 1617 */ 1618 if (cur->numa_group == env->p->numa_group) { 1619 imp = taskimp + task_weight(cur, env->src_nid, dist) - 1620 task_weight(cur, env->dst_nid, dist); 1621 /* 1622 * Add some hysteresis to prevent swapping the 1623 * tasks within a group over tiny differences. 1624 */ 1625 if (cur->numa_group) 1626 imp -= imp / 16; 1627 } else { 1628 /* 1629 * Compare the group weights. If a task is all by itself 1630 * (not part of a group), use the task weight instead. 1631 */ 1632 if (cur->numa_group && env->p->numa_group) 1633 imp += group_weight(cur, env->src_nid, dist) - 1634 group_weight(cur, env->dst_nid, dist); 1635 else 1636 imp += task_weight(cur, env->src_nid, dist) - 1637 task_weight(cur, env->dst_nid, dist); 1638 } 1639 1640 if (maymove && moveimp > imp && moveimp > env->best_imp) { 1641 imp = moveimp; 1642 cur = NULL; 1643 goto assign; 1644 } 1645 1646 /* 1647 * If the NUMA importance is less than SMALLIMP, 1648 * task migration might only result in ping pong 1649 * of tasks and also hurt performance due to cache 1650 * misses. 1651 */ 1652 if (imp < SMALLIMP || imp <= env->best_imp + SMALLIMP / 2) 1653 goto unlock; 1654 1655 /* 1656 * In the overloaded case, try and keep the load balanced. 1657 */ 1658 load = task_h_load(env->p) - task_h_load(cur); 1659 if (!load) 1660 goto assign; 1661 1662 dst_load = env->dst_stats.load + load; 1663 src_load = env->src_stats.load - load; 1664 1665 if (load_too_imbalanced(src_load, dst_load, env)) 1666 goto unlock; 1667 1668 assign: 1669 /* 1670 * One idle CPU per node is evaluated for a task numa move. 1671 * Call select_idle_sibling to maybe find a better one. 1672 */ 1673 if (!cur) { 1674 /* 1675 * select_idle_siblings() uses an per-CPU cpumask that 1676 * can be used from IRQ context. 1677 */ 1678 local_irq_disable(); 1679 env->dst_cpu = select_idle_sibling(env->p, env->src_cpu, 1680 env->dst_cpu); 1681 local_irq_enable(); 1682 } 1683 1684 task_numa_assign(env, cur, imp); 1685 unlock: 1686 rcu_read_unlock(); 1687 } 1688 1689 static void task_numa_find_cpu(struct task_numa_env *env, 1690 long taskimp, long groupimp) 1691 { 1692 long src_load, dst_load, load; 1693 bool maymove = false; 1694 int cpu; 1695 1696 load = task_h_load(env->p); 1697 dst_load = env->dst_stats.load + load; 1698 src_load = env->src_stats.load - load; 1699 1700 /* 1701 * If the improvement from just moving env->p direction is better 1702 * than swapping tasks around, check if a move is possible. 1703 */ 1704 maymove = !load_too_imbalanced(src_load, dst_load, env); 1705 1706 for_each_cpu(cpu, cpumask_of_node(env->dst_nid)) { 1707 /* Skip this CPU if the source task cannot migrate */ 1708 if (!cpumask_test_cpu(cpu, &env->p->cpus_allowed)) 1709 continue; 1710 1711 env->dst_cpu = cpu; 1712 task_numa_compare(env, taskimp, groupimp, maymove); 1713 } 1714 } 1715 1716 static int task_numa_migrate(struct task_struct *p) 1717 { 1718 struct task_numa_env env = { 1719 .p = p, 1720 1721 .src_cpu = task_cpu(p), 1722 .src_nid = task_node(p), 1723 1724 .imbalance_pct = 112, 1725 1726 .best_task = NULL, 1727 .best_imp = 0, 1728 .best_cpu = -1, 1729 }; 1730 struct sched_domain *sd; 1731 struct rq *best_rq; 1732 unsigned long taskweight, groupweight; 1733 int nid, ret, dist; 1734 long taskimp, groupimp; 1735 1736 /* 1737 * Pick the lowest SD_NUMA domain, as that would have the smallest 1738 * imbalance and would be the first to start moving tasks about. 1739 * 1740 * And we want to avoid any moving of tasks about, as that would create 1741 * random movement of tasks -- counter the numa conditions we're trying 1742 * to satisfy here. 1743 */ 1744 rcu_read_lock(); 1745 sd = rcu_dereference(per_cpu(sd_numa, env.src_cpu)); 1746 if (sd) 1747 env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2; 1748 rcu_read_unlock(); 1749 1750 /* 1751 * Cpusets can break the scheduler domain tree into smaller 1752 * balance domains, some of which do not cross NUMA boundaries. 1753 * Tasks that are "trapped" in such domains cannot be migrated 1754 * elsewhere, so there is no point in (re)trying. 1755 */ 1756 if (unlikely(!sd)) { 1757 sched_setnuma(p, task_node(p)); 1758 return -EINVAL; 1759 } 1760 1761 env.dst_nid = p->numa_preferred_nid; 1762 dist = env.dist = node_distance(env.src_nid, env.dst_nid); 1763 taskweight = task_weight(p, env.src_nid, dist); 1764 groupweight = group_weight(p, env.src_nid, dist); 1765 update_numa_stats(&env.src_stats, env.src_nid); 1766 taskimp = task_weight(p, env.dst_nid, dist) - taskweight; 1767 groupimp = group_weight(p, env.dst_nid, dist) - groupweight; 1768 update_numa_stats(&env.dst_stats, env.dst_nid); 1769 1770 /* Try to find a spot on the preferred nid. */ 1771 task_numa_find_cpu(&env, taskimp, groupimp); 1772 1773 /* 1774 * Look at other nodes in these cases: 1775 * - there is no space available on the preferred_nid 1776 * - the task is part of a numa_group that is interleaved across 1777 * multiple NUMA nodes; in order to better consolidate the group, 1778 * we need to check other locations. 1779 */ 1780 if (env.best_cpu == -1 || (p->numa_group && p->numa_group->active_nodes > 1)) { 1781 for_each_online_node(nid) { 1782 if (nid == env.src_nid || nid == p->numa_preferred_nid) 1783 continue; 1784 1785 dist = node_distance(env.src_nid, env.dst_nid); 1786 if (sched_numa_topology_type == NUMA_BACKPLANE && 1787 dist != env.dist) { 1788 taskweight = task_weight(p, env.src_nid, dist); 1789 groupweight = group_weight(p, env.src_nid, dist); 1790 } 1791 1792 /* Only consider nodes where both task and groups benefit */ 1793 taskimp = task_weight(p, nid, dist) - taskweight; 1794 groupimp = group_weight(p, nid, dist) - groupweight; 1795 if (taskimp < 0 && groupimp < 0) 1796 continue; 1797 1798 env.dist = dist; 1799 env.dst_nid = nid; 1800 update_numa_stats(&env.dst_stats, env.dst_nid); 1801 task_numa_find_cpu(&env, taskimp, groupimp); 1802 } 1803 } 1804 1805 /* 1806 * If the task is part of a workload that spans multiple NUMA nodes, 1807 * and is migrating into one of the workload's active nodes, remember 1808 * this node as the task's preferred numa node, so the workload can 1809 * settle down. 1810 * A task that migrated to a second choice node will be better off 1811 * trying for a better one later. Do not set the preferred node here. 1812 */ 1813 if (p->numa_group) { 1814 if (env.best_cpu == -1) 1815 nid = env.src_nid; 1816 else 1817 nid = cpu_to_node(env.best_cpu); 1818 1819 if (nid != p->numa_preferred_nid) 1820 sched_setnuma(p, nid); 1821 } 1822 1823 /* No better CPU than the current one was found. */ 1824 if (env.best_cpu == -1) 1825 return -EAGAIN; 1826 1827 best_rq = cpu_rq(env.best_cpu); 1828 if (env.best_task == NULL) { 1829 ret = migrate_task_to(p, env.best_cpu); 1830 WRITE_ONCE(best_rq->numa_migrate_on, 0); 1831 if (ret != 0) 1832 trace_sched_stick_numa(p, env.src_cpu, env.best_cpu); 1833 return ret; 1834 } 1835 1836 ret = migrate_swap(p, env.best_task, env.best_cpu, env.src_cpu); 1837 WRITE_ONCE(best_rq->numa_migrate_on, 0); 1838 1839 if (ret != 0) 1840 trace_sched_stick_numa(p, env.src_cpu, task_cpu(env.best_task)); 1841 put_task_struct(env.best_task); 1842 return ret; 1843 } 1844 1845 /* Attempt to migrate a task to a CPU on the preferred node. */ 1846 static void numa_migrate_preferred(struct task_struct *p) 1847 { 1848 unsigned long interval = HZ; 1849 1850 /* This task has no NUMA fault statistics yet */ 1851 if (unlikely(p->numa_preferred_nid == -1 || !p->numa_faults)) 1852 return; 1853 1854 /* Periodically retry migrating the task to the preferred node */ 1855 interval = min(interval, msecs_to_jiffies(p->numa_scan_period) / 16); 1856 p->numa_migrate_retry = jiffies + interval; 1857 1858 /* Success if task is already running on preferred CPU */ 1859 if (task_node(p) == p->numa_preferred_nid) 1860 return; 1861 1862 /* Otherwise, try migrate to a CPU on the preferred node */ 1863 task_numa_migrate(p); 1864 } 1865 1866 /* 1867 * Find out how many nodes on the workload is actively running on. Do this by 1868 * tracking the nodes from which NUMA hinting faults are triggered. This can 1869 * be different from the set of nodes where the workload's memory is currently 1870 * located. 1871 */ 1872 static void numa_group_count_active_nodes(struct numa_group *numa_group) 1873 { 1874 unsigned long faults, max_faults = 0; 1875 int nid, active_nodes = 0; 1876 1877 for_each_online_node(nid) { 1878 faults = group_faults_cpu(numa_group, nid); 1879 if (faults > max_faults) 1880 max_faults = faults; 1881 } 1882 1883 for_each_online_node(nid) { 1884 faults = group_faults_cpu(numa_group, nid); 1885 if (faults * ACTIVE_NODE_FRACTION > max_faults) 1886 active_nodes++; 1887 } 1888 1889 numa_group->max_faults_cpu = max_faults; 1890 numa_group->active_nodes = active_nodes; 1891 } 1892 1893 /* 1894 * When adapting the scan rate, the period is divided into NUMA_PERIOD_SLOTS 1895 * increments. The more local the fault statistics are, the higher the scan 1896 * period will be for the next scan window. If local/(local+remote) ratio is 1897 * below NUMA_PERIOD_THRESHOLD (where range of ratio is 1..NUMA_PERIOD_SLOTS) 1898 * the scan period will decrease. Aim for 70% local accesses. 1899 */ 1900 #define NUMA_PERIOD_SLOTS 10 1901 #define NUMA_PERIOD_THRESHOLD 7 1902 1903 /* 1904 * Increase the scan period (slow down scanning) if the majority of 1905 * our memory is already on our local node, or if the majority of 1906 * the page accesses are shared with other processes. 1907 * Otherwise, decrease the scan period. 1908 */ 1909 static void update_task_scan_period(struct task_struct *p, 1910 unsigned long shared, unsigned long private) 1911 { 1912 unsigned int period_slot; 1913 int lr_ratio, ps_ratio; 1914 int diff; 1915 1916 unsigned long remote = p->numa_faults_locality[0]; 1917 unsigned long local = p->numa_faults_locality[1]; 1918 1919 /* 1920 * If there were no record hinting faults then either the task is 1921 * completely idle or all activity is areas that are not of interest 1922 * to automatic numa balancing. Related to that, if there were failed 1923 * migration then it implies we are migrating too quickly or the local 1924 * node is overloaded. In either case, scan slower 1925 */ 1926 if (local + shared == 0 || p->numa_faults_locality[2]) { 1927 p->numa_scan_period = min(p->numa_scan_period_max, 1928 p->numa_scan_period << 1); 1929 1930 p->mm->numa_next_scan = jiffies + 1931 msecs_to_jiffies(p->numa_scan_period); 1932 1933 return; 1934 } 1935 1936 /* 1937 * Prepare to scale scan period relative to the current period. 1938 * == NUMA_PERIOD_THRESHOLD scan period stays the same 1939 * < NUMA_PERIOD_THRESHOLD scan period decreases (scan faster) 1940 * >= NUMA_PERIOD_THRESHOLD scan period increases (scan slower) 1941 */ 1942 period_slot = DIV_ROUND_UP(p->numa_scan_period, NUMA_PERIOD_SLOTS); 1943 lr_ratio = (local * NUMA_PERIOD_SLOTS) / (local + remote); 1944 ps_ratio = (private * NUMA_PERIOD_SLOTS) / (private + shared); 1945 1946 if (ps_ratio >= NUMA_PERIOD_THRESHOLD) { 1947 /* 1948 * Most memory accesses are local. There is no need to 1949 * do fast NUMA scanning, since memory is already local. 1950 */ 1951 int slot = ps_ratio - NUMA_PERIOD_THRESHOLD; 1952 if (!slot) 1953 slot = 1; 1954 diff = slot * period_slot; 1955 } else if (lr_ratio >= NUMA_PERIOD_THRESHOLD) { 1956 /* 1957 * Most memory accesses are shared with other tasks. 1958 * There is no point in continuing fast NUMA scanning, 1959 * since other tasks may just move the memory elsewhere. 1960 */ 1961 int slot = lr_ratio - NUMA_PERIOD_THRESHOLD; 1962 if (!slot) 1963 slot = 1; 1964 diff = slot * period_slot; 1965 } else { 1966 /* 1967 * Private memory faults exceed (SLOTS-THRESHOLD)/SLOTS, 1968 * yet they are not on the local NUMA node. Speed up 1969 * NUMA scanning to get the memory moved over. 1970 */ 1971 int ratio = max(lr_ratio, ps_ratio); 1972 diff = -(NUMA_PERIOD_THRESHOLD - ratio) * period_slot; 1973 } 1974 1975 p->numa_scan_period = clamp(p->numa_scan_period + diff, 1976 task_scan_min(p), task_scan_max(p)); 1977 memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); 1978 } 1979 1980 /* 1981 * Get the fraction of time the task has been running since the last 1982 * NUMA placement cycle. The scheduler keeps similar statistics, but 1983 * decays those on a 32ms period, which is orders of magnitude off 1984 * from the dozens-of-seconds NUMA balancing period. Use the scheduler 1985 * stats only if the task is so new there are no NUMA statistics yet. 1986 */ 1987 static u64 numa_get_avg_runtime(struct task_struct *p, u64 *period) 1988 { 1989 u64 runtime, delta, now; 1990 /* Use the start of this time slice to avoid calculations. */ 1991 now = p->se.exec_start; 1992 runtime = p->se.sum_exec_runtime; 1993 1994 if (p->last_task_numa_placement) { 1995 delta = runtime - p->last_sum_exec_runtime; 1996 *period = now - p->last_task_numa_placement; 1997 } else { 1998 delta = p->se.avg.load_sum; 1999 *period = LOAD_AVG_MAX; 2000 } 2001 2002 p->last_sum_exec_runtime = runtime; 2003 p->last_task_numa_placement = now; 2004 2005 return delta; 2006 } 2007 2008 /* 2009 * Determine the preferred nid for a task in a numa_group. This needs to 2010 * be done in a way that produces consistent results with group_weight, 2011 * otherwise workloads might not converge. 2012 */ 2013 static int preferred_group_nid(struct task_struct *p, int nid) 2014 { 2015 nodemask_t nodes; 2016 int dist; 2017 2018 /* Direct connections between all NUMA nodes. */ 2019 if (sched_numa_topology_type == NUMA_DIRECT) 2020 return nid; 2021 2022 /* 2023 * On a system with glueless mesh NUMA topology, group_weight 2024 * scores nodes according to the number of NUMA hinting faults on 2025 * both the node itself, and on nearby nodes. 2026 */ 2027 if (sched_numa_topology_type == NUMA_GLUELESS_MESH) { 2028 unsigned long score, max_score = 0; 2029 int node, max_node = nid; 2030 2031 dist = sched_max_numa_distance; 2032 2033 for_each_online_node(node) { 2034 score = group_weight(p, node, dist); 2035 if (score > max_score) { 2036 max_score = score; 2037 max_node = node; 2038 } 2039 } 2040 return max_node; 2041 } 2042 2043 /* 2044 * Finding the preferred nid in a system with NUMA backplane 2045 * interconnect topology is more involved. The goal is to locate 2046 * tasks from numa_groups near each other in the system, and 2047 * untangle workloads from different sides of the system. This requires 2048 * searching down the hierarchy of node groups, recursively searching 2049 * inside the highest scoring group of nodes. The nodemask tricks 2050 * keep the complexity of the search down. 2051 */ 2052 nodes = node_online_map; 2053 for (dist = sched_max_numa_distance; dist > LOCAL_DISTANCE; dist--) { 2054 unsigned long max_faults = 0; 2055 nodemask_t max_group = NODE_MASK_NONE; 2056 int a, b; 2057 2058 /* Are there nodes at this distance from each other? */ 2059 if (!find_numa_distance(dist)) 2060 continue; 2061 2062 for_each_node_mask(a, nodes) { 2063 unsigned long faults = 0; 2064 nodemask_t this_group; 2065 nodes_clear(this_group); 2066 2067 /* Sum group's NUMA faults; includes a==b case. */ 2068 for_each_node_mask(b, nodes) { 2069 if (node_distance(a, b) < dist) { 2070 faults += group_faults(p, b); 2071 node_set(b, this_group); 2072 node_clear(b, nodes); 2073 } 2074 } 2075 2076 /* Remember the top group. */ 2077 if (faults > max_faults) { 2078 max_faults = faults; 2079 max_group = this_group; 2080 /* 2081 * subtle: at the smallest distance there is 2082 * just one node left in each "group", the 2083 * winner is the preferred nid. 2084 */ 2085 nid = a; 2086 } 2087 } 2088 /* Next round, evaluate the nodes within max_group. */ 2089 if (!max_faults) 2090 break; 2091 nodes = max_group; 2092 } 2093 return nid; 2094 } 2095 2096 static void task_numa_placement(struct task_struct *p) 2097 { 2098 int seq, nid, max_nid = -1; 2099 unsigned long max_faults = 0; 2100 unsigned long fault_types[2] = { 0, 0 }; 2101 unsigned long total_faults; 2102 u64 runtime, period; 2103 spinlock_t *group_lock = NULL; 2104 2105 /* 2106 * The p->mm->numa_scan_seq field gets updated without 2107 * exclusive access. Use READ_ONCE() here to ensure 2108 * that the field is read in a single access: 2109 */ 2110 seq = READ_ONCE(p->mm->numa_scan_seq); 2111 if (p->numa_scan_seq == seq) 2112 return; 2113 p->numa_scan_seq = seq; 2114 p->numa_scan_period_max = task_scan_max(p); 2115 2116 total_faults = p->numa_faults_locality[0] + 2117 p->numa_faults_locality[1]; 2118 runtime = numa_get_avg_runtime(p, &period); 2119 2120 /* If the task is part of a group prevent parallel updates to group stats */ 2121 if (p->numa_group) { 2122 group_lock = &p->numa_group->lock; 2123 spin_lock_irq(group_lock); 2124 } 2125 2126 /* Find the node with the highest number of faults */ 2127 for_each_online_node(nid) { 2128 /* Keep track of the offsets in numa_faults array */ 2129 int mem_idx, membuf_idx, cpu_idx, cpubuf_idx; 2130 unsigned long faults = 0, group_faults = 0; 2131 int priv; 2132 2133 for (priv = 0; priv < NR_NUMA_HINT_FAULT_TYPES; priv++) { 2134 long diff, f_diff, f_weight; 2135 2136 mem_idx = task_faults_idx(NUMA_MEM, nid, priv); 2137 membuf_idx = task_faults_idx(NUMA_MEMBUF, nid, priv); 2138 cpu_idx = task_faults_idx(NUMA_CPU, nid, priv); 2139 cpubuf_idx = task_faults_idx(NUMA_CPUBUF, nid, priv); 2140 2141 /* Decay existing window, copy faults since last scan */ 2142 diff = p->numa_faults[membuf_idx] - p->numa_faults[mem_idx] / 2; 2143 fault_types[priv] += p->numa_faults[membuf_idx]; 2144 p->numa_faults[membuf_idx] = 0; 2145 2146 /* 2147 * Normalize the faults_from, so all tasks in a group 2148 * count according to CPU use, instead of by the raw 2149 * number of faults. Tasks with little runtime have 2150 * little over-all impact on throughput, and thus their 2151 * faults are less important. 2152 */ 2153 f_weight = div64_u64(runtime << 16, period + 1); 2154 f_weight = (f_weight * p->numa_faults[cpubuf_idx]) / 2155 (total_faults + 1); 2156 f_diff = f_weight - p->numa_faults[cpu_idx] / 2; 2157 p->numa_faults[cpubuf_idx] = 0; 2158 2159 p->numa_faults[mem_idx] += diff; 2160 p->numa_faults[cpu_idx] += f_diff; 2161 faults += p->numa_faults[mem_idx]; 2162 p->total_numa_faults += diff; 2163 if (p->numa_group) { 2164 /* 2165 * safe because we can only change our own group 2166 * 2167 * mem_idx represents the offset for a given 2168 * nid and priv in a specific region because it 2169 * is at the beginning of the numa_faults array. 2170 */ 2171 p->numa_group->faults[mem_idx] += diff; 2172 p->numa_group->faults_cpu[mem_idx] += f_diff; 2173 p->numa_group->total_faults += diff; 2174 group_faults += p->numa_group->faults[mem_idx]; 2175 } 2176 } 2177 2178 if (!p->numa_group) { 2179 if (faults > max_faults) { 2180 max_faults = faults; 2181 max_nid = nid; 2182 } 2183 } else if (group_faults > max_faults) { 2184 max_faults = group_faults; 2185 max_nid = nid; 2186 } 2187 } 2188 2189 if (p->numa_group) { 2190 numa_group_count_active_nodes(p->numa_group); 2191 spin_unlock_irq(group_lock); 2192 max_nid = preferred_group_nid(p, max_nid); 2193 } 2194 2195 if (max_faults) { 2196 /* Set the new preferred node */ 2197 if (max_nid != p->numa_preferred_nid) 2198 sched_setnuma(p, max_nid); 2199 } 2200 2201 update_task_scan_period(p, fault_types[0], fault_types[1]); 2202 } 2203 2204 static inline int get_numa_group(struct numa_group *grp) 2205 { 2206 return atomic_inc_not_zero(&grp->refcount); 2207 } 2208 2209 static inline void put_numa_group(struct numa_group *grp) 2210 { 2211 if (atomic_dec_and_test(&grp->refcount)) 2212 kfree_rcu(grp, rcu); 2213 } 2214 2215 static void task_numa_group(struct task_struct *p, int cpupid, int flags, 2216 int *priv) 2217 { 2218 struct numa_group *grp, *my_grp; 2219 struct task_struct *tsk; 2220 bool join = false; 2221 int cpu = cpupid_to_cpu(cpupid); 2222 int i; 2223 2224 if (unlikely(!p->numa_group)) { 2225 unsigned int size = sizeof(struct numa_group) + 2226 4*nr_node_ids*sizeof(unsigned long); 2227 2228 grp = kzalloc(size, GFP_KERNEL | __GFP_NOWARN); 2229 if (!grp) 2230 return; 2231 2232 atomic_set(&grp->refcount, 1); 2233 grp->active_nodes = 1; 2234 grp->max_faults_cpu = 0; 2235 spin_lock_init(&grp->lock); 2236 grp->gid = p->pid; 2237 /* Second half of the array tracks nids where faults happen */ 2238 grp->faults_cpu = grp->faults + NR_NUMA_HINT_FAULT_TYPES * 2239 nr_node_ids; 2240 2241 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 2242 grp->faults[i] = p->numa_faults[i]; 2243 2244 grp->total_faults = p->total_numa_faults; 2245 2246 grp->nr_tasks++; 2247 rcu_assign_pointer(p->numa_group, grp); 2248 } 2249 2250 rcu_read_lock(); 2251 tsk = READ_ONCE(cpu_rq(cpu)->curr); 2252 2253 if (!cpupid_match_pid(tsk, cpupid)) 2254 goto no_join; 2255 2256 grp = rcu_dereference(tsk->numa_group); 2257 if (!grp) 2258 goto no_join; 2259 2260 my_grp = p->numa_group; 2261 if (grp == my_grp) 2262 goto no_join; 2263 2264 /* 2265 * Only join the other group if its bigger; if we're the bigger group, 2266 * the other task will join us. 2267 */ 2268 if (my_grp->nr_tasks > grp->nr_tasks) 2269 goto no_join; 2270 2271 /* 2272 * Tie-break on the grp address. 2273 */ 2274 if (my_grp->nr_tasks == grp->nr_tasks && my_grp > grp) 2275 goto no_join; 2276 2277 /* Always join threads in the same process. */ 2278 if (tsk->mm == current->mm) 2279 join = true; 2280 2281 /* Simple filter to avoid false positives due to PID collisions */ 2282 if (flags & TNF_SHARED) 2283 join = true; 2284 2285 /* Update priv based on whether false sharing was detected */ 2286 *priv = !join; 2287 2288 if (join && !get_numa_group(grp)) 2289 goto no_join; 2290 2291 rcu_read_unlock(); 2292 2293 if (!join) 2294 return; 2295 2296 BUG_ON(irqs_disabled()); 2297 double_lock_irq(&my_grp->lock, &grp->lock); 2298 2299 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) { 2300 my_grp->faults[i] -= p->numa_faults[i]; 2301 grp->faults[i] += p->numa_faults[i]; 2302 } 2303 my_grp->total_faults -= p->total_numa_faults; 2304 grp->total_faults += p->total_numa_faults; 2305 2306 my_grp->nr_tasks--; 2307 grp->nr_tasks++; 2308 2309 spin_unlock(&my_grp->lock); 2310 spin_unlock_irq(&grp->lock); 2311 2312 rcu_assign_pointer(p->numa_group, grp); 2313 2314 put_numa_group(my_grp); 2315 return; 2316 2317 no_join: 2318 rcu_read_unlock(); 2319 return; 2320 } 2321 2322 void task_numa_free(struct task_struct *p) 2323 { 2324 struct numa_group *grp = p->numa_group; 2325 void *numa_faults = p->numa_faults; 2326 unsigned long flags; 2327 int i; 2328 2329 if (grp) { 2330 spin_lock_irqsave(&grp->lock, flags); 2331 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 2332 grp->faults[i] -= p->numa_faults[i]; 2333 grp->total_faults -= p->total_numa_faults; 2334 2335 grp->nr_tasks--; 2336 spin_unlock_irqrestore(&grp->lock, flags); 2337 RCU_INIT_POINTER(p->numa_group, NULL); 2338 put_numa_group(grp); 2339 } 2340 2341 p->numa_faults = NULL; 2342 kfree(numa_faults); 2343 } 2344 2345 /* 2346 * Got a PROT_NONE fault for a page on @node. 2347 */ 2348 void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags) 2349 { 2350 struct task_struct *p = current; 2351 bool migrated = flags & TNF_MIGRATED; 2352 int cpu_node = task_node(current); 2353 int local = !!(flags & TNF_FAULT_LOCAL); 2354 struct numa_group *ng; 2355 int priv; 2356 2357 if (!static_branch_likely(&sched_numa_balancing)) 2358 return; 2359 2360 /* for example, ksmd faulting in a user's mm */ 2361 if (!p->mm) 2362 return; 2363 2364 /* Allocate buffer to track faults on a per-node basis */ 2365 if (unlikely(!p->numa_faults)) { 2366 int size = sizeof(*p->numa_faults) * 2367 NR_NUMA_HINT_FAULT_BUCKETS * nr_node_ids; 2368 2369 p->numa_faults = kzalloc(size, GFP_KERNEL|__GFP_NOWARN); 2370 if (!p->numa_faults) 2371 return; 2372 2373 p->total_numa_faults = 0; 2374 memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); 2375 } 2376 2377 /* 2378 * First accesses are treated as private, otherwise consider accesses 2379 * to be private if the accessing pid has not changed 2380 */ 2381 if (unlikely(last_cpupid == (-1 & LAST_CPUPID_MASK))) { 2382 priv = 1; 2383 } else { 2384 priv = cpupid_match_pid(p, last_cpupid); 2385 if (!priv && !(flags & TNF_NO_GROUP)) 2386 task_numa_group(p, last_cpupid, flags, &priv); 2387 } 2388 2389 /* 2390 * If a workload spans multiple NUMA nodes, a shared fault that 2391 * occurs wholly within the set of nodes that the workload is 2392 * actively using should be counted as local. This allows the 2393 * scan rate to slow down when a workload has settled down. 2394 */ 2395 ng = p->numa_group; 2396 if (!priv && !local && ng && ng->active_nodes > 1 && 2397 numa_is_active_node(cpu_node, ng) && 2398 numa_is_active_node(mem_node, ng)) 2399 local = 1; 2400 2401 /* 2402 * Retry to migrate task to preferred node periodically, in case it 2403 * previously failed, or the scheduler moved us. 2404 */ 2405 if (time_after(jiffies, p->numa_migrate_retry)) { 2406 task_numa_placement(p); 2407 numa_migrate_preferred(p); 2408 } 2409 2410 if (migrated) 2411 p->numa_pages_migrated += pages; 2412 if (flags & TNF_MIGRATE_FAIL) 2413 p->numa_faults_locality[2] += pages; 2414 2415 p->numa_faults[task_faults_idx(NUMA_MEMBUF, mem_node, priv)] += pages; 2416 p->numa_faults[task_faults_idx(NUMA_CPUBUF, cpu_node, priv)] += pages; 2417 p->numa_faults_locality[local] += pages; 2418 } 2419 2420 static void reset_ptenuma_scan(struct task_struct *p) 2421 { 2422 /* 2423 * We only did a read acquisition of the mmap sem, so 2424 * p->mm->numa_scan_seq is written to without exclusive access 2425 * and the update is not guaranteed to be atomic. That's not 2426 * much of an issue though, since this is just used for 2427 * statistical sampling. Use READ_ONCE/WRITE_ONCE, which are not 2428 * expensive, to avoid any form of compiler optimizations: 2429 */ 2430 WRITE_ONCE(p->mm->numa_scan_seq, READ_ONCE(p->mm->numa_scan_seq) + 1); 2431 p->mm->numa_scan_offset = 0; 2432 } 2433 2434 /* 2435 * The expensive part of numa migration is done from task_work context. 2436 * Triggered from task_tick_numa(). 2437 */ 2438 void task_numa_work(struct callback_head *work) 2439 { 2440 unsigned long migrate, next_scan, now = jiffies; 2441 struct task_struct *p = current; 2442 struct mm_struct *mm = p->mm; 2443 u64 runtime = p->se.sum_exec_runtime; 2444 struct vm_area_struct *vma; 2445 unsigned long start, end; 2446 unsigned long nr_pte_updates = 0; 2447 long pages, virtpages; 2448 2449 SCHED_WARN_ON(p != container_of(work, struct task_struct, numa_work)); 2450 2451 work->next = work; /* protect against double add */ 2452 /* 2453 * Who cares about NUMA placement when they're dying. 2454 * 2455 * NOTE: make sure not to dereference p->mm before this check, 2456 * exit_task_work() happens _after_ exit_mm() so we could be called 2457 * without p->mm even though we still had it when we enqueued this 2458 * work. 2459 */ 2460 if (p->flags & PF_EXITING) 2461 return; 2462 2463 if (!mm->numa_next_scan) { 2464 mm->numa_next_scan = now + 2465 msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 2466 } 2467 2468 /* 2469 * Enforce maximal scan/migration frequency.. 2470 */ 2471 migrate = mm->numa_next_scan; 2472 if (time_before(now, migrate)) 2473 return; 2474 2475 if (p->numa_scan_period == 0) { 2476 p->numa_scan_period_max = task_scan_max(p); 2477 p->numa_scan_period = task_scan_start(p); 2478 } 2479 2480 next_scan = now + msecs_to_jiffies(p->numa_scan_period); 2481 if (cmpxchg(&mm->numa_next_scan, migrate, next_scan) != migrate) 2482 return; 2483 2484 /* 2485 * Delay this task enough that another task of this mm will likely win 2486 * the next time around. 2487 */ 2488 p->node_stamp += 2 * TICK_NSEC; 2489 2490 start = mm->numa_scan_offset; 2491 pages = sysctl_numa_balancing_scan_size; 2492 pages <<= 20 - PAGE_SHIFT; /* MB in pages */ 2493 virtpages = pages * 8; /* Scan up to this much virtual space */ 2494 if (!pages) 2495 return; 2496 2497 2498 if (!down_read_trylock(&mm->mmap_sem)) 2499 return; 2500 vma = find_vma(mm, start); 2501 if (!vma) { 2502 reset_ptenuma_scan(p); 2503 start = 0; 2504 vma = mm->mmap; 2505 } 2506 for (; vma; vma = vma->vm_next) { 2507 if (!vma_migratable(vma) || !vma_policy_mof(vma) || 2508 is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) { 2509 continue; 2510 } 2511 2512 /* 2513 * Shared library pages mapped by multiple processes are not 2514 * migrated as it is expected they are cache replicated. Avoid 2515 * hinting faults in read-only file-backed mappings or the vdso 2516 * as migrating the pages will be of marginal benefit. 2517 */ 2518 if (!vma->vm_mm || 2519 (vma->vm_file && (vma->vm_flags & (VM_READ|VM_WRITE)) == (VM_READ))) 2520 continue; 2521 2522 /* 2523 * Skip inaccessible VMAs to avoid any confusion between 2524 * PROT_NONE and NUMA hinting ptes 2525 */ 2526 if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))) 2527 continue; 2528 2529 do { 2530 start = max(start, vma->vm_start); 2531 end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE); 2532 end = min(end, vma->vm_end); 2533 nr_pte_updates = change_prot_numa(vma, start, end); 2534 2535 /* 2536 * Try to scan sysctl_numa_balancing_size worth of 2537 * hpages that have at least one present PTE that 2538 * is not already pte-numa. If the VMA contains 2539 * areas that are unused or already full of prot_numa 2540 * PTEs, scan up to virtpages, to skip through those 2541 * areas faster. 2542 */ 2543 if (nr_pte_updates) 2544 pages -= (end - start) >> PAGE_SHIFT; 2545 virtpages -= (end - start) >> PAGE_SHIFT; 2546 2547 start = end; 2548 if (pages <= 0 || virtpages <= 0) 2549 goto out; 2550 2551 cond_resched(); 2552 } while (end != vma->vm_end); 2553 } 2554 2555 out: 2556 /* 2557 * It is possible to reach the end of the VMA list but the last few 2558 * VMAs are not guaranteed to the vma_migratable. If they are not, we 2559 * would find the !migratable VMA on the next scan but not reset the 2560 * scanner to the start so check it now. 2561 */ 2562 if (vma) 2563 mm->numa_scan_offset = start; 2564 else 2565 reset_ptenuma_scan(p); 2566 up_read(&mm->mmap_sem); 2567 2568 /* 2569 * Make sure tasks use at least 32x as much time to run other code 2570 * than they used here, to limit NUMA PTE scanning overhead to 3% max. 2571 * Usually update_task_scan_period slows down scanning enough; on an 2572 * overloaded system we need to limit overhead on a per task basis. 2573 */ 2574 if (unlikely(p->se.sum_exec_runtime != runtime)) { 2575 u64 diff = p->se.sum_exec_runtime - runtime; 2576 p->node_stamp += 32 * diff; 2577 } 2578 } 2579 2580 /* 2581 * Drive the periodic memory faults.. 2582 */ 2583 void task_tick_numa(struct rq *rq, struct task_struct *curr) 2584 { 2585 struct callback_head *work = &curr->numa_work; 2586 u64 period, now; 2587 2588 /* 2589 * We don't care about NUMA placement if we don't have memory. 2590 */ 2591 if (!curr->mm || (curr->flags & PF_EXITING) || work->next != work) 2592 return; 2593 2594 /* 2595 * Using runtime rather than walltime has the dual advantage that 2596 * we (mostly) drive the selection from busy threads and that the 2597 * task needs to have done some actual work before we bother with 2598 * NUMA placement. 2599 */ 2600 now = curr->se.sum_exec_runtime; 2601 period = (u64)curr->numa_scan_period * NSEC_PER_MSEC; 2602 2603 if (now > curr->node_stamp + period) { 2604 if (!curr->node_stamp) 2605 curr->numa_scan_period = task_scan_start(curr); 2606 curr->node_stamp += period; 2607 2608 if (!time_before(jiffies, curr->mm->numa_next_scan)) { 2609 init_task_work(work, task_numa_work); /* TODO: move this into sched_fork() */ 2610 task_work_add(curr, work, true); 2611 } 2612 } 2613 } 2614 2615 static void update_scan_period(struct task_struct *p, int new_cpu) 2616 { 2617 int src_nid = cpu_to_node(task_cpu(p)); 2618 int dst_nid = cpu_to_node(new_cpu); 2619 2620 if (!static_branch_likely(&sched_numa_balancing)) 2621 return; 2622 2623 if (!p->mm || !p->numa_faults || (p->flags & PF_EXITING)) 2624 return; 2625 2626 if (src_nid == dst_nid) 2627 return; 2628 2629 /* 2630 * Allow resets if faults have been trapped before one scan 2631 * has completed. This is most likely due to a new task that 2632 * is pulled cross-node due to wakeups or load balancing. 2633 */ 2634 if (p->numa_scan_seq) { 2635 /* 2636 * Avoid scan adjustments if moving to the preferred 2637 * node or if the task was not previously running on 2638 * the preferred node. 2639 */ 2640 if (dst_nid == p->numa_preferred_nid || 2641 (p->numa_preferred_nid != -1 && src_nid != p->numa_preferred_nid)) 2642 return; 2643 } 2644 2645 p->numa_scan_period = task_scan_start(p); 2646 } 2647 2648 #else 2649 static void task_tick_numa(struct rq *rq, struct task_struct *curr) 2650 { 2651 } 2652 2653 static inline void account_numa_enqueue(struct rq *rq, struct task_struct *p) 2654 { 2655 } 2656 2657 static inline void account_numa_dequeue(struct rq *rq, struct task_struct *p) 2658 { 2659 } 2660 2661 static inline void update_scan_period(struct task_struct *p, int new_cpu) 2662 { 2663 } 2664 2665 #endif /* CONFIG_NUMA_BALANCING */ 2666 2667 static void 2668 account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) 2669 { 2670 update_load_add(&cfs_rq->load, se->load.weight); 2671 if (!parent_entity(se)) 2672 update_load_add(&rq_of(cfs_rq)->load, se->load.weight); 2673 #ifdef CONFIG_SMP 2674 if (entity_is_task(se)) { 2675 struct rq *rq = rq_of(cfs_rq); 2676 2677 account_numa_enqueue(rq, task_of(se)); 2678 list_add(&se->group_node, &rq->cfs_tasks); 2679 } 2680 #endif 2681 cfs_rq->nr_running++; 2682 } 2683 2684 static void 2685 account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) 2686 { 2687 update_load_sub(&cfs_rq->load, se->load.weight); 2688 if (!parent_entity(se)) 2689 update_load_sub(&rq_of(cfs_rq)->load, se->load.weight); 2690 #ifdef CONFIG_SMP 2691 if (entity_is_task(se)) { 2692 account_numa_dequeue(rq_of(cfs_rq), task_of(se)); 2693 list_del_init(&se->group_node); 2694 } 2695 #endif 2696 cfs_rq->nr_running--; 2697 } 2698 2699 /* 2700 * Signed add and clamp on underflow. 2701 * 2702 * Explicitly do a load-store to ensure the intermediate value never hits 2703 * memory. This allows lockless observations without ever seeing the negative 2704 * values. 2705 */ 2706 #define add_positive(_ptr, _val) do { \ 2707 typeof(_ptr) ptr = (_ptr); \ 2708 typeof(_val) val = (_val); \ 2709 typeof(*ptr) res, var = READ_ONCE(*ptr); \ 2710 \ 2711 res = var + val; \ 2712 \ 2713 if (val < 0 && res > var) \ 2714 res = 0; \ 2715 \ 2716 WRITE_ONCE(*ptr, res); \ 2717 } while (0) 2718 2719 /* 2720 * Unsigned subtract and clamp on underflow. 2721 * 2722 * Explicitly do a load-store to ensure the intermediate value never hits 2723 * memory. This allows lockless observations without ever seeing the negative 2724 * values. 2725 */ 2726 #define sub_positive(_ptr, _val) do { \ 2727 typeof(_ptr) ptr = (_ptr); \ 2728 typeof(*ptr) val = (_val); \ 2729 typeof(*ptr) res, var = READ_ONCE(*ptr); \ 2730 res = var - val; \ 2731 if (res > var) \ 2732 res = 0; \ 2733 WRITE_ONCE(*ptr, res); \ 2734 } while (0) 2735 2736 /* 2737 * Remove and clamp on negative, from a local variable. 2738 * 2739 * A variant of sub_positive(), which does not use explicit load-store 2740 * and is thus optimized for local variable updates. 2741 */ 2742 #define lsub_positive(_ptr, _val) do { \ 2743 typeof(_ptr) ptr = (_ptr); \ 2744 *ptr -= min_t(typeof(*ptr), *ptr, _val); \ 2745 } while (0) 2746 2747 #ifdef CONFIG_SMP 2748 static inline void 2749 enqueue_runnable_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 2750 { 2751 cfs_rq->runnable_weight += se->runnable_weight; 2752 2753 cfs_rq->avg.runnable_load_avg += se->avg.runnable_load_avg; 2754 cfs_rq->avg.runnable_load_sum += se_runnable(se) * se->avg.runnable_load_sum; 2755 } 2756 2757 static inline void 2758 dequeue_runnable_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 2759 { 2760 cfs_rq->runnable_weight -= se->runnable_weight; 2761 2762 sub_positive(&cfs_rq->avg.runnable_load_avg, se->avg.runnable_load_avg); 2763 sub_positive(&cfs_rq->avg.runnable_load_sum, 2764 se_runnable(se) * se->avg.runnable_load_sum); 2765 } 2766 2767 static inline void 2768 enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 2769 { 2770 cfs_rq->avg.load_avg += se->avg.load_avg; 2771 cfs_rq->avg.load_sum += se_weight(se) * se->avg.load_sum; 2772 } 2773 2774 static inline void 2775 dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 2776 { 2777 sub_positive(&cfs_rq->avg.load_avg, se->avg.load_avg); 2778 sub_positive(&cfs_rq->avg.load_sum, se_weight(se) * se->avg.load_sum); 2779 } 2780 #else 2781 static inline void 2782 enqueue_runnable_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { } 2783 static inline void 2784 dequeue_runnable_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { } 2785 static inline void 2786 enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { } 2787 static inline void 2788 dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { } 2789 #endif 2790 2791 static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, 2792 unsigned long weight, unsigned long runnable) 2793 { 2794 if (se->on_rq) { 2795 /* commit outstanding execution time */ 2796 if (cfs_rq->curr == se) 2797 update_curr(cfs_rq); 2798 account_entity_dequeue(cfs_rq, se); 2799 dequeue_runnable_load_avg(cfs_rq, se); 2800 } 2801 dequeue_load_avg(cfs_rq, se); 2802 2803 se->runnable_weight = runnable; 2804 update_load_set(&se->load, weight); 2805 2806 #ifdef CONFIG_SMP 2807 do { 2808 u32 divider = LOAD_AVG_MAX - 1024 + se->avg.period_contrib; 2809 2810 se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider); 2811 se->avg.runnable_load_avg = 2812 div_u64(se_runnable(se) * se->avg.runnable_load_sum, divider); 2813 } while (0); 2814 #endif 2815 2816 enqueue_load_avg(cfs_rq, se); 2817 if (se->on_rq) { 2818 account_entity_enqueue(cfs_rq, se); 2819 enqueue_runnable_load_avg(cfs_rq, se); 2820 } 2821 } 2822 2823 void reweight_task(struct task_struct *p, int prio) 2824 { 2825 struct sched_entity *se = &p->se; 2826 struct cfs_rq *cfs_rq = cfs_rq_of(se); 2827 struct load_weight *load = &se->load; 2828 unsigned long weight = scale_load(sched_prio_to_weight[prio]); 2829 2830 reweight_entity(cfs_rq, se, weight, weight); 2831 load->inv_weight = sched_prio_to_wmult[prio]; 2832 } 2833 2834 #ifdef CONFIG_FAIR_GROUP_SCHED 2835 #ifdef CONFIG_SMP 2836 /* 2837 * All this does is approximate the hierarchical proportion which includes that 2838 * global sum we all love to hate. 2839 * 2840 * That is, the weight of a group entity, is the proportional share of the 2841 * group weight based on the group runqueue weights. That is: 2842 * 2843 * tg->weight * grq->load.weight 2844 * ge->load.weight = ----------------------------- (1) 2845 * \Sum grq->load.weight 2846 * 2847 * Now, because computing that sum is prohibitively expensive to compute (been 2848 * there, done that) we approximate it with this average stuff. The average 2849 * moves slower and therefore the approximation is cheaper and more stable. 2850 * 2851 * So instead of the above, we substitute: 2852 * 2853 * grq->load.weight -> grq->avg.load_avg (2) 2854 * 2855 * which yields the following: 2856 * 2857 * tg->weight * grq->avg.load_avg 2858 * ge->load.weight = ------------------------------ (3) 2859 * tg->load_avg 2860 * 2861 * Where: tg->load_avg ~= \Sum grq->avg.load_avg 2862 * 2863 * That is shares_avg, and it is right (given the approximation (2)). 2864 * 2865 * The problem with it is that because the average is slow -- it was designed 2866 * to be exactly that of course -- this leads to transients in boundary 2867 * conditions. In specific, the case where the group was idle and we start the 2868 * one task. It takes time for our CPU's grq->avg.load_avg to build up, 2869 * yielding bad latency etc.. 2870 * 2871 * Now, in that special case (1) reduces to: 2872 * 2873 * tg->weight * grq->load.weight 2874 * ge->load.weight = ----------------------------- = tg->weight (4) 2875 * grp->load.weight 2876 * 2877 * That is, the sum collapses because all other CPUs are idle; the UP scenario. 2878 * 2879 * So what we do is modify our approximation (3) to approach (4) in the (near) 2880 * UP case, like: 2881 * 2882 * ge->load.weight = 2883 * 2884 * tg->weight * grq->load.weight 2885 * --------------------------------------------------- (5) 2886 * tg->load_avg - grq->avg.load_avg + grq->load.weight 2887 * 2888 * But because grq->load.weight can drop to 0, resulting in a divide by zero, 2889 * we need to use grq->avg.load_avg as its lower bound, which then gives: 2890 * 2891 * 2892 * tg->weight * grq->load.weight 2893 * ge->load.weight = ----------------------------- (6) 2894 * tg_load_avg' 2895 * 2896 * Where: 2897 * 2898 * tg_load_avg' = tg->load_avg - grq->avg.load_avg + 2899 * max(grq->load.weight, grq->avg.load_avg) 2900 * 2901 * And that is shares_weight and is icky. In the (near) UP case it approaches 2902 * (4) while in the normal case it approaches (3). It consistently 2903 * overestimates the ge->load.weight and therefore: 2904 * 2905 * \Sum ge->load.weight >= tg->weight 2906 * 2907 * hence icky! 2908 */ 2909 static long calc_group_shares(struct cfs_rq *cfs_rq) 2910 { 2911 long tg_weight, tg_shares, load, shares; 2912 struct task_group *tg = cfs_rq->tg; 2913 2914 tg_shares = READ_ONCE(tg->shares); 2915 2916 load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg); 2917 2918 tg_weight = atomic_long_read(&tg->load_avg); 2919 2920 /* Ensure tg_weight >= load */ 2921 tg_weight -= cfs_rq->tg_load_avg_contrib; 2922 tg_weight += load; 2923 2924 shares = (tg_shares * load); 2925 if (tg_weight) 2926 shares /= tg_weight; 2927 2928 /* 2929 * MIN_SHARES has to be unscaled here to support per-CPU partitioning 2930 * of a group with small tg->shares value. It is a floor value which is 2931 * assigned as a minimum load.weight to the sched_entity representing 2932 * the group on a CPU. 2933 * 2934 * E.g. on 64-bit for a group with tg->shares of scale_load(15)=15*1024 2935 * on an 8-core system with 8 tasks each runnable on one CPU shares has 2936 * to be 15*1024*1/8=1920 instead of scale_load(MIN_SHARES)=2*1024. In 2937 * case no task is runnable on a CPU MIN_SHARES=2 should be returned 2938 * instead of 0. 2939 */ 2940 return clamp_t(long, shares, MIN_SHARES, tg_shares); 2941 } 2942 2943 /* 2944 * This calculates the effective runnable weight for a group entity based on 2945 * the group entity weight calculated above. 2946 * 2947 * Because of the above approximation (2), our group entity weight is 2948 * an load_avg based ratio (3). This means that it includes blocked load and 2949 * does not represent the runnable weight. 2950 * 2951 * Approximate the group entity's runnable weight per ratio from the group 2952 * runqueue: 2953 * 2954 * grq->avg.runnable_load_avg 2955 * ge->runnable_weight = ge->load.weight * -------------------------- (7) 2956 * grq->avg.load_avg 2957 * 2958 * However, analogous to above, since the avg numbers are slow, this leads to 2959 * transients in the from-idle case. Instead we use: 2960 * 2961 * ge->runnable_weight = ge->load.weight * 2962 * 2963 * max(grq->avg.runnable_load_avg, grq->runnable_weight) 2964 * ----------------------------------------------------- (8) 2965 * max(grq->avg.load_avg, grq->load.weight) 2966 * 2967 * Where these max() serve both to use the 'instant' values to fix the slow 2968 * from-idle and avoid the /0 on to-idle, similar to (6). 2969 */ 2970 static long calc_group_runnable(struct cfs_rq *cfs_rq, long shares) 2971 { 2972 long runnable, load_avg; 2973 2974 load_avg = max(cfs_rq->avg.load_avg, 2975 scale_load_down(cfs_rq->load.weight)); 2976 2977 runnable = max(cfs_rq->avg.runnable_load_avg, 2978 scale_load_down(cfs_rq->runnable_weight)); 2979 2980 runnable *= shares; 2981 if (load_avg) 2982 runnable /= load_avg; 2983 2984 return clamp_t(long, runnable, MIN_SHARES, shares); 2985 } 2986 #endif /* CONFIG_SMP */ 2987 2988 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); 2989 2990 /* 2991 * Recomputes the group entity based on the current state of its group 2992 * runqueue. 2993 */ 2994 static void update_cfs_group(struct sched_entity *se) 2995 { 2996 struct cfs_rq *gcfs_rq = group_cfs_rq(se); 2997 long shares, runnable; 2998 2999 if (!gcfs_rq) 3000 return; 3001 3002 if (throttled_hierarchy(gcfs_rq)) 3003 return; 3004 3005 #ifndef CONFIG_SMP 3006 runnable = shares = READ_ONCE(gcfs_rq->tg->shares); 3007 3008 if (likely(se->load.weight == shares)) 3009 return; 3010 #else 3011 shares = calc_group_shares(gcfs_rq); 3012 runnable = calc_group_runnable(gcfs_rq, shares); 3013 #endif 3014 3015 reweight_entity(cfs_rq_of(se), se, shares, runnable); 3016 } 3017 3018 #else /* CONFIG_FAIR_GROUP_SCHED */ 3019 static inline void update_cfs_group(struct sched_entity *se) 3020 { 3021 } 3022 #endif /* CONFIG_FAIR_GROUP_SCHED */ 3023 3024 static inline void cfs_rq_util_change(struct cfs_rq *cfs_rq, int flags) 3025 { 3026 struct rq *rq = rq_of(cfs_rq); 3027 3028 if (&rq->cfs == cfs_rq || (flags & SCHED_CPUFREQ_MIGRATION)) { 3029 /* 3030 * There are a few boundary cases this might miss but it should 3031 * get called often enough that that should (hopefully) not be 3032 * a real problem. 3033 * 3034 * It will not get called when we go idle, because the idle 3035 * thread is a different class (!fair), nor will the utilization 3036 * number include things like RT tasks. 3037 * 3038 * As is, the util number is not freq-invariant (we'd have to 3039 * implement arch_scale_freq_capacity() for that). 3040 * 3041 * See cpu_util(). 3042 */ 3043 cpufreq_update_util(rq, flags); 3044 } 3045 } 3046 3047 #ifdef CONFIG_SMP 3048 #ifdef CONFIG_FAIR_GROUP_SCHED 3049 /** 3050 * update_tg_load_avg - update the tg's load avg 3051 * @cfs_rq: the cfs_rq whose avg changed 3052 * @force: update regardless of how small the difference 3053 * 3054 * This function 'ensures': tg->load_avg := \Sum tg->cfs_rq[]->avg.load. 3055 * However, because tg->load_avg is a global value there are performance 3056 * considerations. 3057 * 3058 * In order to avoid having to look at the other cfs_rq's, we use a 3059 * differential update where we store the last value we propagated. This in 3060 * turn allows skipping updates if the differential is 'small'. 3061 * 3062 * Updating tg's load_avg is necessary before update_cfs_share(). 3063 */ 3064 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq, int force) 3065 { 3066 long delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; 3067 3068 /* 3069 * No need to update load_avg for root_task_group as it is not used. 3070 */ 3071 if (cfs_rq->tg == &root_task_group) 3072 return; 3073 3074 if (force || abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { 3075 atomic_long_add(delta, &cfs_rq->tg->load_avg); 3076 cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; 3077 } 3078 } 3079 3080 /* 3081 * Called within set_task_rq() right before setting a task's CPU. The 3082 * caller only guarantees p->pi_lock is held; no other assumptions, 3083 * including the state of rq->lock, should be made. 3084 */ 3085 void set_task_rq_fair(struct sched_entity *se, 3086 struct cfs_rq *prev, struct cfs_rq *next) 3087 { 3088 u64 p_last_update_time; 3089 u64 n_last_update_time; 3090 3091 if (!sched_feat(ATTACH_AGE_LOAD)) 3092 return; 3093 3094 /* 3095 * We are supposed to update the task to "current" time, then its up to 3096 * date and ready to go to new CPU/cfs_rq. But we have difficulty in 3097 * getting what current time is, so simply throw away the out-of-date 3098 * time. This will result in the wakee task is less decayed, but giving 3099 * the wakee more load sounds not bad. 3100 */ 3101 if (!(se->avg.last_update_time && prev)) 3102 return; 3103 3104 #ifndef CONFIG_64BIT 3105 { 3106 u64 p_last_update_time_copy; 3107 u64 n_last_update_time_copy; 3108 3109 do { 3110 p_last_update_time_copy = prev->load_last_update_time_copy; 3111 n_last_update_time_copy = next->load_last_update_time_copy; 3112 3113 smp_rmb(); 3114 3115 p_last_update_time = prev->avg.last_update_time; 3116 n_last_update_time = next->avg.last_update_time; 3117 3118 } while (p_last_update_time != p_last_update_time_copy || 3119 n_last_update_time != n_last_update_time_copy); 3120 } 3121 #else 3122 p_last_update_time = prev->avg.last_update_time; 3123 n_last_update_time = next->avg.last_update_time; 3124 #endif 3125 __update_load_avg_blocked_se(p_last_update_time, cpu_of(rq_of(prev)), se); 3126 se->avg.last_update_time = n_last_update_time; 3127 } 3128 3129 3130 /* 3131 * When on migration a sched_entity joins/leaves the PELT hierarchy, we need to 3132 * propagate its contribution. The key to this propagation is the invariant 3133 * that for each group: 3134 * 3135 * ge->avg == grq->avg (1) 3136 * 3137 * _IFF_ we look at the pure running and runnable sums. Because they 3138 * represent the very same entity, just at different points in the hierarchy. 3139 * 3140 * Per the above update_tg_cfs_util() is trivial and simply copies the running 3141 * sum over (but still wrong, because the group entity and group rq do not have 3142 * their PELT windows aligned). 3143 * 3144 * However, update_tg_cfs_runnable() is more complex. So we have: 3145 * 3146 * ge->avg.load_avg = ge->load.weight * ge->avg.runnable_avg (2) 3147 * 3148 * And since, like util, the runnable part should be directly transferable, 3149 * the following would _appear_ to be the straight forward approach: 3150 * 3151 * grq->avg.load_avg = grq->load.weight * grq->avg.runnable_avg (3) 3152 * 3153 * And per (1) we have: 3154 * 3155 * ge->avg.runnable_avg == grq->avg.runnable_avg 3156 * 3157 * Which gives: 3158 * 3159 * ge->load.weight * grq->avg.load_avg 3160 * ge->avg.load_avg = ----------------------------------- (4) 3161 * grq->load.weight 3162 * 3163 * Except that is wrong! 3164 * 3165 * Because while for entities historical weight is not important and we 3166 * really only care about our future and therefore can consider a pure 3167 * runnable sum, runqueues can NOT do this. 3168 * 3169 * We specifically want runqueues to have a load_avg that includes 3170 * historical weights. Those represent the blocked load, the load we expect 3171 * to (shortly) return to us. This only works by keeping the weights as 3172 * integral part of the sum. We therefore cannot decompose as per (3). 3173 * 3174 * Another reason this doesn't work is that runnable isn't a 0-sum entity. 3175 * Imagine a rq with 2 tasks that each are runnable 2/3 of the time. Then the 3176 * rq itself is runnable anywhere between 2/3 and 1 depending on how the 3177 * runnable section of these tasks overlap (or not). If they were to perfectly 3178 * align the rq as a whole would be runnable 2/3 of the time. If however we 3179 * always have at least 1 runnable task, the rq as a whole is always runnable. 3180 * 3181 * So we'll have to approximate.. :/ 3182 * 3183 * Given the constraint: 3184 * 3185 * ge->avg.running_sum <= ge->avg.runnable_sum <= LOAD_AVG_MAX 3186 * 3187 * We can construct a rule that adds runnable to a rq by assuming minimal 3188 * overlap. 3189 * 3190 * On removal, we'll assume each task is equally runnable; which yields: 3191 * 3192 * grq->avg.runnable_sum = grq->avg.load_sum / grq->load.weight 3193 * 3194 * XXX: only do this for the part of runnable > running ? 3195 * 3196 */ 3197 3198 static inline void 3199 update_tg_cfs_util(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 3200 { 3201 long delta = gcfs_rq->avg.util_avg - se->avg.util_avg; 3202 3203 /* Nothing to update */ 3204 if (!delta) 3205 return; 3206 3207 /* 3208 * The relation between sum and avg is: 3209 * 3210 * LOAD_AVG_MAX - 1024 + sa->period_contrib 3211 * 3212 * however, the PELT windows are not aligned between grq and gse. 3213 */ 3214 3215 /* Set new sched_entity's utilization */ 3216 se->avg.util_avg = gcfs_rq->avg.util_avg; 3217 se->avg.util_sum = se->avg.util_avg * LOAD_AVG_MAX; 3218 3219 /* Update parent cfs_rq utilization */ 3220 add_positive(&cfs_rq->avg.util_avg, delta); 3221 cfs_rq->avg.util_sum = cfs_rq->avg.util_avg * LOAD_AVG_MAX; 3222 } 3223 3224 static inline void 3225 update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 3226 { 3227 long delta_avg, running_sum, runnable_sum = gcfs_rq->prop_runnable_sum; 3228 unsigned long runnable_load_avg, load_avg; 3229 u64 runnable_load_sum, load_sum = 0; 3230 s64 delta_sum; 3231 3232 if (!runnable_sum) 3233 return; 3234 3235 gcfs_rq->prop_runnable_sum = 0; 3236 3237 if (runnable_sum >= 0) { 3238 /* 3239 * Add runnable; clip at LOAD_AVG_MAX. Reflects that until 3240 * the CPU is saturated running == runnable. 3241 */ 3242 runnable_sum += se->avg.load_sum; 3243 runnable_sum = min(runnable_sum, (long)LOAD_AVG_MAX); 3244 } else { 3245 /* 3246 * Estimate the new unweighted runnable_sum of the gcfs_rq by 3247 * assuming all tasks are equally runnable. 3248 */ 3249 if (scale_load_down(gcfs_rq->load.weight)) { 3250 load_sum = div_s64(gcfs_rq->avg.load_sum, 3251 scale_load_down(gcfs_rq->load.weight)); 3252 } 3253 3254 /* But make sure to not inflate se's runnable */ 3255 runnable_sum = min(se->avg.load_sum, load_sum); 3256 } 3257 3258 /* 3259 * runnable_sum can't be lower than running_sum 3260 * As running sum is scale with CPU capacity wehreas the runnable sum 3261 * is not we rescale running_sum 1st 3262 */ 3263 running_sum = se->avg.util_sum / 3264 arch_scale_cpu_capacity(NULL, cpu_of(rq_of(cfs_rq))); 3265 runnable_sum = max(runnable_sum, running_sum); 3266 3267 load_sum = (s64)se_weight(se) * runnable_sum; 3268 load_avg = div_s64(load_sum, LOAD_AVG_MAX); 3269 3270 delta_sum = load_sum - (s64)se_weight(se) * se->avg.load_sum; 3271 delta_avg = load_avg - se->avg.load_avg; 3272 3273 se->avg.load_sum = runnable_sum; 3274 se->avg.load_avg = load_avg; 3275 add_positive(&cfs_rq->avg.load_avg, delta_avg); 3276 add_positive(&cfs_rq->avg.load_sum, delta_sum); 3277 3278 runnable_load_sum = (s64)se_runnable(se) * runnable_sum; 3279 runnable_load_avg = div_s64(runnable_load_sum, LOAD_AVG_MAX); 3280 delta_sum = runnable_load_sum - se_weight(se) * se->avg.runnable_load_sum; 3281 delta_avg = runnable_load_avg - se->avg.runnable_load_avg; 3282 3283 se->avg.runnable_load_sum = runnable_sum; 3284 se->avg.runnable_load_avg = runnable_load_avg; 3285 3286 if (se->on_rq) { 3287 add_positive(&cfs_rq->avg.runnable_load_avg, delta_avg); 3288 add_positive(&cfs_rq->avg.runnable_load_sum, delta_sum); 3289 } 3290 } 3291 3292 static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) 3293 { 3294 cfs_rq->propagate = 1; 3295 cfs_rq->prop_runnable_sum += runnable_sum; 3296 } 3297 3298 /* Update task and its cfs_rq load average */ 3299 static inline int propagate_entity_load_avg(struct sched_entity *se) 3300 { 3301 struct cfs_rq *cfs_rq, *gcfs_rq; 3302 3303 if (entity_is_task(se)) 3304 return 0; 3305 3306 gcfs_rq = group_cfs_rq(se); 3307 if (!gcfs_rq->propagate) 3308 return 0; 3309 3310 gcfs_rq->propagate = 0; 3311 3312 cfs_rq = cfs_rq_of(se); 3313 3314 add_tg_cfs_propagate(cfs_rq, gcfs_rq->prop_runnable_sum); 3315 3316 update_tg_cfs_util(cfs_rq, se, gcfs_rq); 3317 update_tg_cfs_runnable(cfs_rq, se, gcfs_rq); 3318 3319 return 1; 3320 } 3321 3322 /* 3323 * Check if we need to update the load and the utilization of a blocked 3324 * group_entity: 3325 */ 3326 static inline bool skip_blocked_update(struct sched_entity *se) 3327 { 3328 struct cfs_rq *gcfs_rq = group_cfs_rq(se); 3329 3330 /* 3331 * If sched_entity still have not zero load or utilization, we have to 3332 * decay it: 3333 */ 3334 if (se->avg.load_avg || se->avg.util_avg) 3335 return false; 3336 3337 /* 3338 * If there is a pending propagation, we have to update the load and 3339 * the utilization of the sched_entity: 3340 */ 3341 if (gcfs_rq->propagate) 3342 return false; 3343 3344 /* 3345 * Otherwise, the load and the utilization of the sched_entity is 3346 * already zero and there is no pending propagation, so it will be a 3347 * waste of time to try to decay it: 3348 */ 3349 return true; 3350 } 3351 3352 #else /* CONFIG_FAIR_GROUP_SCHED */ 3353 3354 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq, int force) {} 3355 3356 static inline int propagate_entity_load_avg(struct sched_entity *se) 3357 { 3358 return 0; 3359 } 3360 3361 static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) {} 3362 3363 #endif /* CONFIG_FAIR_GROUP_SCHED */ 3364 3365 /** 3366 * update_cfs_rq_load_avg - update the cfs_rq's load/util averages 3367 * @now: current time, as per cfs_rq_clock_task() 3368 * @cfs_rq: cfs_rq to update 3369 * 3370 * The cfs_rq avg is the direct sum of all its entities (blocked and runnable) 3371 * avg. The immediate corollary is that all (fair) tasks must be attached, see 3372 * post_init_entity_util_avg(). 3373 * 3374 * cfs_rq->avg is used for task_h_load() and update_cfs_share() for example. 3375 * 3376 * Returns true if the load decayed or we removed load. 3377 * 3378 * Since both these conditions indicate a changed cfs_rq->avg.load we should 3379 * call update_tg_load_avg() when this function returns true. 3380 */ 3381 static inline int 3382 update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) 3383 { 3384 unsigned long removed_load = 0, removed_util = 0, removed_runnable_sum = 0; 3385 struct sched_avg *sa = &cfs_rq->avg; 3386 int decayed = 0; 3387 3388 if (cfs_rq->removed.nr) { 3389 unsigned long r; 3390 u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib; 3391 3392 raw_spin_lock(&cfs_rq->removed.lock); 3393 swap(cfs_rq->removed.util_avg, removed_util); 3394 swap(cfs_rq->removed.load_avg, removed_load); 3395 swap(cfs_rq->removed.runnable_sum, removed_runnable_sum); 3396 cfs_rq->removed.nr = 0; 3397 raw_spin_unlock(&cfs_rq->removed.lock); 3398 3399 r = removed_load; 3400 sub_positive(&sa->load_avg, r); 3401 sub_positive(&sa->load_sum, r * divider); 3402 3403 r = removed_util; 3404 sub_positive(&sa->util_avg, r); 3405 sub_positive(&sa->util_sum, r * divider); 3406 3407 add_tg_cfs_propagate(cfs_rq, -(long)removed_runnable_sum); 3408 3409 decayed = 1; 3410 } 3411 3412 decayed |= __update_load_avg_cfs_rq(now, cpu_of(rq_of(cfs_rq)), cfs_rq); 3413 3414 #ifndef CONFIG_64BIT 3415 smp_wmb(); 3416 cfs_rq->load_last_update_time_copy = sa->last_update_time; 3417 #endif 3418 3419 if (decayed) 3420 cfs_rq_util_change(cfs_rq, 0); 3421 3422 return decayed; 3423 } 3424 3425 /** 3426 * attach_entity_load_avg - attach this entity to its cfs_rq load avg 3427 * @cfs_rq: cfs_rq to attach to 3428 * @se: sched_entity to attach 3429 * @flags: migration hints 3430 * 3431 * Must call update_cfs_rq_load_avg() before this, since we rely on 3432 * cfs_rq->avg.last_update_time being current. 3433 */ 3434 static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 3435 { 3436 u32 divider = LOAD_AVG_MAX - 1024 + cfs_rq->avg.period_contrib; 3437 3438 /* 3439 * When we attach the @se to the @cfs_rq, we must align the decay 3440 * window because without that, really weird and wonderful things can 3441 * happen. 3442 * 3443 * XXX illustrate 3444 */ 3445 se->avg.last_update_time = cfs_rq->avg.last_update_time; 3446 se->avg.period_contrib = cfs_rq->avg.period_contrib; 3447 3448 /* 3449 * Hell(o) Nasty stuff.. we need to recompute _sum based on the new 3450 * period_contrib. This isn't strictly correct, but since we're 3451 * entirely outside of the PELT hierarchy, nobody cares if we truncate 3452 * _sum a little. 3453 */ 3454 se->avg.util_sum = se->avg.util_avg * divider; 3455 3456 se->avg.load_sum = divider; 3457 if (se_weight(se)) { 3458 se->avg.load_sum = 3459 div_u64(se->avg.load_avg * se->avg.load_sum, se_weight(se)); 3460 } 3461 3462 se->avg.runnable_load_sum = se->avg.load_sum; 3463 3464 enqueue_load_avg(cfs_rq, se); 3465 cfs_rq->avg.util_avg += se->avg.util_avg; 3466 cfs_rq->avg.util_sum += se->avg.util_sum; 3467 3468 add_tg_cfs_propagate(cfs_rq, se->avg.load_sum); 3469 3470 cfs_rq_util_change(cfs_rq, flags); 3471 } 3472 3473 /** 3474 * detach_entity_load_avg - detach this entity from its cfs_rq load avg 3475 * @cfs_rq: cfs_rq to detach from 3476 * @se: sched_entity to detach 3477 * 3478 * Must call update_cfs_rq_load_avg() before this, since we rely on 3479 * cfs_rq->avg.last_update_time being current. 3480 */ 3481 static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 3482 { 3483 dequeue_load_avg(cfs_rq, se); 3484 sub_positive(&cfs_rq->avg.util_avg, se->avg.util_avg); 3485 sub_positive(&cfs_rq->avg.util_sum, se->avg.util_sum); 3486 3487 add_tg_cfs_propagate(cfs_rq, -se->avg.load_sum); 3488 3489 cfs_rq_util_change(cfs_rq, 0); 3490 } 3491 3492 /* 3493 * Optional action to be done while updating the load average 3494 */ 3495 #define UPDATE_TG 0x1 3496 #define SKIP_AGE_LOAD 0x2 3497 #define DO_ATTACH 0x4 3498 3499 /* Update task and its cfs_rq load average */ 3500 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 3501 { 3502 u64 now = cfs_rq_clock_task(cfs_rq); 3503 struct rq *rq = rq_of(cfs_rq); 3504 int cpu = cpu_of(rq); 3505 int decayed; 3506 3507 /* 3508 * Track task load average for carrying it to new CPU after migrated, and 3509 * track group sched_entity load average for task_h_load calc in migration 3510 */ 3511 if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD)) 3512 __update_load_avg_se(now, cpu, cfs_rq, se); 3513 3514 decayed = update_cfs_rq_load_avg(now, cfs_rq); 3515 decayed |= propagate_entity_load_avg(se); 3516 3517 if (!se->avg.last_update_time && (flags & DO_ATTACH)) { 3518 3519 /* 3520 * DO_ATTACH means we're here from enqueue_entity(). 3521 * !last_update_time means we've passed through 3522 * migrate_task_rq_fair() indicating we migrated. 3523 * 3524 * IOW we're enqueueing a task on a new CPU. 3525 */ 3526 attach_entity_load_avg(cfs_rq, se, SCHED_CPUFREQ_MIGRATION); 3527 update_tg_load_avg(cfs_rq, 0); 3528 3529 } else if (decayed && (flags & UPDATE_TG)) 3530 update_tg_load_avg(cfs_rq, 0); 3531 } 3532 3533 #ifndef CONFIG_64BIT 3534 static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq) 3535 { 3536 u64 last_update_time_copy; 3537 u64 last_update_time; 3538 3539 do { 3540 last_update_time_copy = cfs_rq->load_last_update_time_copy; 3541 smp_rmb(); 3542 last_update_time = cfs_rq->avg.last_update_time; 3543 } while (last_update_time != last_update_time_copy); 3544 3545 return last_update_time; 3546 } 3547 #else 3548 static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq) 3549 { 3550 return cfs_rq->avg.last_update_time; 3551 } 3552 #endif 3553 3554 /* 3555 * Synchronize entity load avg of dequeued entity without locking 3556 * the previous rq. 3557 */ 3558 void sync_entity_load_avg(struct sched_entity *se) 3559 { 3560 struct cfs_rq *cfs_rq = cfs_rq_of(se); 3561 u64 last_update_time; 3562 3563 last_update_time = cfs_rq_last_update_time(cfs_rq); 3564 __update_load_avg_blocked_se(last_update_time, cpu_of(rq_of(cfs_rq)), se); 3565 } 3566 3567 /* 3568 * Task first catches up with cfs_rq, and then subtract 3569 * itself from the cfs_rq (task must be off the queue now). 3570 */ 3571 void remove_entity_load_avg(struct sched_entity *se) 3572 { 3573 struct cfs_rq *cfs_rq = cfs_rq_of(se); 3574 unsigned long flags; 3575 3576 /* 3577 * tasks cannot exit without having gone through wake_up_new_task() -> 3578 * post_init_entity_util_avg() which will have added things to the 3579 * cfs_rq, so we can remove unconditionally. 3580 * 3581 * Similarly for groups, they will have passed through 3582 * post_init_entity_util_avg() before unregister_sched_fair_group() 3583 * calls this. 3584 */ 3585 3586 sync_entity_load_avg(se); 3587 3588 raw_spin_lock_irqsave(&cfs_rq->removed.lock, flags); 3589 ++cfs_rq->removed.nr; 3590 cfs_rq->removed.util_avg += se->avg.util_avg; 3591 cfs_rq->removed.load_avg += se->avg.load_avg; 3592 cfs_rq->removed.runnable_sum += se->avg.load_sum; /* == runnable_sum */ 3593 raw_spin_unlock_irqrestore(&cfs_rq->removed.lock, flags); 3594 } 3595 3596 static inline unsigned long cfs_rq_runnable_load_avg(struct cfs_rq *cfs_rq) 3597 { 3598 return cfs_rq->avg.runnable_load_avg; 3599 } 3600 3601 static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq) 3602 { 3603 return cfs_rq->avg.load_avg; 3604 } 3605 3606 static int idle_balance(struct rq *this_rq, struct rq_flags *rf); 3607 3608 static inline unsigned long task_util(struct task_struct *p) 3609 { 3610 return READ_ONCE(p->se.avg.util_avg); 3611 } 3612 3613 static inline unsigned long _task_util_est(struct task_struct *p) 3614 { 3615 struct util_est ue = READ_ONCE(p->se.avg.util_est); 3616 3617 return (max(ue.ewma, ue.enqueued) | UTIL_AVG_UNCHANGED); 3618 } 3619 3620 static inline unsigned long task_util_est(struct task_struct *p) 3621 { 3622 return max(task_util(p), _task_util_est(p)); 3623 } 3624 3625 static inline void util_est_enqueue(struct cfs_rq *cfs_rq, 3626 struct task_struct *p) 3627 { 3628 unsigned int enqueued; 3629 3630 if (!sched_feat(UTIL_EST)) 3631 return; 3632 3633 /* Update root cfs_rq's estimated utilization */ 3634 enqueued = cfs_rq->avg.util_est.enqueued; 3635 enqueued += _task_util_est(p); 3636 WRITE_ONCE(cfs_rq->avg.util_est.enqueued, enqueued); 3637 } 3638 3639 /* 3640 * Check if a (signed) value is within a specified (unsigned) margin, 3641 * based on the observation that: 3642 * 3643 * abs(x) < y := (unsigned)(x + y - 1) < (2 * y - 1) 3644 * 3645 * NOTE: this only works when value + maring < INT_MAX. 3646 */ 3647 static inline bool within_margin(int value, int margin) 3648 { 3649 return ((unsigned int)(value + margin - 1) < (2 * margin - 1)); 3650 } 3651 3652 static void 3653 util_est_dequeue(struct cfs_rq *cfs_rq, struct task_struct *p, bool task_sleep) 3654 { 3655 long last_ewma_diff; 3656 struct util_est ue; 3657 3658 if (!sched_feat(UTIL_EST)) 3659 return; 3660 3661 /* Update root cfs_rq's estimated utilization */ 3662 ue.enqueued = cfs_rq->avg.util_est.enqueued; 3663 ue.enqueued -= min_t(unsigned int, ue.enqueued, _task_util_est(p)); 3664 WRITE_ONCE(cfs_rq->avg.util_est.enqueued, ue.enqueued); 3665 3666 /* 3667 * Skip update of task's estimated utilization when the task has not 3668 * yet completed an activation, e.g. being migrated. 3669 */ 3670 if (!task_sleep) 3671 return; 3672 3673 /* 3674 * If the PELT values haven't changed since enqueue time, 3675 * skip the util_est update. 3676 */ 3677 ue = p->se.avg.util_est; 3678 if (ue.enqueued & UTIL_AVG_UNCHANGED) 3679 return; 3680 3681 /* 3682 * Skip update of task's estimated utilization when its EWMA is 3683 * already ~1% close to its last activation value. 3684 */ 3685 ue.enqueued = (task_util(p) | UTIL_AVG_UNCHANGED); 3686 last_ewma_diff = ue.enqueued - ue.ewma; 3687 if (within_margin(last_ewma_diff, (SCHED_CAPACITY_SCALE / 100))) 3688 return; 3689 3690 /* 3691 * Update Task's estimated utilization 3692 * 3693 * When *p completes an activation we can consolidate another sample 3694 * of the task size. This is done by storing the current PELT value 3695 * as ue.enqueued and by using this value to update the Exponential 3696 * Weighted Moving Average (EWMA): 3697 * 3698 * ewma(t) = w * task_util(p) + (1-w) * ewma(t-1) 3699 * = w * task_util(p) + ewma(t-1) - w * ewma(t-1) 3700 * = w * (task_util(p) - ewma(t-1)) + ewma(t-1) 3701 * = w * ( last_ewma_diff ) + ewma(t-1) 3702 * = w * (last_ewma_diff + ewma(t-1) / w) 3703 * 3704 * Where 'w' is the weight of new samples, which is configured to be 3705 * 0.25, thus making w=1/4 ( >>= UTIL_EST_WEIGHT_SHIFT) 3706 */ 3707 ue.ewma <<= UTIL_EST_WEIGHT_SHIFT; 3708 ue.ewma += last_ewma_diff; 3709 ue.ewma >>= UTIL_EST_WEIGHT_SHIFT; 3710 WRITE_ONCE(p->se.avg.util_est, ue); 3711 } 3712 3713 static inline int task_fits_capacity(struct task_struct *p, long capacity) 3714 { 3715 return capacity * 1024 > task_util_est(p) * capacity_margin; 3716 } 3717 3718 static inline void update_misfit_status(struct task_struct *p, struct rq *rq) 3719 { 3720 if (!static_branch_unlikely(&sched_asym_cpucapacity)) 3721 return; 3722 3723 if (!p) { 3724 rq->misfit_task_load = 0; 3725 return; 3726 } 3727 3728 if (task_fits_capacity(p, capacity_of(cpu_of(rq)))) { 3729 rq->misfit_task_load = 0; 3730 return; 3731 } 3732 3733 rq->misfit_task_load = task_h_load(p); 3734 } 3735 3736 #else /* CONFIG_SMP */ 3737 3738 #define UPDATE_TG 0x0 3739 #define SKIP_AGE_LOAD 0x0 3740 #define DO_ATTACH 0x0 3741 3742 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int not_used1) 3743 { 3744 cfs_rq_util_change(cfs_rq, 0); 3745 } 3746 3747 static inline void remove_entity_load_avg(struct sched_entity *se) {} 3748 3749 static inline void 3750 attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) {} 3751 static inline void 3752 detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} 3753 3754 static inline int idle_balance(struct rq *rq, struct rq_flags *rf) 3755 { 3756 return 0; 3757 } 3758 3759 static inline void 3760 util_est_enqueue(struct cfs_rq *cfs_rq, struct task_struct *p) {} 3761 3762 static inline void 3763 util_est_dequeue(struct cfs_rq *cfs_rq, struct task_struct *p, 3764 bool task_sleep) {} 3765 static inline void update_misfit_status(struct task_struct *p, struct rq *rq) {} 3766 3767 #endif /* CONFIG_SMP */ 3768 3769 static void check_spread(struct cfs_rq *cfs_rq, struct sched_entity *se) 3770 { 3771 #ifdef CONFIG_SCHED_DEBUG 3772 s64 d = se->vruntime - cfs_rq->min_vruntime; 3773 3774 if (d < 0) 3775 d = -d; 3776 3777 if (d > 3*sysctl_sched_latency) 3778 schedstat_inc(cfs_rq->nr_spread_over); 3779 #endif 3780 } 3781 3782 static void 3783 place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial) 3784 { 3785 u64 vruntime = cfs_rq->min_vruntime; 3786 3787 /* 3788 * The 'current' period is already promised to the current tasks, 3789 * however the extra weight of the new task will slow them down a 3790 * little, place the new task so that it fits in the slot that 3791 * stays open at the end. 3792 */ 3793 if (initial && sched_feat(START_DEBIT)) 3794 vruntime += sched_vslice(cfs_rq, se); 3795 3796 /* sleeps up to a single latency don't count. */ 3797 if (!initial) { 3798 unsigned long thresh = sysctl_sched_latency; 3799 3800 /* 3801 * Halve their sleep time's effect, to allow 3802 * for a gentler effect of sleepers: 3803 */ 3804 if (sched_feat(GENTLE_FAIR_SLEEPERS)) 3805 thresh >>= 1; 3806 3807 vruntime -= thresh; 3808 } 3809 3810 /* ensure we never gain time by being placed backwards. */ 3811 se->vruntime = max_vruntime(se->vruntime, vruntime); 3812 } 3813 3814 static void check_enqueue_throttle(struct cfs_rq *cfs_rq); 3815 3816 static inline void check_schedstat_required(void) 3817 { 3818 #ifdef CONFIG_SCHEDSTATS 3819 if (schedstat_enabled()) 3820 return; 3821 3822 /* Force schedstat enabled if a dependent tracepoint is active */ 3823 if (trace_sched_stat_wait_enabled() || 3824 trace_sched_stat_sleep_enabled() || 3825 trace_sched_stat_iowait_enabled() || 3826 trace_sched_stat_blocked_enabled() || 3827 trace_sched_stat_runtime_enabled()) { 3828 printk_deferred_once("Scheduler tracepoints stat_sleep, stat_iowait, " 3829 "stat_blocked and stat_runtime require the " 3830 "kernel parameter schedstats=enable or " 3831 "kernel.sched_schedstats=1\n"); 3832 } 3833 #endif 3834 } 3835 3836 3837 /* 3838 * MIGRATION 3839 * 3840 * dequeue 3841 * update_curr() 3842 * update_min_vruntime() 3843 * vruntime -= min_vruntime 3844 * 3845 * enqueue 3846 * update_curr() 3847 * update_min_vruntime() 3848 * vruntime += min_vruntime 3849 * 3850 * this way the vruntime transition between RQs is done when both 3851 * min_vruntime are up-to-date. 3852 * 3853 * WAKEUP (remote) 3854 * 3855 * ->migrate_task_rq_fair() (p->state == TASK_WAKING) 3856 * vruntime -= min_vruntime 3857 * 3858 * enqueue 3859 * update_curr() 3860 * update_min_vruntime() 3861 * vruntime += min_vruntime 3862 * 3863 * this way we don't have the most up-to-date min_vruntime on the originating 3864 * CPU and an up-to-date min_vruntime on the destination CPU. 3865 */ 3866 3867 static void 3868 enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 3869 { 3870 bool renorm = !(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_MIGRATED); 3871 bool curr = cfs_rq->curr == se; 3872 3873 /* 3874 * If we're the current task, we must renormalise before calling 3875 * update_curr(). 3876 */ 3877 if (renorm && curr) 3878 se->vruntime += cfs_rq->min_vruntime; 3879 3880 update_curr(cfs_rq); 3881 3882 /* 3883 * Otherwise, renormalise after, such that we're placed at the current 3884 * moment in time, instead of some random moment in the past. Being 3885 * placed in the past could significantly boost this task to the 3886 * fairness detriment of existing tasks. 3887 */ 3888 if (renorm && !curr) 3889 se->vruntime += cfs_rq->min_vruntime; 3890 3891 /* 3892 * When enqueuing a sched_entity, we must: 3893 * - Update loads to have both entity and cfs_rq synced with now. 3894 * - Add its load to cfs_rq->runnable_avg 3895 * - For group_entity, update its weight to reflect the new share of 3896 * its group cfs_rq 3897 * - Add its new weight to cfs_rq->load.weight 3898 */ 3899 update_load_avg(cfs_rq, se, UPDATE_TG | DO_ATTACH); 3900 update_cfs_group(se); 3901 enqueue_runnable_load_avg(cfs_rq, se); 3902 account_entity_enqueue(cfs_rq, se); 3903 3904 if (flags & ENQUEUE_WAKEUP) 3905 place_entity(cfs_rq, se, 0); 3906 3907 check_schedstat_required(); 3908 update_stats_enqueue(cfs_rq, se, flags); 3909 check_spread(cfs_rq, se); 3910 if (!curr) 3911 __enqueue_entity(cfs_rq, se); 3912 se->on_rq = 1; 3913 3914 if (cfs_rq->nr_running == 1) { 3915 list_add_leaf_cfs_rq(cfs_rq); 3916 check_enqueue_throttle(cfs_rq); 3917 } 3918 } 3919 3920 static void __clear_buddies_last(struct sched_entity *se) 3921 { 3922 for_each_sched_entity(se) { 3923 struct cfs_rq *cfs_rq = cfs_rq_of(se); 3924 if (cfs_rq->last != se) 3925 break; 3926 3927 cfs_rq->last = NULL; 3928 } 3929 } 3930 3931 static void __clear_buddies_next(struct sched_entity *se) 3932 { 3933 for_each_sched_entity(se) { 3934 struct cfs_rq *cfs_rq = cfs_rq_of(se); 3935 if (cfs_rq->next != se) 3936 break; 3937 3938 cfs_rq->next = NULL; 3939 } 3940 } 3941 3942 static void __clear_buddies_skip(struct sched_entity *se) 3943 { 3944 for_each_sched_entity(se) { 3945 struct cfs_rq *cfs_rq = cfs_rq_of(se); 3946 if (cfs_rq->skip != se) 3947 break; 3948 3949 cfs_rq->skip = NULL; 3950 } 3951 } 3952 3953 static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) 3954 { 3955 if (cfs_rq->last == se) 3956 __clear_buddies_last(se); 3957 3958 if (cfs_rq->next == se) 3959 __clear_buddies_next(se); 3960 3961 if (cfs_rq->skip == se) 3962 __clear_buddies_skip(se); 3963 } 3964 3965 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq); 3966 3967 static void 3968 dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 3969 { 3970 /* 3971 * Update run-time statistics of the 'current'. 3972 */ 3973 update_curr(cfs_rq); 3974 3975 /* 3976 * When dequeuing a sched_entity, we must: 3977 * - Update loads to have both entity and cfs_rq synced with now. 3978 * - Subtract its load from the cfs_rq->runnable_avg. 3979 * - Subtract its previous weight from cfs_rq->load.weight. 3980 * - For group entity, update its weight to reflect the new share 3981 * of its group cfs_rq. 3982 */ 3983 update_load_avg(cfs_rq, se, UPDATE_TG); 3984 dequeue_runnable_load_avg(cfs_rq, se); 3985 3986 update_stats_dequeue(cfs_rq, se, flags); 3987 3988 clear_buddies(cfs_rq, se); 3989 3990 if (se != cfs_rq->curr) 3991 __dequeue_entity(cfs_rq, se); 3992 se->on_rq = 0; 3993 account_entity_dequeue(cfs_rq, se); 3994 3995 /* 3996 * Normalize after update_curr(); which will also have moved 3997 * min_vruntime if @se is the one holding it back. But before doing 3998 * update_min_vruntime() again, which will discount @se's position and 3999 * can move min_vruntime forward still more. 4000 */ 4001 if (!(flags & DEQUEUE_SLEEP)) 4002 se->vruntime -= cfs_rq->min_vruntime; 4003 4004 /* return excess runtime on last dequeue */ 4005 return_cfs_rq_runtime(cfs_rq); 4006 4007 update_cfs_group(se); 4008 4009 /* 4010 * Now advance min_vruntime if @se was the entity holding it back, 4011 * except when: DEQUEUE_SAVE && !DEQUEUE_MOVE, in this case we'll be 4012 * put back on, and if we advance min_vruntime, we'll be placed back 4013 * further than we started -- ie. we'll be penalized. 4014 */ 4015 if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) != DEQUEUE_SAVE) 4016 update_min_vruntime(cfs_rq); 4017 } 4018 4019 /* 4020 * Preempt the current task with a newly woken task if needed: 4021 */ 4022 static void 4023 check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr) 4024 { 4025 unsigned long ideal_runtime, delta_exec; 4026 struct sched_entity *se; 4027 s64 delta; 4028 4029 ideal_runtime = sched_slice(cfs_rq, curr); 4030 delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime; 4031 if (delta_exec > ideal_runtime) { 4032 resched_curr(rq_of(cfs_rq)); 4033 /* 4034 * The current task ran long enough, ensure it doesn't get 4035 * re-elected due to buddy favours. 4036 */ 4037 clear_buddies(cfs_rq, curr); 4038 return; 4039 } 4040 4041 /* 4042 * Ensure that a task that missed wakeup preemption by a 4043 * narrow margin doesn't have to wait for a full slice. 4044 * This also mitigates buddy induced latencies under load. 4045 */ 4046 if (delta_exec < sysctl_sched_min_granularity) 4047 return; 4048 4049 se = __pick_first_entity(cfs_rq); 4050 delta = curr->vruntime - se->vruntime; 4051 4052 if (delta < 0) 4053 return; 4054 4055 if (delta > ideal_runtime) 4056 resched_curr(rq_of(cfs_rq)); 4057 } 4058 4059 static void 4060 set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 4061 { 4062 /* 'current' is not kept within the tree. */ 4063 if (se->on_rq) { 4064 /* 4065 * Any task has to be enqueued before it get to execute on 4066 * a CPU. So account for the time it spent waiting on the 4067 * runqueue. 4068 */ 4069 update_stats_wait_end(cfs_rq, se); 4070 __dequeue_entity(cfs_rq, se); 4071 update_load_avg(cfs_rq, se, UPDATE_TG); 4072 } 4073 4074 update_stats_curr_start(cfs_rq, se); 4075 cfs_rq->curr = se; 4076 4077 /* 4078 * Track our maximum slice length, if the CPU's load is at 4079 * least twice that of our own weight (i.e. dont track it 4080 * when there are only lesser-weight tasks around): 4081 */ 4082 if (schedstat_enabled() && rq_of(cfs_rq)->load.weight >= 2*se->load.weight) { 4083 schedstat_set(se->statistics.slice_max, 4084 max((u64)schedstat_val(se->statistics.slice_max), 4085 se->sum_exec_runtime - se->prev_sum_exec_runtime)); 4086 } 4087 4088 se->prev_sum_exec_runtime = se->sum_exec_runtime; 4089 } 4090 4091 static int 4092 wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se); 4093 4094 /* 4095 * Pick the next process, keeping these things in mind, in this order: 4096 * 1) keep things fair between processes/task groups 4097 * 2) pick the "next" process, since someone really wants that to run 4098 * 3) pick the "last" process, for cache locality 4099 * 4) do not run the "skip" process, if something else is available 4100 */ 4101 static struct sched_entity * 4102 pick_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *curr) 4103 { 4104 struct sched_entity *left = __pick_first_entity(cfs_rq); 4105 struct sched_entity *se; 4106 4107 /* 4108 * If curr is set we have to see if its left of the leftmost entity 4109 * still in the tree, provided there was anything in the tree at all. 4110 */ 4111 if (!left || (curr && entity_before(curr, left))) 4112 left = curr; 4113 4114 se = left; /* ideally we run the leftmost entity */ 4115 4116 /* 4117 * Avoid running the skip buddy, if running something else can 4118 * be done without getting too unfair. 4119 */ 4120 if (cfs_rq->skip == se) { 4121 struct sched_entity *second; 4122 4123 if (se == curr) { 4124 second = __pick_first_entity(cfs_rq); 4125 } else { 4126 second = __pick_next_entity(se); 4127 if (!second || (curr && entity_before(curr, second))) 4128 second = curr; 4129 } 4130 4131 if (second && wakeup_preempt_entity(second, left) < 1) 4132 se = second; 4133 } 4134 4135 /* 4136 * Prefer last buddy, try to return the CPU to a preempted task. 4137 */ 4138 if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, left) < 1) 4139 se = cfs_rq->last; 4140 4141 /* 4142 * Someone really wants this to run. If it's not unfair, run it. 4143 */ 4144 if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, left) < 1) 4145 se = cfs_rq->next; 4146 4147 clear_buddies(cfs_rq, se); 4148 4149 return se; 4150 } 4151 4152 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq); 4153 4154 static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) 4155 { 4156 /* 4157 * If still on the runqueue then deactivate_task() 4158 * was not called and update_curr() has to be done: 4159 */ 4160 if (prev->on_rq) 4161 update_curr(cfs_rq); 4162 4163 /* throttle cfs_rqs exceeding runtime */ 4164 check_cfs_rq_runtime(cfs_rq); 4165 4166 check_spread(cfs_rq, prev); 4167 4168 if (prev->on_rq) { 4169 update_stats_wait_start(cfs_rq, prev); 4170 /* Put 'current' back into the tree. */ 4171 __enqueue_entity(cfs_rq, prev); 4172 /* in !on_rq case, update occurred at dequeue */ 4173 update_load_avg(cfs_rq, prev, 0); 4174 } 4175 cfs_rq->curr = NULL; 4176 } 4177 4178 static void 4179 entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) 4180 { 4181 /* 4182 * Update run-time statistics of the 'current'. 4183 */ 4184 update_curr(cfs_rq); 4185 4186 /* 4187 * Ensure that runnable average is periodically updated. 4188 */ 4189 update_load_avg(cfs_rq, curr, UPDATE_TG); 4190 update_cfs_group(curr); 4191 4192 #ifdef CONFIG_SCHED_HRTICK 4193 /* 4194 * queued ticks are scheduled to match the slice, so don't bother 4195 * validating it and just reschedule. 4196 */ 4197 if (queued) { 4198 resched_curr(rq_of(cfs_rq)); 4199 return; 4200 } 4201 /* 4202 * don't let the period tick interfere with the hrtick preemption 4203 */ 4204 if (!sched_feat(DOUBLE_TICK) && 4205 hrtimer_active(&rq_of(cfs_rq)->hrtick_timer)) 4206 return; 4207 #endif 4208 4209 if (cfs_rq->nr_running > 1) 4210 check_preempt_tick(cfs_rq, curr); 4211 } 4212 4213 4214 /************************************************** 4215 * CFS bandwidth control machinery 4216 */ 4217 4218 #ifdef CONFIG_CFS_BANDWIDTH 4219 4220 #ifdef CONFIG_JUMP_LABEL 4221 static struct static_key __cfs_bandwidth_used; 4222 4223 static inline bool cfs_bandwidth_used(void) 4224 { 4225 return static_key_false(&__cfs_bandwidth_used); 4226 } 4227 4228 void cfs_bandwidth_usage_inc(void) 4229 { 4230 static_key_slow_inc_cpuslocked(&__cfs_bandwidth_used); 4231 } 4232 4233 void cfs_bandwidth_usage_dec(void) 4234 { 4235 static_key_slow_dec_cpuslocked(&__cfs_bandwidth_used); 4236 } 4237 #else /* CONFIG_JUMP_LABEL */ 4238 static bool cfs_bandwidth_used(void) 4239 { 4240 return true; 4241 } 4242 4243 void cfs_bandwidth_usage_inc(void) {} 4244 void cfs_bandwidth_usage_dec(void) {} 4245 #endif /* CONFIG_JUMP_LABEL */ 4246 4247 /* 4248 * default period for cfs group bandwidth. 4249 * default: 0.1s, units: nanoseconds 4250 */ 4251 static inline u64 default_cfs_period(void) 4252 { 4253 return 100000000ULL; 4254 } 4255 4256 static inline u64 sched_cfs_bandwidth_slice(void) 4257 { 4258 return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC; 4259 } 4260 4261 /* 4262 * Replenish runtime according to assigned quota and update expiration time. 4263 * We use sched_clock_cpu directly instead of rq->clock to avoid adding 4264 * additional synchronization around rq->lock. 4265 * 4266 * requires cfs_b->lock 4267 */ 4268 void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b) 4269 { 4270 u64 now; 4271 4272 if (cfs_b->quota == RUNTIME_INF) 4273 return; 4274 4275 now = sched_clock_cpu(smp_processor_id()); 4276 cfs_b->runtime = cfs_b->quota; 4277 cfs_b->runtime_expires = now + ktime_to_ns(cfs_b->period); 4278 cfs_b->expires_seq++; 4279 } 4280 4281 static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) 4282 { 4283 return &tg->cfs_bandwidth; 4284 } 4285 4286 /* rq->task_clock normalized against any time this cfs_rq has spent throttled */ 4287 static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq) 4288 { 4289 if (unlikely(cfs_rq->throttle_count)) 4290 return cfs_rq->throttled_clock_task - cfs_rq->throttled_clock_task_time; 4291 4292 return rq_clock_task(rq_of(cfs_rq)) - cfs_rq->throttled_clock_task_time; 4293 } 4294 4295 /* returns 0 on failure to allocate runtime */ 4296 static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq) 4297 { 4298 struct task_group *tg = cfs_rq->tg; 4299 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg); 4300 u64 amount = 0, min_amount, expires; 4301 int expires_seq; 4302 4303 /* note: this is a positive sum as runtime_remaining <= 0 */ 4304 min_amount = sched_cfs_bandwidth_slice() - cfs_rq->runtime_remaining; 4305 4306 raw_spin_lock(&cfs_b->lock); 4307 if (cfs_b->quota == RUNTIME_INF) 4308 amount = min_amount; 4309 else { 4310 start_cfs_bandwidth(cfs_b); 4311 4312 if (cfs_b->runtime > 0) { 4313 amount = min(cfs_b->runtime, min_amount); 4314 cfs_b->runtime -= amount; 4315 cfs_b->idle = 0; 4316 } 4317 } 4318 expires_seq = cfs_b->expires_seq; 4319 expires = cfs_b->runtime_expires; 4320 raw_spin_unlock(&cfs_b->lock); 4321 4322 cfs_rq->runtime_remaining += amount; 4323 /* 4324 * we may have advanced our local expiration to account for allowed 4325 * spread between our sched_clock and the one on which runtime was 4326 * issued. 4327 */ 4328 if (cfs_rq->expires_seq != expires_seq) { 4329 cfs_rq->expires_seq = expires_seq; 4330 cfs_rq->runtime_expires = expires; 4331 } 4332 4333 return cfs_rq->runtime_remaining > 0; 4334 } 4335 4336 /* 4337 * Note: This depends on the synchronization provided by sched_clock and the 4338 * fact that rq->clock snapshots this value. 4339 */ 4340 static void expire_cfs_rq_runtime(struct cfs_rq *cfs_rq) 4341 { 4342 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 4343 4344 /* if the deadline is ahead of our clock, nothing to do */ 4345 if (likely((s64)(rq_clock(rq_of(cfs_rq)) - cfs_rq->runtime_expires) < 0)) 4346 return; 4347 4348 if (cfs_rq->runtime_remaining < 0) 4349 return; 4350 4351 /* 4352 * If the local deadline has passed we have to consider the 4353 * possibility that our sched_clock is 'fast' and the global deadline 4354 * has not truly expired. 4355 * 4356 * Fortunately we can check determine whether this the case by checking 4357 * whether the global deadline(cfs_b->expires_seq) has advanced. 4358 */ 4359 if (cfs_rq->expires_seq == cfs_b->expires_seq) { 4360 /* extend local deadline, drift is bounded above by 2 ticks */ 4361 cfs_rq->runtime_expires += TICK_NSEC; 4362 } else { 4363 /* global deadline is ahead, expiration has passed */ 4364 cfs_rq->runtime_remaining = 0; 4365 } 4366 } 4367 4368 static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) 4369 { 4370 /* dock delta_exec before expiring quota (as it could span periods) */ 4371 cfs_rq->runtime_remaining -= delta_exec; 4372 expire_cfs_rq_runtime(cfs_rq); 4373 4374 if (likely(cfs_rq->runtime_remaining > 0)) 4375 return; 4376 4377 /* 4378 * if we're unable to extend our runtime we resched so that the active 4379 * hierarchy can be throttled 4380 */ 4381 if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr)) 4382 resched_curr(rq_of(cfs_rq)); 4383 } 4384 4385 static __always_inline 4386 void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) 4387 { 4388 if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled) 4389 return; 4390 4391 __account_cfs_rq_runtime(cfs_rq, delta_exec); 4392 } 4393 4394 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) 4395 { 4396 return cfs_bandwidth_used() && cfs_rq->throttled; 4397 } 4398 4399 /* check whether cfs_rq, or any parent, is throttled */ 4400 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) 4401 { 4402 return cfs_bandwidth_used() && cfs_rq->throttle_count; 4403 } 4404 4405 /* 4406 * Ensure that neither of the group entities corresponding to src_cpu or 4407 * dest_cpu are members of a throttled hierarchy when performing group 4408 * load-balance operations. 4409 */ 4410 static inline int throttled_lb_pair(struct task_group *tg, 4411 int src_cpu, int dest_cpu) 4412 { 4413 struct cfs_rq *src_cfs_rq, *dest_cfs_rq; 4414 4415 src_cfs_rq = tg->cfs_rq[src_cpu]; 4416 dest_cfs_rq = tg->cfs_rq[dest_cpu]; 4417 4418 return throttled_hierarchy(src_cfs_rq) || 4419 throttled_hierarchy(dest_cfs_rq); 4420 } 4421 4422 static int tg_unthrottle_up(struct task_group *tg, void *data) 4423 { 4424 struct rq *rq = data; 4425 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 4426 4427 cfs_rq->throttle_count--; 4428 if (!cfs_rq->throttle_count) { 4429 /* adjust cfs_rq_clock_task() */ 4430 cfs_rq->throttled_clock_task_time += rq_clock_task(rq) - 4431 cfs_rq->throttled_clock_task; 4432 } 4433 4434 return 0; 4435 } 4436 4437 static int tg_throttle_down(struct task_group *tg, void *data) 4438 { 4439 struct rq *rq = data; 4440 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 4441 4442 /* group is entering throttled state, stop time */ 4443 if (!cfs_rq->throttle_count) 4444 cfs_rq->throttled_clock_task = rq_clock_task(rq); 4445 cfs_rq->throttle_count++; 4446 4447 return 0; 4448 } 4449 4450 static void throttle_cfs_rq(struct cfs_rq *cfs_rq) 4451 { 4452 struct rq *rq = rq_of(cfs_rq); 4453 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 4454 struct sched_entity *se; 4455 long task_delta, dequeue = 1; 4456 bool empty; 4457 4458 se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))]; 4459 4460 /* freeze hierarchy runnable averages while throttled */ 4461 rcu_read_lock(); 4462 walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq); 4463 rcu_read_unlock(); 4464 4465 task_delta = cfs_rq->h_nr_running; 4466 for_each_sched_entity(se) { 4467 struct cfs_rq *qcfs_rq = cfs_rq_of(se); 4468 /* throttled entity or throttle-on-deactivate */ 4469 if (!se->on_rq) 4470 break; 4471 4472 if (dequeue) 4473 dequeue_entity(qcfs_rq, se, DEQUEUE_SLEEP); 4474 qcfs_rq->h_nr_running -= task_delta; 4475 4476 if (qcfs_rq->load.weight) 4477 dequeue = 0; 4478 } 4479 4480 if (!se) 4481 sub_nr_running(rq, task_delta); 4482 4483 cfs_rq->throttled = 1; 4484 cfs_rq->throttled_clock = rq_clock(rq); 4485 raw_spin_lock(&cfs_b->lock); 4486 empty = list_empty(&cfs_b->throttled_cfs_rq); 4487 4488 /* 4489 * Add to the _head_ of the list, so that an already-started 4490 * distribute_cfs_runtime will not see us. If disribute_cfs_runtime is 4491 * not running add to the tail so that later runqueues don't get starved. 4492 */ 4493 if (cfs_b->distribute_running) 4494 list_add_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq); 4495 else 4496 list_add_tail_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq); 4497 4498 /* 4499 * If we're the first throttled task, make sure the bandwidth 4500 * timer is running. 4501 */ 4502 if (empty) 4503 start_cfs_bandwidth(cfs_b); 4504 4505 raw_spin_unlock(&cfs_b->lock); 4506 } 4507 4508 void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) 4509 { 4510 struct rq *rq = rq_of(cfs_rq); 4511 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 4512 struct sched_entity *se; 4513 int enqueue = 1; 4514 long task_delta; 4515 4516 se = cfs_rq->tg->se[cpu_of(rq)]; 4517 4518 cfs_rq->throttled = 0; 4519 4520 update_rq_clock(rq); 4521 4522 raw_spin_lock(&cfs_b->lock); 4523 cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock; 4524 list_del_rcu(&cfs_rq->throttled_list); 4525 raw_spin_unlock(&cfs_b->lock); 4526 4527 /* update hierarchical throttle state */ 4528 walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq); 4529 4530 if (!cfs_rq->load.weight) 4531 return; 4532 4533 task_delta = cfs_rq->h_nr_running; 4534 for_each_sched_entity(se) { 4535 if (se->on_rq) 4536 enqueue = 0; 4537 4538 cfs_rq = cfs_rq_of(se); 4539 if (enqueue) 4540 enqueue_entity(cfs_rq, se, ENQUEUE_WAKEUP); 4541 cfs_rq->h_nr_running += task_delta; 4542 4543 if (cfs_rq_throttled(cfs_rq)) 4544 break; 4545 } 4546 4547 if (!se) 4548 add_nr_running(rq, task_delta); 4549 4550 /* Determine whether we need to wake up potentially idle CPU: */ 4551 if (rq->curr == rq->idle && rq->cfs.nr_running) 4552 resched_curr(rq); 4553 } 4554 4555 static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b, 4556 u64 remaining, u64 expires) 4557 { 4558 struct cfs_rq *cfs_rq; 4559 u64 runtime; 4560 u64 starting_runtime = remaining; 4561 4562 rcu_read_lock(); 4563 list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq, 4564 throttled_list) { 4565 struct rq *rq = rq_of(cfs_rq); 4566 struct rq_flags rf; 4567 4568 rq_lock(rq, &rf); 4569 if (!cfs_rq_throttled(cfs_rq)) 4570 goto next; 4571 4572 runtime = -cfs_rq->runtime_remaining + 1; 4573 if (runtime > remaining) 4574 runtime = remaining; 4575 remaining -= runtime; 4576 4577 cfs_rq->runtime_remaining += runtime; 4578 cfs_rq->runtime_expires = expires; 4579 4580 /* we check whether we're throttled above */ 4581 if (cfs_rq->runtime_remaining > 0) 4582 unthrottle_cfs_rq(cfs_rq); 4583 4584 next: 4585 rq_unlock(rq, &rf); 4586 4587 if (!remaining) 4588 break; 4589 } 4590 rcu_read_unlock(); 4591 4592 return starting_runtime - remaining; 4593 } 4594 4595 /* 4596 * Responsible for refilling a task_group's bandwidth and unthrottling its 4597 * cfs_rqs as appropriate. If there has been no activity within the last 4598 * period the timer is deactivated until scheduling resumes; cfs_b->idle is 4599 * used to track this state. 4600 */ 4601 static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun) 4602 { 4603 u64 runtime, runtime_expires; 4604 int throttled; 4605 4606 /* no need to continue the timer with no bandwidth constraint */ 4607 if (cfs_b->quota == RUNTIME_INF) 4608 goto out_deactivate; 4609 4610 throttled = !list_empty(&cfs_b->throttled_cfs_rq); 4611 cfs_b->nr_periods += overrun; 4612 4613 /* 4614 * idle depends on !throttled (for the case of a large deficit), and if 4615 * we're going inactive then everything else can be deferred 4616 */ 4617 if (cfs_b->idle && !throttled) 4618 goto out_deactivate; 4619 4620 __refill_cfs_bandwidth_runtime(cfs_b); 4621 4622 if (!throttled) { 4623 /* mark as potentially idle for the upcoming period */ 4624 cfs_b->idle = 1; 4625 return 0; 4626 } 4627 4628 /* account preceding periods in which throttling occurred */ 4629 cfs_b->nr_throttled += overrun; 4630 4631 runtime_expires = cfs_b->runtime_expires; 4632 4633 /* 4634 * This check is repeated as we are holding onto the new bandwidth while 4635 * we unthrottle. This can potentially race with an unthrottled group 4636 * trying to acquire new bandwidth from the global pool. This can result 4637 * in us over-using our runtime if it is all used during this loop, but 4638 * only by limited amounts in that extreme case. 4639 */ 4640 while (throttled && cfs_b->runtime > 0 && !cfs_b->distribute_running) { 4641 runtime = cfs_b->runtime; 4642 cfs_b->distribute_running = 1; 4643 raw_spin_unlock(&cfs_b->lock); 4644 /* we can't nest cfs_b->lock while distributing bandwidth */ 4645 runtime = distribute_cfs_runtime(cfs_b, runtime, 4646 runtime_expires); 4647 raw_spin_lock(&cfs_b->lock); 4648 4649 cfs_b->distribute_running = 0; 4650 throttled = !list_empty(&cfs_b->throttled_cfs_rq); 4651 4652 lsub_positive(&cfs_b->runtime, runtime); 4653 } 4654 4655 /* 4656 * While we are ensured activity in the period following an 4657 * unthrottle, this also covers the case in which the new bandwidth is 4658 * insufficient to cover the existing bandwidth deficit. (Forcing the 4659 * timer to remain active while there are any throttled entities.) 4660 */ 4661 cfs_b->idle = 0; 4662 4663 return 0; 4664 4665 out_deactivate: 4666 return 1; 4667 } 4668 4669 /* a cfs_rq won't donate quota below this amount */ 4670 static const u64 min_cfs_rq_runtime = 1 * NSEC_PER_MSEC; 4671 /* minimum remaining period time to redistribute slack quota */ 4672 static const u64 min_bandwidth_expiration = 2 * NSEC_PER_MSEC; 4673 /* how long we wait to gather additional slack before distributing */ 4674 static const u64 cfs_bandwidth_slack_period = 5 * NSEC_PER_MSEC; 4675 4676 /* 4677 * Are we near the end of the current quota period? 4678 * 4679 * Requires cfs_b->lock for hrtimer_expires_remaining to be safe against the 4680 * hrtimer base being cleared by hrtimer_start. In the case of 4681 * migrate_hrtimers, base is never cleared, so we are fine. 4682 */ 4683 static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire) 4684 { 4685 struct hrtimer *refresh_timer = &cfs_b->period_timer; 4686 u64 remaining; 4687 4688 /* if the call-back is running a quota refresh is already occurring */ 4689 if (hrtimer_callback_running(refresh_timer)) 4690 return 1; 4691 4692 /* is a quota refresh about to occur? */ 4693 remaining = ktime_to_ns(hrtimer_expires_remaining(refresh_timer)); 4694 if (remaining < min_expire) 4695 return 1; 4696 4697 return 0; 4698 } 4699 4700 static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b) 4701 { 4702 u64 min_left = cfs_bandwidth_slack_period + min_bandwidth_expiration; 4703 4704 /* if there's a quota refresh soon don't bother with slack */ 4705 if (runtime_refresh_within(cfs_b, min_left)) 4706 return; 4707 4708 hrtimer_start(&cfs_b->slack_timer, 4709 ns_to_ktime(cfs_bandwidth_slack_period), 4710 HRTIMER_MODE_REL); 4711 } 4712 4713 /* we know any runtime found here is valid as update_curr() precedes return */ 4714 static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq) 4715 { 4716 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 4717 s64 slack_runtime = cfs_rq->runtime_remaining - min_cfs_rq_runtime; 4718 4719 if (slack_runtime <= 0) 4720 return; 4721 4722 raw_spin_lock(&cfs_b->lock); 4723 if (cfs_b->quota != RUNTIME_INF && 4724 cfs_rq->runtime_expires == cfs_b->runtime_expires) { 4725 cfs_b->runtime += slack_runtime; 4726 4727 /* we are under rq->lock, defer unthrottling using a timer */ 4728 if (cfs_b->runtime > sched_cfs_bandwidth_slice() && 4729 !list_empty(&cfs_b->throttled_cfs_rq)) 4730 start_cfs_slack_bandwidth(cfs_b); 4731 } 4732 raw_spin_unlock(&cfs_b->lock); 4733 4734 /* even if it's not valid for return we don't want to try again */ 4735 cfs_rq->runtime_remaining -= slack_runtime; 4736 } 4737 4738 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) 4739 { 4740 if (!cfs_bandwidth_used()) 4741 return; 4742 4743 if (!cfs_rq->runtime_enabled || cfs_rq->nr_running) 4744 return; 4745 4746 __return_cfs_rq_runtime(cfs_rq); 4747 } 4748 4749 /* 4750 * This is done with a timer (instead of inline with bandwidth return) since 4751 * it's necessary to juggle rq->locks to unthrottle their respective cfs_rqs. 4752 */ 4753 static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) 4754 { 4755 u64 runtime = 0, slice = sched_cfs_bandwidth_slice(); 4756 u64 expires; 4757 4758 /* confirm we're still not at a refresh boundary */ 4759 raw_spin_lock(&cfs_b->lock); 4760 if (cfs_b->distribute_running) { 4761 raw_spin_unlock(&cfs_b->lock); 4762 return; 4763 } 4764 4765 if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) { 4766 raw_spin_unlock(&cfs_b->lock); 4767 return; 4768 } 4769 4770 if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice) 4771 runtime = cfs_b->runtime; 4772 4773 expires = cfs_b->runtime_expires; 4774 if (runtime) 4775 cfs_b->distribute_running = 1; 4776 4777 raw_spin_unlock(&cfs_b->lock); 4778 4779 if (!runtime) 4780 return; 4781 4782 runtime = distribute_cfs_runtime(cfs_b, runtime, expires); 4783 4784 raw_spin_lock(&cfs_b->lock); 4785 if (expires == cfs_b->runtime_expires) 4786 lsub_positive(&cfs_b->runtime, runtime); 4787 cfs_b->distribute_running = 0; 4788 raw_spin_unlock(&cfs_b->lock); 4789 } 4790 4791 /* 4792 * When a group wakes up we want to make sure that its quota is not already 4793 * expired/exceeded, otherwise it may be allowed to steal additional ticks of 4794 * runtime as update_curr() throttling can not not trigger until it's on-rq. 4795 */ 4796 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) 4797 { 4798 if (!cfs_bandwidth_used()) 4799 return; 4800 4801 /* an active group must be handled by the update_curr()->put() path */ 4802 if (!cfs_rq->runtime_enabled || cfs_rq->curr) 4803 return; 4804 4805 /* ensure the group is not already throttled */ 4806 if (cfs_rq_throttled(cfs_rq)) 4807 return; 4808 4809 /* update runtime allocation */ 4810 account_cfs_rq_runtime(cfs_rq, 0); 4811 if (cfs_rq->runtime_remaining <= 0) 4812 throttle_cfs_rq(cfs_rq); 4813 } 4814 4815 static void sync_throttle(struct task_group *tg, int cpu) 4816 { 4817 struct cfs_rq *pcfs_rq, *cfs_rq; 4818 4819 if (!cfs_bandwidth_used()) 4820 return; 4821 4822 if (!tg->parent) 4823 return; 4824 4825 cfs_rq = tg->cfs_rq[cpu]; 4826 pcfs_rq = tg->parent->cfs_rq[cpu]; 4827 4828 cfs_rq->throttle_count = pcfs_rq->throttle_count; 4829 cfs_rq->throttled_clock_task = rq_clock_task(cpu_rq(cpu)); 4830 } 4831 4832 /* conditionally throttle active cfs_rq's from put_prev_entity() */ 4833 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) 4834 { 4835 if (!cfs_bandwidth_used()) 4836 return false; 4837 4838 if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0)) 4839 return false; 4840 4841 /* 4842 * it's possible for a throttled entity to be forced into a running 4843 * state (e.g. set_curr_task), in this case we're finished. 4844 */ 4845 if (cfs_rq_throttled(cfs_rq)) 4846 return true; 4847 4848 throttle_cfs_rq(cfs_rq); 4849 return true; 4850 } 4851 4852 static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer) 4853 { 4854 struct cfs_bandwidth *cfs_b = 4855 container_of(timer, struct cfs_bandwidth, slack_timer); 4856 4857 do_sched_cfs_slack_timer(cfs_b); 4858 4859 return HRTIMER_NORESTART; 4860 } 4861 4862 static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer) 4863 { 4864 struct cfs_bandwidth *cfs_b = 4865 container_of(timer, struct cfs_bandwidth, period_timer); 4866 int overrun; 4867 int idle = 0; 4868 4869 raw_spin_lock(&cfs_b->lock); 4870 for (;;) { 4871 overrun = hrtimer_forward_now(timer, cfs_b->period); 4872 if (!overrun) 4873 break; 4874 4875 idle = do_sched_cfs_period_timer(cfs_b, overrun); 4876 } 4877 if (idle) 4878 cfs_b->period_active = 0; 4879 raw_spin_unlock(&cfs_b->lock); 4880 4881 return idle ? HRTIMER_NORESTART : HRTIMER_RESTART; 4882 } 4883 4884 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) 4885 { 4886 raw_spin_lock_init(&cfs_b->lock); 4887 cfs_b->runtime = 0; 4888 cfs_b->quota = RUNTIME_INF; 4889 cfs_b->period = ns_to_ktime(default_cfs_period()); 4890 4891 INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq); 4892 hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED); 4893 cfs_b->period_timer.function = sched_cfs_period_timer; 4894 hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 4895 cfs_b->slack_timer.function = sched_cfs_slack_timer; 4896 cfs_b->distribute_running = 0; 4897 } 4898 4899 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) 4900 { 4901 cfs_rq->runtime_enabled = 0; 4902 INIT_LIST_HEAD(&cfs_rq->throttled_list); 4903 } 4904 4905 void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b) 4906 { 4907 u64 overrun; 4908 4909 lockdep_assert_held(&cfs_b->lock); 4910 4911 if (cfs_b->period_active) 4912 return; 4913 4914 cfs_b->period_active = 1; 4915 overrun = hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period); 4916 cfs_b->runtime_expires += (overrun + 1) * ktime_to_ns(cfs_b->period); 4917 cfs_b->expires_seq++; 4918 hrtimer_start_expires(&cfs_b->period_timer, HRTIMER_MODE_ABS_PINNED); 4919 } 4920 4921 static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) 4922 { 4923 /* init_cfs_bandwidth() was not called */ 4924 if (!cfs_b->throttled_cfs_rq.next) 4925 return; 4926 4927 hrtimer_cancel(&cfs_b->period_timer); 4928 hrtimer_cancel(&cfs_b->slack_timer); 4929 } 4930 4931 /* 4932 * Both these CPU hotplug callbacks race against unregister_fair_sched_group() 4933 * 4934 * The race is harmless, since modifying bandwidth settings of unhooked group 4935 * bits doesn't do much. 4936 */ 4937 4938 /* cpu online calback */ 4939 static void __maybe_unused update_runtime_enabled(struct rq *rq) 4940 { 4941 struct task_group *tg; 4942 4943 lockdep_assert_held(&rq->lock); 4944 4945 rcu_read_lock(); 4946 list_for_each_entry_rcu(tg, &task_groups, list) { 4947 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 4948 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 4949 4950 raw_spin_lock(&cfs_b->lock); 4951 cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF; 4952 raw_spin_unlock(&cfs_b->lock); 4953 } 4954 rcu_read_unlock(); 4955 } 4956 4957 /* cpu offline callback */ 4958 static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq) 4959 { 4960 struct task_group *tg; 4961 4962 lockdep_assert_held(&rq->lock); 4963 4964 rcu_read_lock(); 4965 list_for_each_entry_rcu(tg, &task_groups, list) { 4966 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 4967 4968 if (!cfs_rq->runtime_enabled) 4969 continue; 4970 4971 /* 4972 * clock_task is not advancing so we just need to make sure 4973 * there's some valid quota amount 4974 */ 4975 cfs_rq->runtime_remaining = 1; 4976 /* 4977 * Offline rq is schedulable till CPU is completely disabled 4978 * in take_cpu_down(), so we prevent new cfs throttling here. 4979 */ 4980 cfs_rq->runtime_enabled = 0; 4981 4982 if (cfs_rq_throttled(cfs_rq)) 4983 unthrottle_cfs_rq(cfs_rq); 4984 } 4985 rcu_read_unlock(); 4986 } 4987 4988 #else /* CONFIG_CFS_BANDWIDTH */ 4989 static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq) 4990 { 4991 return rq_clock_task(rq_of(cfs_rq)); 4992 } 4993 4994 static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) {} 4995 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) { return false; } 4996 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {} 4997 static inline void sync_throttle(struct task_group *tg, int cpu) {} 4998 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} 4999 5000 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) 5001 { 5002 return 0; 5003 } 5004 5005 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) 5006 { 5007 return 0; 5008 } 5009 5010 static inline int throttled_lb_pair(struct task_group *tg, 5011 int src_cpu, int dest_cpu) 5012 { 5013 return 0; 5014 } 5015 5016 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {} 5017 5018 #ifdef CONFIG_FAIR_GROUP_SCHED 5019 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} 5020 #endif 5021 5022 static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) 5023 { 5024 return NULL; 5025 } 5026 static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {} 5027 static inline void update_runtime_enabled(struct rq *rq) {} 5028 static inline void unthrottle_offline_cfs_rqs(struct rq *rq) {} 5029 5030 #endif /* CONFIG_CFS_BANDWIDTH */ 5031 5032 /************************************************** 5033 * CFS operations on tasks: 5034 */ 5035 5036 #ifdef CONFIG_SCHED_HRTICK 5037 static void hrtick_start_fair(struct rq *rq, struct task_struct *p) 5038 { 5039 struct sched_entity *se = &p->se; 5040 struct cfs_rq *cfs_rq = cfs_rq_of(se); 5041 5042 SCHED_WARN_ON(task_rq(p) != rq); 5043 5044 if (rq->cfs.h_nr_running > 1) { 5045 u64 slice = sched_slice(cfs_rq, se); 5046 u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime; 5047 s64 delta = slice - ran; 5048 5049 if (delta < 0) { 5050 if (rq->curr == p) 5051 resched_curr(rq); 5052 return; 5053 } 5054 hrtick_start(rq, delta); 5055 } 5056 } 5057 5058 /* 5059 * called from enqueue/dequeue and updates the hrtick when the 5060 * current task is from our class and nr_running is low enough 5061 * to matter. 5062 */ 5063 static void hrtick_update(struct rq *rq) 5064 { 5065 struct task_struct *curr = rq->curr; 5066 5067 if (!hrtick_enabled(rq) || curr->sched_class != &fair_sched_class) 5068 return; 5069 5070 if (cfs_rq_of(&curr->se)->nr_running < sched_nr_latency) 5071 hrtick_start_fair(rq, curr); 5072 } 5073 #else /* !CONFIG_SCHED_HRTICK */ 5074 static inline void 5075 hrtick_start_fair(struct rq *rq, struct task_struct *p) 5076 { 5077 } 5078 5079 static inline void hrtick_update(struct rq *rq) 5080 { 5081 } 5082 #endif 5083 5084 #ifdef CONFIG_SMP 5085 static inline unsigned long cpu_util(int cpu); 5086 static unsigned long capacity_of(int cpu); 5087 5088 static inline bool cpu_overutilized(int cpu) 5089 { 5090 return (capacity_of(cpu) * 1024) < (cpu_util(cpu) * capacity_margin); 5091 } 5092 5093 static inline void update_overutilized_status(struct rq *rq) 5094 { 5095 if (!READ_ONCE(rq->rd->overutilized) && cpu_overutilized(rq->cpu)) 5096 WRITE_ONCE(rq->rd->overutilized, SG_OVERUTILIZED); 5097 } 5098 #else 5099 static inline void update_overutilized_status(struct rq *rq) { } 5100 #endif 5101 5102 /* 5103 * The enqueue_task method is called before nr_running is 5104 * increased. Here we update the fair scheduling stats and 5105 * then put the task into the rbtree: 5106 */ 5107 static void 5108 enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) 5109 { 5110 struct cfs_rq *cfs_rq; 5111 struct sched_entity *se = &p->se; 5112 5113 /* 5114 * The code below (indirectly) updates schedutil which looks at 5115 * the cfs_rq utilization to select a frequency. 5116 * Let's add the task's estimated utilization to the cfs_rq's 5117 * estimated utilization, before we update schedutil. 5118 */ 5119 util_est_enqueue(&rq->cfs, p); 5120 5121 /* 5122 * If in_iowait is set, the code below may not trigger any cpufreq 5123 * utilization updates, so do it here explicitly with the IOWAIT flag 5124 * passed. 5125 */ 5126 if (p->in_iowait) 5127 cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT); 5128 5129 for_each_sched_entity(se) { 5130 if (se->on_rq) 5131 break; 5132 cfs_rq = cfs_rq_of(se); 5133 enqueue_entity(cfs_rq, se, flags); 5134 5135 /* 5136 * end evaluation on encountering a throttled cfs_rq 5137 * 5138 * note: in the case of encountering a throttled cfs_rq we will 5139 * post the final h_nr_running increment below. 5140 */ 5141 if (cfs_rq_throttled(cfs_rq)) 5142 break; 5143 cfs_rq->h_nr_running++; 5144 5145 flags = ENQUEUE_WAKEUP; 5146 } 5147 5148 for_each_sched_entity(se) { 5149 cfs_rq = cfs_rq_of(se); 5150 cfs_rq->h_nr_running++; 5151 5152 if (cfs_rq_throttled(cfs_rq)) 5153 break; 5154 5155 update_load_avg(cfs_rq, se, UPDATE_TG); 5156 update_cfs_group(se); 5157 } 5158 5159 if (!se) { 5160 add_nr_running(rq, 1); 5161 /* 5162 * Since new tasks are assigned an initial util_avg equal to 5163 * half of the spare capacity of their CPU, tiny tasks have the 5164 * ability to cross the overutilized threshold, which will 5165 * result in the load balancer ruining all the task placement 5166 * done by EAS. As a way to mitigate that effect, do not account 5167 * for the first enqueue operation of new tasks during the 5168 * overutilized flag detection. 5169 * 5170 * A better way of solving this problem would be to wait for 5171 * the PELT signals of tasks to converge before taking them 5172 * into account, but that is not straightforward to implement, 5173 * and the following generally works well enough in practice. 5174 */ 5175 if (flags & ENQUEUE_WAKEUP) 5176 update_overutilized_status(rq); 5177 5178 } 5179 5180 hrtick_update(rq); 5181 } 5182 5183 static void set_next_buddy(struct sched_entity *se); 5184 5185 /* 5186 * The dequeue_task method is called before nr_running is 5187 * decreased. We remove the task from the rbtree and 5188 * update the fair scheduling stats: 5189 */ 5190 static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) 5191 { 5192 struct cfs_rq *cfs_rq; 5193 struct sched_entity *se = &p->se; 5194 int task_sleep = flags & DEQUEUE_SLEEP; 5195 5196 for_each_sched_entity(se) { 5197 cfs_rq = cfs_rq_of(se); 5198 dequeue_entity(cfs_rq, se, flags); 5199 5200 /* 5201 * end evaluation on encountering a throttled cfs_rq 5202 * 5203 * note: in the case of encountering a throttled cfs_rq we will 5204 * post the final h_nr_running decrement below. 5205 */ 5206 if (cfs_rq_throttled(cfs_rq)) 5207 break; 5208 cfs_rq->h_nr_running--; 5209 5210 /* Don't dequeue parent if it has other entities besides us */ 5211 if (cfs_rq->load.weight) { 5212 /* Avoid re-evaluating load for this entity: */ 5213 se = parent_entity(se); 5214 /* 5215 * Bias pick_next to pick a task from this cfs_rq, as 5216 * p is sleeping when it is within its sched_slice. 5217 */ 5218 if (task_sleep && se && !throttled_hierarchy(cfs_rq)) 5219 set_next_buddy(se); 5220 break; 5221 } 5222 flags |= DEQUEUE_SLEEP; 5223 } 5224 5225 for_each_sched_entity(se) { 5226 cfs_rq = cfs_rq_of(se); 5227 cfs_rq->h_nr_running--; 5228 5229 if (cfs_rq_throttled(cfs_rq)) 5230 break; 5231 5232 update_load_avg(cfs_rq, se, UPDATE_TG); 5233 update_cfs_group(se); 5234 } 5235 5236 if (!se) 5237 sub_nr_running(rq, 1); 5238 5239 util_est_dequeue(&rq->cfs, p, task_sleep); 5240 hrtick_update(rq); 5241 } 5242 5243 #ifdef CONFIG_SMP 5244 5245 /* Working cpumask for: load_balance, load_balance_newidle. */ 5246 DEFINE_PER_CPU(cpumask_var_t, load_balance_mask); 5247 DEFINE_PER_CPU(cpumask_var_t, select_idle_mask); 5248 5249 #ifdef CONFIG_NO_HZ_COMMON 5250 /* 5251 * per rq 'load' arrray crap; XXX kill this. 5252 */ 5253 5254 /* 5255 * The exact cpuload calculated at every tick would be: 5256 * 5257 * load' = (1 - 1/2^i) * load + (1/2^i) * cur_load 5258 * 5259 * If a CPU misses updates for n ticks (as it was idle) and update gets 5260 * called on the n+1-th tick when CPU may be busy, then we have: 5261 * 5262 * load_n = (1 - 1/2^i)^n * load_0 5263 * load_n+1 = (1 - 1/2^i) * load_n + (1/2^i) * cur_load 5264 * 5265 * decay_load_missed() below does efficient calculation of 5266 * 5267 * load' = (1 - 1/2^i)^n * load 5268 * 5269 * Because x^(n+m) := x^n * x^m we can decompose any x^n in power-of-2 factors. 5270 * This allows us to precompute the above in said factors, thereby allowing the 5271 * reduction of an arbitrary n in O(log_2 n) steps. (See also 5272 * fixed_power_int()) 5273 * 5274 * The calculation is approximated on a 128 point scale. 5275 */ 5276 #define DEGRADE_SHIFT 7 5277 5278 static const u8 degrade_zero_ticks[CPU_LOAD_IDX_MAX] = {0, 8, 32, 64, 128}; 5279 static const u8 degrade_factor[CPU_LOAD_IDX_MAX][DEGRADE_SHIFT + 1] = { 5280 { 0, 0, 0, 0, 0, 0, 0, 0 }, 5281 { 64, 32, 8, 0, 0, 0, 0, 0 }, 5282 { 96, 72, 40, 12, 1, 0, 0, 0 }, 5283 { 112, 98, 75, 43, 15, 1, 0, 0 }, 5284 { 120, 112, 98, 76, 45, 16, 2, 0 } 5285 }; 5286 5287 /* 5288 * Update cpu_load for any missed ticks, due to tickless idle. The backlog 5289 * would be when CPU is idle and so we just decay the old load without 5290 * adding any new load. 5291 */ 5292 static unsigned long 5293 decay_load_missed(unsigned long load, unsigned long missed_updates, int idx) 5294 { 5295 int j = 0; 5296 5297 if (!missed_updates) 5298 return load; 5299 5300 if (missed_updates >= degrade_zero_ticks[idx]) 5301 return 0; 5302 5303 if (idx == 1) 5304 return load >> missed_updates; 5305 5306 while (missed_updates) { 5307 if (missed_updates % 2) 5308 load = (load * degrade_factor[idx][j]) >> DEGRADE_SHIFT; 5309 5310 missed_updates >>= 1; 5311 j++; 5312 } 5313 return load; 5314 } 5315 5316 static struct { 5317 cpumask_var_t idle_cpus_mask; 5318 atomic_t nr_cpus; 5319 int has_blocked; /* Idle CPUS has blocked load */ 5320 unsigned long next_balance; /* in jiffy units */ 5321 unsigned long next_blocked; /* Next update of blocked load in jiffies */ 5322 } nohz ____cacheline_aligned; 5323 5324 #endif /* CONFIG_NO_HZ_COMMON */ 5325 5326 /** 5327 * __cpu_load_update - update the rq->cpu_load[] statistics 5328 * @this_rq: The rq to update statistics for 5329 * @this_load: The current load 5330 * @pending_updates: The number of missed updates 5331 * 5332 * Update rq->cpu_load[] statistics. This function is usually called every 5333 * scheduler tick (TICK_NSEC). 5334 * 5335 * This function computes a decaying average: 5336 * 5337 * load[i]' = (1 - 1/2^i) * load[i] + (1/2^i) * load 5338 * 5339 * Because of NOHZ it might not get called on every tick which gives need for 5340 * the @pending_updates argument. 5341 * 5342 * load[i]_n = (1 - 1/2^i) * load[i]_n-1 + (1/2^i) * load_n-1 5343 * = A * load[i]_n-1 + B ; A := (1 - 1/2^i), B := (1/2^i) * load 5344 * = A * (A * load[i]_n-2 + B) + B 5345 * = A * (A * (A * load[i]_n-3 + B) + B) + B 5346 * = A^3 * load[i]_n-3 + (A^2 + A + 1) * B 5347 * = A^n * load[i]_0 + (A^(n-1) + A^(n-2) + ... + 1) * B 5348 * = A^n * load[i]_0 + ((1 - A^n) / (1 - A)) * B 5349 * = (1 - 1/2^i)^n * (load[i]_0 - load) + load 5350 * 5351 * In the above we've assumed load_n := load, which is true for NOHZ_FULL as 5352 * any change in load would have resulted in the tick being turned back on. 5353 * 5354 * For regular NOHZ, this reduces to: 5355 * 5356 * load[i]_n = (1 - 1/2^i)^n * load[i]_0 5357 * 5358 * see decay_load_misses(). For NOHZ_FULL we get to subtract and add the extra 5359 * term. 5360 */ 5361 static void cpu_load_update(struct rq *this_rq, unsigned long this_load, 5362 unsigned long pending_updates) 5363 { 5364 unsigned long __maybe_unused tickless_load = this_rq->cpu_load[0]; 5365 int i, scale; 5366 5367 this_rq->nr_load_updates++; 5368 5369 /* Update our load: */ 5370 this_rq->cpu_load[0] = this_load; /* Fasttrack for idx 0 */ 5371 for (i = 1, scale = 2; i < CPU_LOAD_IDX_MAX; i++, scale += scale) { 5372 unsigned long old_load, new_load; 5373 5374 /* scale is effectively 1 << i now, and >> i divides by scale */ 5375 5376 old_load = this_rq->cpu_load[i]; 5377 #ifdef CONFIG_NO_HZ_COMMON 5378 old_load = decay_load_missed(old_load, pending_updates - 1, i); 5379 if (tickless_load) { 5380 old_load -= decay_load_missed(tickless_load, pending_updates - 1, i); 5381 /* 5382 * old_load can never be a negative value because a 5383 * decayed tickless_load cannot be greater than the 5384 * original tickless_load. 5385 */ 5386 old_load += tickless_load; 5387 } 5388 #endif 5389 new_load = this_load; 5390 /* 5391 * Round up the averaging division if load is increasing. This 5392 * prevents us from getting stuck on 9 if the load is 10, for 5393 * example. 5394 */ 5395 if (new_load > old_load) 5396 new_load += scale - 1; 5397 5398 this_rq->cpu_load[i] = (old_load * (scale - 1) + new_load) >> i; 5399 } 5400 } 5401 5402 /* Used instead of source_load when we know the type == 0 */ 5403 static unsigned long weighted_cpuload(struct rq *rq) 5404 { 5405 return cfs_rq_runnable_load_avg(&rq->cfs); 5406 } 5407 5408 #ifdef CONFIG_NO_HZ_COMMON 5409 /* 5410 * There is no sane way to deal with nohz on smp when using jiffies because the 5411 * CPU doing the jiffies update might drift wrt the CPU doing the jiffy reading 5412 * causing off-by-one errors in observed deltas; {0,2} instead of {1,1}. 5413 * 5414 * Therefore we need to avoid the delta approach from the regular tick when 5415 * possible since that would seriously skew the load calculation. This is why we 5416 * use cpu_load_update_periodic() for CPUs out of nohz. However we'll rely on 5417 * jiffies deltas for updates happening while in nohz mode (idle ticks, idle 5418 * loop exit, nohz_idle_balance, nohz full exit...) 5419 * 5420 * This means we might still be one tick off for nohz periods. 5421 */ 5422 5423 static void cpu_load_update_nohz(struct rq *this_rq, 5424 unsigned long curr_jiffies, 5425 unsigned long load) 5426 { 5427 unsigned long pending_updates; 5428 5429 pending_updates = curr_jiffies - this_rq->last_load_update_tick; 5430 if (pending_updates) { 5431 this_rq->last_load_update_tick = curr_jiffies; 5432 /* 5433 * In the regular NOHZ case, we were idle, this means load 0. 5434 * In the NOHZ_FULL case, we were non-idle, we should consider 5435 * its weighted load. 5436 */ 5437 cpu_load_update(this_rq, load, pending_updates); 5438 } 5439 } 5440 5441 /* 5442 * Called from nohz_idle_balance() to update the load ratings before doing the 5443 * idle balance. 5444 */ 5445 static void cpu_load_update_idle(struct rq *this_rq) 5446 { 5447 /* 5448 * bail if there's load or we're actually up-to-date. 5449 */ 5450 if (weighted_cpuload(this_rq)) 5451 return; 5452 5453 cpu_load_update_nohz(this_rq, READ_ONCE(jiffies), 0); 5454 } 5455 5456 /* 5457 * Record CPU load on nohz entry so we know the tickless load to account 5458 * on nohz exit. cpu_load[0] happens then to be updated more frequently 5459 * than other cpu_load[idx] but it should be fine as cpu_load readers 5460 * shouldn't rely into synchronized cpu_load[*] updates. 5461 */ 5462 void cpu_load_update_nohz_start(void) 5463 { 5464 struct rq *this_rq = this_rq(); 5465 5466 /* 5467 * This is all lockless but should be fine. If weighted_cpuload changes 5468 * concurrently we'll exit nohz. And cpu_load write can race with 5469 * cpu_load_update_idle() but both updater would be writing the same. 5470 */ 5471 this_rq->cpu_load[0] = weighted_cpuload(this_rq); 5472 } 5473 5474 /* 5475 * Account the tickless load in the end of a nohz frame. 5476 */ 5477 void cpu_load_update_nohz_stop(void) 5478 { 5479 unsigned long curr_jiffies = READ_ONCE(jiffies); 5480 struct rq *this_rq = this_rq(); 5481 unsigned long load; 5482 struct rq_flags rf; 5483 5484 if (curr_jiffies == this_rq->last_load_update_tick) 5485 return; 5486 5487 load = weighted_cpuload(this_rq); 5488 rq_lock(this_rq, &rf); 5489 update_rq_clock(this_rq); 5490 cpu_load_update_nohz(this_rq, curr_jiffies, load); 5491 rq_unlock(this_rq, &rf); 5492 } 5493 #else /* !CONFIG_NO_HZ_COMMON */ 5494 static inline void cpu_load_update_nohz(struct rq *this_rq, 5495 unsigned long curr_jiffies, 5496 unsigned long load) { } 5497 #endif /* CONFIG_NO_HZ_COMMON */ 5498 5499 static void cpu_load_update_periodic(struct rq *this_rq, unsigned long load) 5500 { 5501 #ifdef CONFIG_NO_HZ_COMMON 5502 /* See the mess around cpu_load_update_nohz(). */ 5503 this_rq->last_load_update_tick = READ_ONCE(jiffies); 5504 #endif 5505 cpu_load_update(this_rq, load, 1); 5506 } 5507 5508 /* 5509 * Called from scheduler_tick() 5510 */ 5511 void cpu_load_update_active(struct rq *this_rq) 5512 { 5513 unsigned long load = weighted_cpuload(this_rq); 5514 5515 if (tick_nohz_tick_stopped()) 5516 cpu_load_update_nohz(this_rq, READ_ONCE(jiffies), load); 5517 else 5518 cpu_load_update_periodic(this_rq, load); 5519 } 5520 5521 /* 5522 * Return a low guess at the load of a migration-source CPU weighted 5523 * according to the scheduling class and "nice" value. 5524 * 5525 * We want to under-estimate the load of migration sources, to 5526 * balance conservatively. 5527 */ 5528 static unsigned long source_load(int cpu, int type) 5529 { 5530 struct rq *rq = cpu_rq(cpu); 5531 unsigned long total = weighted_cpuload(rq); 5532 5533 if (type == 0 || !sched_feat(LB_BIAS)) 5534 return total; 5535 5536 return min(rq->cpu_load[type-1], total); 5537 } 5538 5539 /* 5540 * Return a high guess at the load of a migration-target CPU weighted 5541 * according to the scheduling class and "nice" value. 5542 */ 5543 static unsigned long target_load(int cpu, int type) 5544 { 5545 struct rq *rq = cpu_rq(cpu); 5546 unsigned long total = weighted_cpuload(rq); 5547 5548 if (type == 0 || !sched_feat(LB_BIAS)) 5549 return total; 5550 5551 return max(rq->cpu_load[type-1], total); 5552 } 5553 5554 static unsigned long capacity_of(int cpu) 5555 { 5556 return cpu_rq(cpu)->cpu_capacity; 5557 } 5558 5559 static unsigned long capacity_orig_of(int cpu) 5560 { 5561 return cpu_rq(cpu)->cpu_capacity_orig; 5562 } 5563 5564 static unsigned long cpu_avg_load_per_task(int cpu) 5565 { 5566 struct rq *rq = cpu_rq(cpu); 5567 unsigned long nr_running = READ_ONCE(rq->cfs.h_nr_running); 5568 unsigned long load_avg = weighted_cpuload(rq); 5569 5570 if (nr_running) 5571 return load_avg / nr_running; 5572 5573 return 0; 5574 } 5575 5576 static void record_wakee(struct task_struct *p) 5577 { 5578 /* 5579 * Only decay a single time; tasks that have less then 1 wakeup per 5580 * jiffy will not have built up many flips. 5581 */ 5582 if (time_after(jiffies, current->wakee_flip_decay_ts + HZ)) { 5583 current->wakee_flips >>= 1; 5584 current->wakee_flip_decay_ts = jiffies; 5585 } 5586 5587 if (current->last_wakee != p) { 5588 current->last_wakee = p; 5589 current->wakee_flips++; 5590 } 5591 } 5592 5593 /* 5594 * Detect M:N waker/wakee relationships via a switching-frequency heuristic. 5595 * 5596 * A waker of many should wake a different task than the one last awakened 5597 * at a frequency roughly N times higher than one of its wakees. 5598 * 5599 * In order to determine whether we should let the load spread vs consolidating 5600 * to shared cache, we look for a minimum 'flip' frequency of llc_size in one 5601 * partner, and a factor of lls_size higher frequency in the other. 5602 * 5603 * With both conditions met, we can be relatively sure that the relationship is 5604 * non-monogamous, with partner count exceeding socket size. 5605 * 5606 * Waker/wakee being client/server, worker/dispatcher, interrupt source or 5607 * whatever is irrelevant, spread criteria is apparent partner count exceeds 5608 * socket size. 5609 */ 5610 static int wake_wide(struct task_struct *p) 5611 { 5612 unsigned int master = current->wakee_flips; 5613 unsigned int slave = p->wakee_flips; 5614 int factor = this_cpu_read(sd_llc_size); 5615 5616 if (master < slave) 5617 swap(master, slave); 5618 if (slave < factor || master < slave * factor) 5619 return 0; 5620 return 1; 5621 } 5622 5623 /* 5624 * The purpose of wake_affine() is to quickly determine on which CPU we can run 5625 * soonest. For the purpose of speed we only consider the waking and previous 5626 * CPU. 5627 * 5628 * wake_affine_idle() - only considers 'now', it check if the waking CPU is 5629 * cache-affine and is (or will be) idle. 5630 * 5631 * wake_affine_weight() - considers the weight to reflect the average 5632 * scheduling latency of the CPUs. This seems to work 5633 * for the overloaded case. 5634 */ 5635 static int 5636 wake_affine_idle(int this_cpu, int prev_cpu, int sync) 5637 { 5638 /* 5639 * If this_cpu is idle, it implies the wakeup is from interrupt 5640 * context. Only allow the move if cache is shared. Otherwise an 5641 * interrupt intensive workload could force all tasks onto one 5642 * node depending on the IO topology or IRQ affinity settings. 5643 * 5644 * If the prev_cpu is idle and cache affine then avoid a migration. 5645 * There is no guarantee that the cache hot data from an interrupt 5646 * is more important than cache hot data on the prev_cpu and from 5647 * a cpufreq perspective, it's better to have higher utilisation 5648 * on one CPU. 5649 */ 5650 if (available_idle_cpu(this_cpu) && cpus_share_cache(this_cpu, prev_cpu)) 5651 return available_idle_cpu(prev_cpu) ? prev_cpu : this_cpu; 5652 5653 if (sync && cpu_rq(this_cpu)->nr_running == 1) 5654 return this_cpu; 5655 5656 return nr_cpumask_bits; 5657 } 5658 5659 static int 5660 wake_affine_weight(struct sched_domain *sd, struct task_struct *p, 5661 int this_cpu, int prev_cpu, int sync) 5662 { 5663 s64 this_eff_load, prev_eff_load; 5664 unsigned long task_load; 5665 5666 this_eff_load = target_load(this_cpu, sd->wake_idx); 5667 5668 if (sync) { 5669 unsigned long current_load = task_h_load(current); 5670 5671 if (current_load > this_eff_load) 5672 return this_cpu; 5673 5674 this_eff_load -= current_load; 5675 } 5676 5677 task_load = task_h_load(p); 5678 5679 this_eff_load += task_load; 5680 if (sched_feat(WA_BIAS)) 5681 this_eff_load *= 100; 5682 this_eff_load *= capacity_of(prev_cpu); 5683 5684 prev_eff_load = source_load(prev_cpu, sd->wake_idx); 5685 prev_eff_load -= task_load; 5686 if (sched_feat(WA_BIAS)) 5687 prev_eff_load *= 100 + (sd->imbalance_pct - 100) / 2; 5688 prev_eff_load *= capacity_of(this_cpu); 5689 5690 /* 5691 * If sync, adjust the weight of prev_eff_load such that if 5692 * prev_eff == this_eff that select_idle_sibling() will consider 5693 * stacking the wakee on top of the waker if no other CPU is 5694 * idle. 5695 */ 5696 if (sync) 5697 prev_eff_load += 1; 5698 5699 return this_eff_load < prev_eff_load ? this_cpu : nr_cpumask_bits; 5700 } 5701 5702 static int wake_affine(struct sched_domain *sd, struct task_struct *p, 5703 int this_cpu, int prev_cpu, int sync) 5704 { 5705 int target = nr_cpumask_bits; 5706 5707 if (sched_feat(WA_IDLE)) 5708 target = wake_affine_idle(this_cpu, prev_cpu, sync); 5709 5710 if (sched_feat(WA_WEIGHT) && target == nr_cpumask_bits) 5711 target = wake_affine_weight(sd, p, this_cpu, prev_cpu, sync); 5712 5713 schedstat_inc(p->se.statistics.nr_wakeups_affine_attempts); 5714 if (target == nr_cpumask_bits) 5715 return prev_cpu; 5716 5717 schedstat_inc(sd->ttwu_move_affine); 5718 schedstat_inc(p->se.statistics.nr_wakeups_affine); 5719 return target; 5720 } 5721 5722 static unsigned long cpu_util_without(int cpu, struct task_struct *p); 5723 5724 static unsigned long capacity_spare_without(int cpu, struct task_struct *p) 5725 { 5726 return max_t(long, capacity_of(cpu) - cpu_util_without(cpu, p), 0); 5727 } 5728 5729 /* 5730 * find_idlest_group finds and returns the least busy CPU group within the 5731 * domain. 5732 * 5733 * Assumes p is allowed on at least one CPU in sd. 5734 */ 5735 static struct sched_group * 5736 find_idlest_group(struct sched_domain *sd, struct task_struct *p, 5737 int this_cpu, int sd_flag) 5738 { 5739 struct sched_group *idlest = NULL, *group = sd->groups; 5740 struct sched_group *most_spare_sg = NULL; 5741 unsigned long min_runnable_load = ULONG_MAX; 5742 unsigned long this_runnable_load = ULONG_MAX; 5743 unsigned long min_avg_load = ULONG_MAX, this_avg_load = ULONG_MAX; 5744 unsigned long most_spare = 0, this_spare = 0; 5745 int load_idx = sd->forkexec_idx; 5746 int imbalance_scale = 100 + (sd->imbalance_pct-100)/2; 5747 unsigned long imbalance = scale_load_down(NICE_0_LOAD) * 5748 (sd->imbalance_pct-100) / 100; 5749 5750 if (sd_flag & SD_BALANCE_WAKE) 5751 load_idx = sd->wake_idx; 5752 5753 do { 5754 unsigned long load, avg_load, runnable_load; 5755 unsigned long spare_cap, max_spare_cap; 5756 int local_group; 5757 int i; 5758 5759 /* Skip over this group if it has no CPUs allowed */ 5760 if (!cpumask_intersects(sched_group_span(group), 5761 &p->cpus_allowed)) 5762 continue; 5763 5764 local_group = cpumask_test_cpu(this_cpu, 5765 sched_group_span(group)); 5766 5767 /* 5768 * Tally up the load of all CPUs in the group and find 5769 * the group containing the CPU with most spare capacity. 5770 */ 5771 avg_load = 0; 5772 runnable_load = 0; 5773 max_spare_cap = 0; 5774 5775 for_each_cpu(i, sched_group_span(group)) { 5776 /* Bias balancing toward CPUs of our domain */ 5777 if (local_group) 5778 load = source_load(i, load_idx); 5779 else 5780 load = target_load(i, load_idx); 5781 5782 runnable_load += load; 5783 5784 avg_load += cfs_rq_load_avg(&cpu_rq(i)->cfs); 5785 5786 spare_cap = capacity_spare_without(i, p); 5787 5788 if (spare_cap > max_spare_cap) 5789 max_spare_cap = spare_cap; 5790 } 5791 5792 /* Adjust by relative CPU capacity of the group */ 5793 avg_load = (avg_load * SCHED_CAPACITY_SCALE) / 5794 group->sgc->capacity; 5795 runnable_load = (runnable_load * SCHED_CAPACITY_SCALE) / 5796 group->sgc->capacity; 5797 5798 if (local_group) { 5799 this_runnable_load = runnable_load; 5800 this_avg_load = avg_load; 5801 this_spare = max_spare_cap; 5802 } else { 5803 if (min_runnable_load > (runnable_load + imbalance)) { 5804 /* 5805 * The runnable load is significantly smaller 5806 * so we can pick this new CPU: 5807 */ 5808 min_runnable_load = runnable_load; 5809 min_avg_load = avg_load; 5810 idlest = group; 5811 } else if ((runnable_load < (min_runnable_load + imbalance)) && 5812 (100*min_avg_load > imbalance_scale*avg_load)) { 5813 /* 5814 * The runnable loads are close so take the 5815 * blocked load into account through avg_load: 5816 */ 5817 min_avg_load = avg_load; 5818 idlest = group; 5819 } 5820 5821 if (most_spare < max_spare_cap) { 5822 most_spare = max_spare_cap; 5823 most_spare_sg = group; 5824 } 5825 } 5826 } while (group = group->next, group != sd->groups); 5827 5828 /* 5829 * The cross-over point between using spare capacity or least load 5830 * is too conservative for high utilization tasks on partially 5831 * utilized systems if we require spare_capacity > task_util(p), 5832 * so we allow for some task stuffing by using 5833 * spare_capacity > task_util(p)/2. 5834 * 5835 * Spare capacity can't be used for fork because the utilization has 5836 * not been set yet, we must first select a rq to compute the initial 5837 * utilization. 5838 */ 5839 if (sd_flag & SD_BALANCE_FORK) 5840 goto skip_spare; 5841 5842 if (this_spare > task_util(p) / 2 && 5843 imbalance_scale*this_spare > 100*most_spare) 5844 return NULL; 5845 5846 if (most_spare > task_util(p) / 2) 5847 return most_spare_sg; 5848 5849 skip_spare: 5850 if (!idlest) 5851 return NULL; 5852 5853 /* 5854 * When comparing groups across NUMA domains, it's possible for the 5855 * local domain to be very lightly loaded relative to the remote 5856 * domains but "imbalance" skews the comparison making remote CPUs 5857 * look much more favourable. When considering cross-domain, add 5858 * imbalance to the runnable load on the remote node and consider 5859 * staying local. 5860 */ 5861 if ((sd->flags & SD_NUMA) && 5862 min_runnable_load + imbalance >= this_runnable_load) 5863 return NULL; 5864 5865 if (min_runnable_load > (this_runnable_load + imbalance)) 5866 return NULL; 5867 5868 if ((this_runnable_load < (min_runnable_load + imbalance)) && 5869 (100*this_avg_load < imbalance_scale*min_avg_load)) 5870 return NULL; 5871 5872 return idlest; 5873 } 5874 5875 /* 5876 * find_idlest_group_cpu - find the idlest CPU among the CPUs in the group. 5877 */ 5878 static int 5879 find_idlest_group_cpu(struct sched_group *group, struct task_struct *p, int this_cpu) 5880 { 5881 unsigned long load, min_load = ULONG_MAX; 5882 unsigned int min_exit_latency = UINT_MAX; 5883 u64 latest_idle_timestamp = 0; 5884 int least_loaded_cpu = this_cpu; 5885 int shallowest_idle_cpu = -1; 5886 int i; 5887 5888 /* Check if we have any choice: */ 5889 if (group->group_weight == 1) 5890 return cpumask_first(sched_group_span(group)); 5891 5892 /* Traverse only the allowed CPUs */ 5893 for_each_cpu_and(i, sched_group_span(group), &p->cpus_allowed) { 5894 if (available_idle_cpu(i)) { 5895 struct rq *rq = cpu_rq(i); 5896 struct cpuidle_state *idle = idle_get_state(rq); 5897 if (idle && idle->exit_latency < min_exit_latency) { 5898 /* 5899 * We give priority to a CPU whose idle state 5900 * has the smallest exit latency irrespective 5901 * of any idle timestamp. 5902 */ 5903 min_exit_latency = idle->exit_latency; 5904 latest_idle_timestamp = rq->idle_stamp; 5905 shallowest_idle_cpu = i; 5906 } else if ((!idle || idle->exit_latency == min_exit_latency) && 5907 rq->idle_stamp > latest_idle_timestamp) { 5908 /* 5909 * If equal or no active idle state, then 5910 * the most recently idled CPU might have 5911 * a warmer cache. 5912 */ 5913 latest_idle_timestamp = rq->idle_stamp; 5914 shallowest_idle_cpu = i; 5915 } 5916 } else if (shallowest_idle_cpu == -1) { 5917 load = weighted_cpuload(cpu_rq(i)); 5918 if (load < min_load) { 5919 min_load = load; 5920 least_loaded_cpu = i; 5921 } 5922 } 5923 } 5924 5925 return shallowest_idle_cpu != -1 ? shallowest_idle_cpu : least_loaded_cpu; 5926 } 5927 5928 static inline int find_idlest_cpu(struct sched_domain *sd, struct task_struct *p, 5929 int cpu, int prev_cpu, int sd_flag) 5930 { 5931 int new_cpu = cpu; 5932 5933 if (!cpumask_intersects(sched_domain_span(sd), &p->cpus_allowed)) 5934 return prev_cpu; 5935 5936 /* 5937 * We need task's util for capacity_spare_without, sync it up to 5938 * prev_cpu's last_update_time. 5939 */ 5940 if (!(sd_flag & SD_BALANCE_FORK)) 5941 sync_entity_load_avg(&p->se); 5942 5943 while (sd) { 5944 struct sched_group *group; 5945 struct sched_domain *tmp; 5946 int weight; 5947 5948 if (!(sd->flags & sd_flag)) { 5949 sd = sd->child; 5950 continue; 5951 } 5952 5953 group = find_idlest_group(sd, p, cpu, sd_flag); 5954 if (!group) { 5955 sd = sd->child; 5956 continue; 5957 } 5958 5959 new_cpu = find_idlest_group_cpu(group, p, cpu); 5960 if (new_cpu == cpu) { 5961 /* Now try balancing at a lower domain level of 'cpu': */ 5962 sd = sd->child; 5963 continue; 5964 } 5965 5966 /* Now try balancing at a lower domain level of 'new_cpu': */ 5967 cpu = new_cpu; 5968 weight = sd->span_weight; 5969 sd = NULL; 5970 for_each_domain(cpu, tmp) { 5971 if (weight <= tmp->span_weight) 5972 break; 5973 if (tmp->flags & sd_flag) 5974 sd = tmp; 5975 } 5976 } 5977 5978 return new_cpu; 5979 } 5980 5981 #ifdef CONFIG_SCHED_SMT 5982 DEFINE_STATIC_KEY_FALSE(sched_smt_present); 5983 EXPORT_SYMBOL_GPL(sched_smt_present); 5984 5985 static inline void set_idle_cores(int cpu, int val) 5986 { 5987 struct sched_domain_shared *sds; 5988 5989 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); 5990 if (sds) 5991 WRITE_ONCE(sds->has_idle_cores, val); 5992 } 5993 5994 static inline bool test_idle_cores(int cpu, bool def) 5995 { 5996 struct sched_domain_shared *sds; 5997 5998 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); 5999 if (sds) 6000 return READ_ONCE(sds->has_idle_cores); 6001 6002 return def; 6003 } 6004 6005 /* 6006 * Scans the local SMT mask to see if the entire core is idle, and records this 6007 * information in sd_llc_shared->has_idle_cores. 6008 * 6009 * Since SMT siblings share all cache levels, inspecting this limited remote 6010 * state should be fairly cheap. 6011 */ 6012 void __update_idle_core(struct rq *rq) 6013 { 6014 int core = cpu_of(rq); 6015 int cpu; 6016 6017 rcu_read_lock(); 6018 if (test_idle_cores(core, true)) 6019 goto unlock; 6020 6021 for_each_cpu(cpu, cpu_smt_mask(core)) { 6022 if (cpu == core) 6023 continue; 6024 6025 if (!available_idle_cpu(cpu)) 6026 goto unlock; 6027 } 6028 6029 set_idle_cores(core, 1); 6030 unlock: 6031 rcu_read_unlock(); 6032 } 6033 6034 /* 6035 * Scan the entire LLC domain for idle cores; this dynamically switches off if 6036 * there are no idle cores left in the system; tracked through 6037 * sd_llc->shared->has_idle_cores and enabled through update_idle_core() above. 6038 */ 6039 static int select_idle_core(struct task_struct *p, struct sched_domain *sd, int target) 6040 { 6041 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_idle_mask); 6042 int core, cpu; 6043 6044 if (!static_branch_likely(&sched_smt_present)) 6045 return -1; 6046 6047 if (!test_idle_cores(target, false)) 6048 return -1; 6049 6050 cpumask_and(cpus, sched_domain_span(sd), &p->cpus_allowed); 6051 6052 for_each_cpu_wrap(core, cpus, target) { 6053 bool idle = true; 6054 6055 for_each_cpu(cpu, cpu_smt_mask(core)) { 6056 cpumask_clear_cpu(cpu, cpus); 6057 if (!available_idle_cpu(cpu)) 6058 idle = false; 6059 } 6060 6061 if (idle) 6062 return core; 6063 } 6064 6065 /* 6066 * Failed to find an idle core; stop looking for one. 6067 */ 6068 set_idle_cores(target, 0); 6069 6070 return -1; 6071 } 6072 6073 /* 6074 * Scan the local SMT mask for idle CPUs. 6075 */ 6076 static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target) 6077 { 6078 int cpu; 6079 6080 if (!static_branch_likely(&sched_smt_present)) 6081 return -1; 6082 6083 for_each_cpu(cpu, cpu_smt_mask(target)) { 6084 if (!cpumask_test_cpu(cpu, &p->cpus_allowed)) 6085 continue; 6086 if (available_idle_cpu(cpu)) 6087 return cpu; 6088 } 6089 6090 return -1; 6091 } 6092 6093 #else /* CONFIG_SCHED_SMT */ 6094 6095 static inline int select_idle_core(struct task_struct *p, struct sched_domain *sd, int target) 6096 { 6097 return -1; 6098 } 6099 6100 static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target) 6101 { 6102 return -1; 6103 } 6104 6105 #endif /* CONFIG_SCHED_SMT */ 6106 6107 /* 6108 * Scan the LLC domain for idle CPUs; this is dynamically regulated by 6109 * comparing the average scan cost (tracked in sd->avg_scan_cost) against the 6110 * average idle time for this rq (as found in rq->avg_idle). 6111 */ 6112 static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int target) 6113 { 6114 struct sched_domain *this_sd; 6115 u64 avg_cost, avg_idle; 6116 u64 time, cost; 6117 s64 delta; 6118 int cpu, nr = INT_MAX; 6119 6120 this_sd = rcu_dereference(*this_cpu_ptr(&sd_llc)); 6121 if (!this_sd) 6122 return -1; 6123 6124 /* 6125 * Due to large variance we need a large fuzz factor; hackbench in 6126 * particularly is sensitive here. 6127 */ 6128 avg_idle = this_rq()->avg_idle / 512; 6129 avg_cost = this_sd->avg_scan_cost + 1; 6130 6131 if (sched_feat(SIS_AVG_CPU) && avg_idle < avg_cost) 6132 return -1; 6133 6134 if (sched_feat(SIS_PROP)) { 6135 u64 span_avg = sd->span_weight * avg_idle; 6136 if (span_avg > 4*avg_cost) 6137 nr = div_u64(span_avg, avg_cost); 6138 else 6139 nr = 4; 6140 } 6141 6142 time = local_clock(); 6143 6144 for_each_cpu_wrap(cpu, sched_domain_span(sd), target) { 6145 if (!--nr) 6146 return -1; 6147 if (!cpumask_test_cpu(cpu, &p->cpus_allowed)) 6148 continue; 6149 if (available_idle_cpu(cpu)) 6150 break; 6151 } 6152 6153 time = local_clock() - time; 6154 cost = this_sd->avg_scan_cost; 6155 delta = (s64)(time - cost) / 8; 6156 this_sd->avg_scan_cost += delta; 6157 6158 return cpu; 6159 } 6160 6161 /* 6162 * Try and locate an idle core/thread in the LLC cache domain. 6163 */ 6164 static int select_idle_sibling(struct task_struct *p, int prev, int target) 6165 { 6166 struct sched_domain *sd; 6167 int i, recent_used_cpu; 6168 6169 if (available_idle_cpu(target)) 6170 return target; 6171 6172 /* 6173 * If the previous CPU is cache affine and idle, don't be stupid: 6174 */ 6175 if (prev != target && cpus_share_cache(prev, target) && available_idle_cpu(prev)) 6176 return prev; 6177 6178 /* Check a recently used CPU as a potential idle candidate: */ 6179 recent_used_cpu = p->recent_used_cpu; 6180 if (recent_used_cpu != prev && 6181 recent_used_cpu != target && 6182 cpus_share_cache(recent_used_cpu, target) && 6183 available_idle_cpu(recent_used_cpu) && 6184 cpumask_test_cpu(p->recent_used_cpu, &p->cpus_allowed)) { 6185 /* 6186 * Replace recent_used_cpu with prev as it is a potential 6187 * candidate for the next wake: 6188 */ 6189 p->recent_used_cpu = prev; 6190 return recent_used_cpu; 6191 } 6192 6193 sd = rcu_dereference(per_cpu(sd_llc, target)); 6194 if (!sd) 6195 return target; 6196 6197 i = select_idle_core(p, sd, target); 6198 if ((unsigned)i < nr_cpumask_bits) 6199 return i; 6200 6201 i = select_idle_cpu(p, sd, target); 6202 if ((unsigned)i < nr_cpumask_bits) 6203 return i; 6204 6205 i = select_idle_smt(p, sd, target); 6206 if ((unsigned)i < nr_cpumask_bits) 6207 return i; 6208 6209 return target; 6210 } 6211 6212 /** 6213 * Amount of capacity of a CPU that is (estimated to be) used by CFS tasks 6214 * @cpu: the CPU to get the utilization of 6215 * 6216 * The unit of the return value must be the one of capacity so we can compare 6217 * the utilization with the capacity of the CPU that is available for CFS task 6218 * (ie cpu_capacity). 6219 * 6220 * cfs_rq.avg.util_avg is the sum of running time of runnable tasks plus the 6221 * recent utilization of currently non-runnable tasks on a CPU. It represents 6222 * the amount of utilization of a CPU in the range [0..capacity_orig] where 6223 * capacity_orig is the cpu_capacity available at the highest frequency 6224 * (arch_scale_freq_capacity()). 6225 * The utilization of a CPU converges towards a sum equal to or less than the 6226 * current capacity (capacity_curr <= capacity_orig) of the CPU because it is 6227 * the running time on this CPU scaled by capacity_curr. 6228 * 6229 * The estimated utilization of a CPU is defined to be the maximum between its 6230 * cfs_rq.avg.util_avg and the sum of the estimated utilization of the tasks 6231 * currently RUNNABLE on that CPU. 6232 * This allows to properly represent the expected utilization of a CPU which 6233 * has just got a big task running since a long sleep period. At the same time 6234 * however it preserves the benefits of the "blocked utilization" in 6235 * describing the potential for other tasks waking up on the same CPU. 6236 * 6237 * Nevertheless, cfs_rq.avg.util_avg can be higher than capacity_curr or even 6238 * higher than capacity_orig because of unfortunate rounding in 6239 * cfs.avg.util_avg or just after migrating tasks and new task wakeups until 6240 * the average stabilizes with the new running time. We need to check that the 6241 * utilization stays within the range of [0..capacity_orig] and cap it if 6242 * necessary. Without utilization capping, a group could be seen as overloaded 6243 * (CPU0 utilization at 121% + CPU1 utilization at 80%) whereas CPU1 has 20% of 6244 * available capacity. We allow utilization to overshoot capacity_curr (but not 6245 * capacity_orig) as it useful for predicting the capacity required after task 6246 * migrations (scheduler-driven DVFS). 6247 * 6248 * Return: the (estimated) utilization for the specified CPU 6249 */ 6250 static inline unsigned long cpu_util(int cpu) 6251 { 6252 struct cfs_rq *cfs_rq; 6253 unsigned int util; 6254 6255 cfs_rq = &cpu_rq(cpu)->cfs; 6256 util = READ_ONCE(cfs_rq->avg.util_avg); 6257 6258 if (sched_feat(UTIL_EST)) 6259 util = max(util, READ_ONCE(cfs_rq->avg.util_est.enqueued)); 6260 6261 return min_t(unsigned long, util, capacity_orig_of(cpu)); 6262 } 6263 6264 /* 6265 * cpu_util_without: compute cpu utilization without any contributions from *p 6266 * @cpu: the CPU which utilization is requested 6267 * @p: the task which utilization should be discounted 6268 * 6269 * The utilization of a CPU is defined by the utilization of tasks currently 6270 * enqueued on that CPU as well as tasks which are currently sleeping after an 6271 * execution on that CPU. 6272 * 6273 * This method returns the utilization of the specified CPU by discounting the 6274 * utilization of the specified task, whenever the task is currently 6275 * contributing to the CPU utilization. 6276 */ 6277 static unsigned long cpu_util_without(int cpu, struct task_struct *p) 6278 { 6279 struct cfs_rq *cfs_rq; 6280 unsigned int util; 6281 6282 /* Task has no contribution or is new */ 6283 if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 6284 return cpu_util(cpu); 6285 6286 cfs_rq = &cpu_rq(cpu)->cfs; 6287 util = READ_ONCE(cfs_rq->avg.util_avg); 6288 6289 /* Discount task's util from CPU's util */ 6290 lsub_positive(&util, task_util(p)); 6291 6292 /* 6293 * Covered cases: 6294 * 6295 * a) if *p is the only task sleeping on this CPU, then: 6296 * cpu_util (== task_util) > util_est (== 0) 6297 * and thus we return: 6298 * cpu_util_without = (cpu_util - task_util) = 0 6299 * 6300 * b) if other tasks are SLEEPING on this CPU, which is now exiting 6301 * IDLE, then: 6302 * cpu_util >= task_util 6303 * cpu_util > util_est (== 0) 6304 * and thus we discount *p's blocked utilization to return: 6305 * cpu_util_without = (cpu_util - task_util) >= 0 6306 * 6307 * c) if other tasks are RUNNABLE on that CPU and 6308 * util_est > cpu_util 6309 * then we use util_est since it returns a more restrictive 6310 * estimation of the spare capacity on that CPU, by just 6311 * considering the expected utilization of tasks already 6312 * runnable on that CPU. 6313 * 6314 * Cases a) and b) are covered by the above code, while case c) is 6315 * covered by the following code when estimated utilization is 6316 * enabled. 6317 */ 6318 if (sched_feat(UTIL_EST)) { 6319 unsigned int estimated = 6320 READ_ONCE(cfs_rq->avg.util_est.enqueued); 6321 6322 /* 6323 * Despite the following checks we still have a small window 6324 * for a possible race, when an execl's select_task_rq_fair() 6325 * races with LB's detach_task(): 6326 * 6327 * detach_task() 6328 * p->on_rq = TASK_ON_RQ_MIGRATING; 6329 * ---------------------------------- A 6330 * deactivate_task() \ 6331 * dequeue_task() + RaceTime 6332 * util_est_dequeue() / 6333 * ---------------------------------- B 6334 * 6335 * The additional check on "current == p" it's required to 6336 * properly fix the execl regression and it helps in further 6337 * reducing the chances for the above race. 6338 */ 6339 if (unlikely(task_on_rq_queued(p) || current == p)) 6340 lsub_positive(&estimated, _task_util_est(p)); 6341 6342 util = max(util, estimated); 6343 } 6344 6345 /* 6346 * Utilization (estimated) can exceed the CPU capacity, thus let's 6347 * clamp to the maximum CPU capacity to ensure consistency with 6348 * the cpu_util call. 6349 */ 6350 return min_t(unsigned long, util, capacity_orig_of(cpu)); 6351 } 6352 6353 /* 6354 * Disable WAKE_AFFINE in the case where task @p doesn't fit in the 6355 * capacity of either the waking CPU @cpu or the previous CPU @prev_cpu. 6356 * 6357 * In that case WAKE_AFFINE doesn't make sense and we'll let 6358 * BALANCE_WAKE sort things out. 6359 */ 6360 static int wake_cap(struct task_struct *p, int cpu, int prev_cpu) 6361 { 6362 long min_cap, max_cap; 6363 6364 if (!static_branch_unlikely(&sched_asym_cpucapacity)) 6365 return 0; 6366 6367 min_cap = min(capacity_orig_of(prev_cpu), capacity_orig_of(cpu)); 6368 max_cap = cpu_rq(cpu)->rd->max_cpu_capacity; 6369 6370 /* Minimum capacity is close to max, no need to abort wake_affine */ 6371 if (max_cap - min_cap < max_cap >> 3) 6372 return 0; 6373 6374 /* Bring task utilization in sync with prev_cpu */ 6375 sync_entity_load_avg(&p->se); 6376 6377 return !task_fits_capacity(p, min_cap); 6378 } 6379 6380 /* 6381 * Predicts what cpu_util(@cpu) would return if @p was migrated (and enqueued) 6382 * to @dst_cpu. 6383 */ 6384 static unsigned long cpu_util_next(int cpu, struct task_struct *p, int dst_cpu) 6385 { 6386 struct cfs_rq *cfs_rq = &cpu_rq(cpu)->cfs; 6387 unsigned long util_est, util = READ_ONCE(cfs_rq->avg.util_avg); 6388 6389 /* 6390 * If @p migrates from @cpu to another, remove its contribution. Or, 6391 * if @p migrates from another CPU to @cpu, add its contribution. In 6392 * the other cases, @cpu is not impacted by the migration, so the 6393 * util_avg should already be correct. 6394 */ 6395 if (task_cpu(p) == cpu && dst_cpu != cpu) 6396 sub_positive(&util, task_util(p)); 6397 else if (task_cpu(p) != cpu && dst_cpu == cpu) 6398 util += task_util(p); 6399 6400 if (sched_feat(UTIL_EST)) { 6401 util_est = READ_ONCE(cfs_rq->avg.util_est.enqueued); 6402 6403 /* 6404 * During wake-up, the task isn't enqueued yet and doesn't 6405 * appear in the cfs_rq->avg.util_est.enqueued of any rq, 6406 * so just add it (if needed) to "simulate" what will be 6407 * cpu_util() after the task has been enqueued. 6408 */ 6409 if (dst_cpu == cpu) 6410 util_est += _task_util_est(p); 6411 6412 util = max(util, util_est); 6413 } 6414 6415 return min(util, capacity_orig_of(cpu)); 6416 } 6417 6418 /* 6419 * compute_energy(): Estimates the energy that would be consumed if @p was 6420 * migrated to @dst_cpu. compute_energy() predicts what will be the utilization 6421 * landscape of the * CPUs after the task migration, and uses the Energy Model 6422 * to compute what would be the energy if we decided to actually migrate that 6423 * task. 6424 */ 6425 static long 6426 compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd) 6427 { 6428 long util, max_util, sum_util, energy = 0; 6429 int cpu; 6430 6431 for (; pd; pd = pd->next) { 6432 max_util = sum_util = 0; 6433 /* 6434 * The capacity state of CPUs of the current rd can be driven by 6435 * CPUs of another rd if they belong to the same performance 6436 * domain. So, account for the utilization of these CPUs too 6437 * by masking pd with cpu_online_mask instead of the rd span. 6438 * 6439 * If an entire performance domain is outside of the current rd, 6440 * it will not appear in its pd list and will not be accounted 6441 * by compute_energy(). 6442 */ 6443 for_each_cpu_and(cpu, perf_domain_span(pd), cpu_online_mask) { 6444 util = cpu_util_next(cpu, p, dst_cpu); 6445 util = schedutil_energy_util(cpu, util); 6446 max_util = max(util, max_util); 6447 sum_util += util; 6448 } 6449 6450 energy += em_pd_energy(pd->em_pd, max_util, sum_util); 6451 } 6452 6453 return energy; 6454 } 6455 6456 /* 6457 * find_energy_efficient_cpu(): Find most energy-efficient target CPU for the 6458 * waking task. find_energy_efficient_cpu() looks for the CPU with maximum 6459 * spare capacity in each performance domain and uses it as a potential 6460 * candidate to execute the task. Then, it uses the Energy Model to figure 6461 * out which of the CPU candidates is the most energy-efficient. 6462 * 6463 * The rationale for this heuristic is as follows. In a performance domain, 6464 * all the most energy efficient CPU candidates (according to the Energy 6465 * Model) are those for which we'll request a low frequency. When there are 6466 * several CPUs for which the frequency request will be the same, we don't 6467 * have enough data to break the tie between them, because the Energy Model 6468 * only includes active power costs. With this model, if we assume that 6469 * frequency requests follow utilization (e.g. using schedutil), the CPU with 6470 * the maximum spare capacity in a performance domain is guaranteed to be among 6471 * the best candidates of the performance domain. 6472 * 6473 * In practice, it could be preferable from an energy standpoint to pack 6474 * small tasks on a CPU in order to let other CPUs go in deeper idle states, 6475 * but that could also hurt our chances to go cluster idle, and we have no 6476 * ways to tell with the current Energy Model if this is actually a good 6477 * idea or not. So, find_energy_efficient_cpu() basically favors 6478 * cluster-packing, and spreading inside a cluster. That should at least be 6479 * a good thing for latency, and this is consistent with the idea that most 6480 * of the energy savings of EAS come from the asymmetry of the system, and 6481 * not so much from breaking the tie between identical CPUs. That's also the 6482 * reason why EAS is enabled in the topology code only for systems where 6483 * SD_ASYM_CPUCAPACITY is set. 6484 * 6485 * NOTE: Forkees are not accepted in the energy-aware wake-up path because 6486 * they don't have any useful utilization data yet and it's not possible to 6487 * forecast their impact on energy consumption. Consequently, they will be 6488 * placed by find_idlest_cpu() on the least loaded CPU, which might turn out 6489 * to be energy-inefficient in some use-cases. The alternative would be to 6490 * bias new tasks towards specific types of CPUs first, or to try to infer 6491 * their util_avg from the parent task, but those heuristics could hurt 6492 * other use-cases too. So, until someone finds a better way to solve this, 6493 * let's keep things simple by re-using the existing slow path. 6494 */ 6495 6496 static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu) 6497 { 6498 unsigned long prev_energy = ULONG_MAX, best_energy = ULONG_MAX; 6499 struct root_domain *rd = cpu_rq(smp_processor_id())->rd; 6500 int cpu, best_energy_cpu = prev_cpu; 6501 struct perf_domain *head, *pd; 6502 unsigned long cpu_cap, util; 6503 struct sched_domain *sd; 6504 6505 rcu_read_lock(); 6506 pd = rcu_dereference(rd->pd); 6507 if (!pd || READ_ONCE(rd->overutilized)) 6508 goto fail; 6509 head = pd; 6510 6511 /* 6512 * Energy-aware wake-up happens on the lowest sched_domain starting 6513 * from sd_asym_cpucapacity spanning over this_cpu and prev_cpu. 6514 */ 6515 sd = rcu_dereference(*this_cpu_ptr(&sd_asym_cpucapacity)); 6516 while (sd && !cpumask_test_cpu(prev_cpu, sched_domain_span(sd))) 6517 sd = sd->parent; 6518 if (!sd) 6519 goto fail; 6520 6521 sync_entity_load_avg(&p->se); 6522 if (!task_util_est(p)) 6523 goto unlock; 6524 6525 for (; pd; pd = pd->next) { 6526 unsigned long cur_energy, spare_cap, max_spare_cap = 0; 6527 int max_spare_cap_cpu = -1; 6528 6529 for_each_cpu_and(cpu, perf_domain_span(pd), sched_domain_span(sd)) { 6530 if (!cpumask_test_cpu(cpu, &p->cpus_allowed)) 6531 continue; 6532 6533 /* Skip CPUs that will be overutilized. */ 6534 util = cpu_util_next(cpu, p, cpu); 6535 cpu_cap = capacity_of(cpu); 6536 if (cpu_cap * 1024 < util * capacity_margin) 6537 continue; 6538 6539 /* Always use prev_cpu as a candidate. */ 6540 if (cpu == prev_cpu) { 6541 prev_energy = compute_energy(p, prev_cpu, head); 6542 best_energy = min(best_energy, prev_energy); 6543 continue; 6544 } 6545 6546 /* 6547 * Find the CPU with the maximum spare capacity in 6548 * the performance domain 6549 */ 6550 spare_cap = cpu_cap - util; 6551 if (spare_cap > max_spare_cap) { 6552 max_spare_cap = spare_cap; 6553 max_spare_cap_cpu = cpu; 6554 } 6555 } 6556 6557 /* Evaluate the energy impact of using this CPU. */ 6558 if (max_spare_cap_cpu >= 0) { 6559 cur_energy = compute_energy(p, max_spare_cap_cpu, head); 6560 if (cur_energy < best_energy) { 6561 best_energy = cur_energy; 6562 best_energy_cpu = max_spare_cap_cpu; 6563 } 6564 } 6565 } 6566 unlock: 6567 rcu_read_unlock(); 6568 6569 /* 6570 * Pick the best CPU if prev_cpu cannot be used, or if it saves at 6571 * least 6% of the energy used by prev_cpu. 6572 */ 6573 if (prev_energy == ULONG_MAX) 6574 return best_energy_cpu; 6575 6576 if ((prev_energy - best_energy) > (prev_energy >> 4)) 6577 return best_energy_cpu; 6578 6579 return prev_cpu; 6580 6581 fail: 6582 rcu_read_unlock(); 6583 6584 return -1; 6585 } 6586 6587 /* 6588 * select_task_rq_fair: Select target runqueue for the waking task in domains 6589 * that have the 'sd_flag' flag set. In practice, this is SD_BALANCE_WAKE, 6590 * SD_BALANCE_FORK, or SD_BALANCE_EXEC. 6591 * 6592 * Balances load by selecting the idlest CPU in the idlest group, or under 6593 * certain conditions an idle sibling CPU if the domain has SD_WAKE_AFFINE set. 6594 * 6595 * Returns the target CPU number. 6596 * 6597 * preempt must be disabled. 6598 */ 6599 static int 6600 select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_flags) 6601 { 6602 struct sched_domain *tmp, *sd = NULL; 6603 int cpu = smp_processor_id(); 6604 int new_cpu = prev_cpu; 6605 int want_affine = 0; 6606 int sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING); 6607 6608 if (sd_flag & SD_BALANCE_WAKE) { 6609 record_wakee(p); 6610 6611 if (static_branch_unlikely(&sched_energy_present)) { 6612 new_cpu = find_energy_efficient_cpu(p, prev_cpu); 6613 if (new_cpu >= 0) 6614 return new_cpu; 6615 new_cpu = prev_cpu; 6616 } 6617 6618 want_affine = !wake_wide(p) && !wake_cap(p, cpu, prev_cpu) && 6619 cpumask_test_cpu(cpu, &p->cpus_allowed); 6620 } 6621 6622 rcu_read_lock(); 6623 for_each_domain(cpu, tmp) { 6624 if (!(tmp->flags & SD_LOAD_BALANCE)) 6625 break; 6626 6627 /* 6628 * If both 'cpu' and 'prev_cpu' are part of this domain, 6629 * cpu is a valid SD_WAKE_AFFINE target. 6630 */ 6631 if (want_affine && (tmp->flags & SD_WAKE_AFFINE) && 6632 cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) { 6633 if (cpu != prev_cpu) 6634 new_cpu = wake_affine(tmp, p, cpu, prev_cpu, sync); 6635 6636 sd = NULL; /* Prefer wake_affine over balance flags */ 6637 break; 6638 } 6639 6640 if (tmp->flags & sd_flag) 6641 sd = tmp; 6642 else if (!want_affine) 6643 break; 6644 } 6645 6646 if (unlikely(sd)) { 6647 /* Slow path */ 6648 new_cpu = find_idlest_cpu(sd, p, cpu, prev_cpu, sd_flag); 6649 } else if (sd_flag & SD_BALANCE_WAKE) { /* XXX always ? */ 6650 /* Fast path */ 6651 6652 new_cpu = select_idle_sibling(p, prev_cpu, new_cpu); 6653 6654 if (want_affine) 6655 current->recent_used_cpu = cpu; 6656 } 6657 rcu_read_unlock(); 6658 6659 return new_cpu; 6660 } 6661 6662 static void detach_entity_cfs_rq(struct sched_entity *se); 6663 6664 /* 6665 * Called immediately before a task is migrated to a new CPU; task_cpu(p) and 6666 * cfs_rq_of(p) references at time of call are still valid and identify the 6667 * previous CPU. The caller guarantees p->pi_lock or task_rq(p)->lock is held. 6668 */ 6669 static void migrate_task_rq_fair(struct task_struct *p, int new_cpu) 6670 { 6671 /* 6672 * As blocked tasks retain absolute vruntime the migration needs to 6673 * deal with this by subtracting the old and adding the new 6674 * min_vruntime -- the latter is done by enqueue_entity() when placing 6675 * the task on the new runqueue. 6676 */ 6677 if (p->state == TASK_WAKING) { 6678 struct sched_entity *se = &p->se; 6679 struct cfs_rq *cfs_rq = cfs_rq_of(se); 6680 u64 min_vruntime; 6681 6682 #ifndef CONFIG_64BIT 6683 u64 min_vruntime_copy; 6684 6685 do { 6686 min_vruntime_copy = cfs_rq->min_vruntime_copy; 6687 smp_rmb(); 6688 min_vruntime = cfs_rq->min_vruntime; 6689 } while (min_vruntime != min_vruntime_copy); 6690 #else 6691 min_vruntime = cfs_rq->min_vruntime; 6692 #endif 6693 6694 se->vruntime -= min_vruntime; 6695 } 6696 6697 if (p->on_rq == TASK_ON_RQ_MIGRATING) { 6698 /* 6699 * In case of TASK_ON_RQ_MIGRATING we in fact hold the 'old' 6700 * rq->lock and can modify state directly. 6701 */ 6702 lockdep_assert_held(&task_rq(p)->lock); 6703 detach_entity_cfs_rq(&p->se); 6704 6705 } else { 6706 /* 6707 * We are supposed to update the task to "current" time, then 6708 * its up to date and ready to go to new CPU/cfs_rq. But we 6709 * have difficulty in getting what current time is, so simply 6710 * throw away the out-of-date time. This will result in the 6711 * wakee task is less decayed, but giving the wakee more load 6712 * sounds not bad. 6713 */ 6714 remove_entity_load_avg(&p->se); 6715 } 6716 6717 /* Tell new CPU we are migrated */ 6718 p->se.avg.last_update_time = 0; 6719 6720 /* We have migrated, no longer consider this task hot */ 6721 p->se.exec_start = 0; 6722 6723 update_scan_period(p, new_cpu); 6724 } 6725 6726 static void task_dead_fair(struct task_struct *p) 6727 { 6728 remove_entity_load_avg(&p->se); 6729 } 6730 #endif /* CONFIG_SMP */ 6731 6732 static unsigned long wakeup_gran(struct sched_entity *se) 6733 { 6734 unsigned long gran = sysctl_sched_wakeup_granularity; 6735 6736 /* 6737 * Since its curr running now, convert the gran from real-time 6738 * to virtual-time in his units. 6739 * 6740 * By using 'se' instead of 'curr' we penalize light tasks, so 6741 * they get preempted easier. That is, if 'se' < 'curr' then 6742 * the resulting gran will be larger, therefore penalizing the 6743 * lighter, if otoh 'se' > 'curr' then the resulting gran will 6744 * be smaller, again penalizing the lighter task. 6745 * 6746 * This is especially important for buddies when the leftmost 6747 * task is higher priority than the buddy. 6748 */ 6749 return calc_delta_fair(gran, se); 6750 } 6751 6752 /* 6753 * Should 'se' preempt 'curr'. 6754 * 6755 * |s1 6756 * |s2 6757 * |s3 6758 * g 6759 * |<--->|c 6760 * 6761 * w(c, s1) = -1 6762 * w(c, s2) = 0 6763 * w(c, s3) = 1 6764 * 6765 */ 6766 static int 6767 wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se) 6768 { 6769 s64 gran, vdiff = curr->vruntime - se->vruntime; 6770 6771 if (vdiff <= 0) 6772 return -1; 6773 6774 gran = wakeup_gran(se); 6775 if (vdiff > gran) 6776 return 1; 6777 6778 return 0; 6779 } 6780 6781 static void set_last_buddy(struct sched_entity *se) 6782 { 6783 if (entity_is_task(se) && unlikely(task_has_idle_policy(task_of(se)))) 6784 return; 6785 6786 for_each_sched_entity(se) { 6787 if (SCHED_WARN_ON(!se->on_rq)) 6788 return; 6789 cfs_rq_of(se)->last = se; 6790 } 6791 } 6792 6793 static void set_next_buddy(struct sched_entity *se) 6794 { 6795 if (entity_is_task(se) && unlikely(task_has_idle_policy(task_of(se)))) 6796 return; 6797 6798 for_each_sched_entity(se) { 6799 if (SCHED_WARN_ON(!se->on_rq)) 6800 return; 6801 cfs_rq_of(se)->next = se; 6802 } 6803 } 6804 6805 static void set_skip_buddy(struct sched_entity *se) 6806 { 6807 for_each_sched_entity(se) 6808 cfs_rq_of(se)->skip = se; 6809 } 6810 6811 /* 6812 * Preempt the current task with a newly woken task if needed: 6813 */ 6814 static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_flags) 6815 { 6816 struct task_struct *curr = rq->curr; 6817 struct sched_entity *se = &curr->se, *pse = &p->se; 6818 struct cfs_rq *cfs_rq = task_cfs_rq(curr); 6819 int scale = cfs_rq->nr_running >= sched_nr_latency; 6820 int next_buddy_marked = 0; 6821 6822 if (unlikely(se == pse)) 6823 return; 6824 6825 /* 6826 * This is possible from callers such as attach_tasks(), in which we 6827 * unconditionally check_prempt_curr() after an enqueue (which may have 6828 * lead to a throttle). This both saves work and prevents false 6829 * next-buddy nomination below. 6830 */ 6831 if (unlikely(throttled_hierarchy(cfs_rq_of(pse)))) 6832 return; 6833 6834 if (sched_feat(NEXT_BUDDY) && scale && !(wake_flags & WF_FORK)) { 6835 set_next_buddy(pse); 6836 next_buddy_marked = 1; 6837 } 6838 6839 /* 6840 * We can come here with TIF_NEED_RESCHED already set from new task 6841 * wake up path. 6842 * 6843 * Note: this also catches the edge-case of curr being in a throttled 6844 * group (e.g. via set_curr_task), since update_curr() (in the 6845 * enqueue of curr) will have resulted in resched being set. This 6846 * prevents us from potentially nominating it as a false LAST_BUDDY 6847 * below. 6848 */ 6849 if (test_tsk_need_resched(curr)) 6850 return; 6851 6852 /* Idle tasks are by definition preempted by non-idle tasks. */ 6853 if (unlikely(task_has_idle_policy(curr)) && 6854 likely(!task_has_idle_policy(p))) 6855 goto preempt; 6856 6857 /* 6858 * Batch and idle tasks do not preempt non-idle tasks (their preemption 6859 * is driven by the tick): 6860 */ 6861 if (unlikely(p->policy != SCHED_NORMAL) || !sched_feat(WAKEUP_PREEMPTION)) 6862 return; 6863 6864 find_matching_se(&se, &pse); 6865 update_curr(cfs_rq_of(se)); 6866 BUG_ON(!pse); 6867 if (wakeup_preempt_entity(se, pse) == 1) { 6868 /* 6869 * Bias pick_next to pick the sched entity that is 6870 * triggering this preemption. 6871 */ 6872 if (!next_buddy_marked) 6873 set_next_buddy(pse); 6874 goto preempt; 6875 } 6876 6877 return; 6878 6879 preempt: 6880 resched_curr(rq); 6881 /* 6882 * Only set the backward buddy when the current task is still 6883 * on the rq. This can happen when a wakeup gets interleaved 6884 * with schedule on the ->pre_schedule() or idle_balance() 6885 * point, either of which can * drop the rq lock. 6886 * 6887 * Also, during early boot the idle thread is in the fair class, 6888 * for obvious reasons its a bad idea to schedule back to it. 6889 */ 6890 if (unlikely(!se->on_rq || curr == rq->idle)) 6891 return; 6892 6893 if (sched_feat(LAST_BUDDY) && scale && entity_is_task(se)) 6894 set_last_buddy(se); 6895 } 6896 6897 static struct task_struct * 6898 pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 6899 { 6900 struct cfs_rq *cfs_rq = &rq->cfs; 6901 struct sched_entity *se; 6902 struct task_struct *p; 6903 int new_tasks; 6904 6905 again: 6906 if (!cfs_rq->nr_running) 6907 goto idle; 6908 6909 #ifdef CONFIG_FAIR_GROUP_SCHED 6910 if (prev->sched_class != &fair_sched_class) 6911 goto simple; 6912 6913 /* 6914 * Because of the set_next_buddy() in dequeue_task_fair() it is rather 6915 * likely that a next task is from the same cgroup as the current. 6916 * 6917 * Therefore attempt to avoid putting and setting the entire cgroup 6918 * hierarchy, only change the part that actually changes. 6919 */ 6920 6921 do { 6922 struct sched_entity *curr = cfs_rq->curr; 6923 6924 /* 6925 * Since we got here without doing put_prev_entity() we also 6926 * have to consider cfs_rq->curr. If it is still a runnable 6927 * entity, update_curr() will update its vruntime, otherwise 6928 * forget we've ever seen it. 6929 */ 6930 if (curr) { 6931 if (curr->on_rq) 6932 update_curr(cfs_rq); 6933 else 6934 curr = NULL; 6935 6936 /* 6937 * This call to check_cfs_rq_runtime() will do the 6938 * throttle and dequeue its entity in the parent(s). 6939 * Therefore the nr_running test will indeed 6940 * be correct. 6941 */ 6942 if (unlikely(check_cfs_rq_runtime(cfs_rq))) { 6943 cfs_rq = &rq->cfs; 6944 6945 if (!cfs_rq->nr_running) 6946 goto idle; 6947 6948 goto simple; 6949 } 6950 } 6951 6952 se = pick_next_entity(cfs_rq, curr); 6953 cfs_rq = group_cfs_rq(se); 6954 } while (cfs_rq); 6955 6956 p = task_of(se); 6957 6958 /* 6959 * Since we haven't yet done put_prev_entity and if the selected task 6960 * is a different task than we started out with, try and touch the 6961 * least amount of cfs_rqs. 6962 */ 6963 if (prev != p) { 6964 struct sched_entity *pse = &prev->se; 6965 6966 while (!(cfs_rq = is_same_group(se, pse))) { 6967 int se_depth = se->depth; 6968 int pse_depth = pse->depth; 6969 6970 if (se_depth <= pse_depth) { 6971 put_prev_entity(cfs_rq_of(pse), pse); 6972 pse = parent_entity(pse); 6973 } 6974 if (se_depth >= pse_depth) { 6975 set_next_entity(cfs_rq_of(se), se); 6976 se = parent_entity(se); 6977 } 6978 } 6979 6980 put_prev_entity(cfs_rq, pse); 6981 set_next_entity(cfs_rq, se); 6982 } 6983 6984 goto done; 6985 simple: 6986 #endif 6987 6988 put_prev_task(rq, prev); 6989 6990 do { 6991 se = pick_next_entity(cfs_rq, NULL); 6992 set_next_entity(cfs_rq, se); 6993 cfs_rq = group_cfs_rq(se); 6994 } while (cfs_rq); 6995 6996 p = task_of(se); 6997 6998 done: __maybe_unused; 6999 #ifdef CONFIG_SMP 7000 /* 7001 * Move the next running task to the front of 7002 * the list, so our cfs_tasks list becomes MRU 7003 * one. 7004 */ 7005 list_move(&p->se.group_node, &rq->cfs_tasks); 7006 #endif 7007 7008 if (hrtick_enabled(rq)) 7009 hrtick_start_fair(rq, p); 7010 7011 update_misfit_status(p, rq); 7012 7013 return p; 7014 7015 idle: 7016 update_misfit_status(NULL, rq); 7017 new_tasks = idle_balance(rq, rf); 7018 7019 /* 7020 * Because idle_balance() releases (and re-acquires) rq->lock, it is 7021 * possible for any higher priority task to appear. In that case we 7022 * must re-start the pick_next_entity() loop. 7023 */ 7024 if (new_tasks < 0) 7025 return RETRY_TASK; 7026 7027 if (new_tasks > 0) 7028 goto again; 7029 7030 return NULL; 7031 } 7032 7033 /* 7034 * Account for a descheduled task: 7035 */ 7036 static void put_prev_task_fair(struct rq *rq, struct task_struct *prev) 7037 { 7038 struct sched_entity *se = &prev->se; 7039 struct cfs_rq *cfs_rq; 7040 7041 for_each_sched_entity(se) { 7042 cfs_rq = cfs_rq_of(se); 7043 put_prev_entity(cfs_rq, se); 7044 } 7045 } 7046 7047 /* 7048 * sched_yield() is very simple 7049 * 7050 * The magic of dealing with the ->skip buddy is in pick_next_entity. 7051 */ 7052 static void yield_task_fair(struct rq *rq) 7053 { 7054 struct task_struct *curr = rq->curr; 7055 struct cfs_rq *cfs_rq = task_cfs_rq(curr); 7056 struct sched_entity *se = &curr->se; 7057 7058 /* 7059 * Are we the only task in the tree? 7060 */ 7061 if (unlikely(rq->nr_running == 1)) 7062 return; 7063 7064 clear_buddies(cfs_rq, se); 7065 7066 if (curr->policy != SCHED_BATCH) { 7067 update_rq_clock(rq); 7068 /* 7069 * Update run-time statistics of the 'current'. 7070 */ 7071 update_curr(cfs_rq); 7072 /* 7073 * Tell update_rq_clock() that we've just updated, 7074 * so we don't do microscopic update in schedule() 7075 * and double the fastpath cost. 7076 */ 7077 rq_clock_skip_update(rq); 7078 } 7079 7080 set_skip_buddy(se); 7081 } 7082 7083 static bool yield_to_task_fair(struct rq *rq, struct task_struct *p, bool preempt) 7084 { 7085 struct sched_entity *se = &p->se; 7086 7087 /* throttled hierarchies are not runnable */ 7088 if (!se->on_rq || throttled_hierarchy(cfs_rq_of(se))) 7089 return false; 7090 7091 /* Tell the scheduler that we'd really like pse to run next. */ 7092 set_next_buddy(se); 7093 7094 yield_task_fair(rq); 7095 7096 return true; 7097 } 7098 7099 #ifdef CONFIG_SMP 7100 /************************************************** 7101 * Fair scheduling class load-balancing methods. 7102 * 7103 * BASICS 7104 * 7105 * The purpose of load-balancing is to achieve the same basic fairness the 7106 * per-CPU scheduler provides, namely provide a proportional amount of compute 7107 * time to each task. This is expressed in the following equation: 7108 * 7109 * W_i,n/P_i == W_j,n/P_j for all i,j (1) 7110 * 7111 * Where W_i,n is the n-th weight average for CPU i. The instantaneous weight 7112 * W_i,0 is defined as: 7113 * 7114 * W_i,0 = \Sum_j w_i,j (2) 7115 * 7116 * Where w_i,j is the weight of the j-th runnable task on CPU i. This weight 7117 * is derived from the nice value as per sched_prio_to_weight[]. 7118 * 7119 * The weight average is an exponential decay average of the instantaneous 7120 * weight: 7121 * 7122 * W'_i,n = (2^n - 1) / 2^n * W_i,n + 1 / 2^n * W_i,0 (3) 7123 * 7124 * C_i is the compute capacity of CPU i, typically it is the 7125 * fraction of 'recent' time available for SCHED_OTHER task execution. But it 7126 * can also include other factors [XXX]. 7127 * 7128 * To achieve this balance we define a measure of imbalance which follows 7129 * directly from (1): 7130 * 7131 * imb_i,j = max{ avg(W/C), W_i/C_i } - min{ avg(W/C), W_j/C_j } (4) 7132 * 7133 * We them move tasks around to minimize the imbalance. In the continuous 7134 * function space it is obvious this converges, in the discrete case we get 7135 * a few fun cases generally called infeasible weight scenarios. 7136 * 7137 * [XXX expand on: 7138 * - infeasible weights; 7139 * - local vs global optima in the discrete case. ] 7140 * 7141 * 7142 * SCHED DOMAINS 7143 * 7144 * In order to solve the imbalance equation (4), and avoid the obvious O(n^2) 7145 * for all i,j solution, we create a tree of CPUs that follows the hardware 7146 * topology where each level pairs two lower groups (or better). This results 7147 * in O(log n) layers. Furthermore we reduce the number of CPUs going up the 7148 * tree to only the first of the previous level and we decrease the frequency 7149 * of load-balance at each level inv. proportional to the number of CPUs in 7150 * the groups. 7151 * 7152 * This yields: 7153 * 7154 * log_2 n 1 n 7155 * \Sum { --- * --- * 2^i } = O(n) (5) 7156 * i = 0 2^i 2^i 7157 * `- size of each group 7158 * | | `- number of CPUs doing load-balance 7159 * | `- freq 7160 * `- sum over all levels 7161 * 7162 * Coupled with a limit on how many tasks we can migrate every balance pass, 7163 * this makes (5) the runtime complexity of the balancer. 7164 * 7165 * An important property here is that each CPU is still (indirectly) connected 7166 * to every other CPU in at most O(log n) steps: 7167 * 7168 * The adjacency matrix of the resulting graph is given by: 7169 * 7170 * log_2 n 7171 * A_i,j = \Union (i % 2^k == 0) && i / 2^(k+1) == j / 2^(k+1) (6) 7172 * k = 0 7173 * 7174 * And you'll find that: 7175 * 7176 * A^(log_2 n)_i,j != 0 for all i,j (7) 7177 * 7178 * Showing there's indeed a path between every CPU in at most O(log n) steps. 7179 * The task movement gives a factor of O(m), giving a convergence complexity 7180 * of: 7181 * 7182 * O(nm log n), n := nr_cpus, m := nr_tasks (8) 7183 * 7184 * 7185 * WORK CONSERVING 7186 * 7187 * In order to avoid CPUs going idle while there's still work to do, new idle 7188 * balancing is more aggressive and has the newly idle CPU iterate up the domain 7189 * tree itself instead of relying on other CPUs to bring it work. 7190 * 7191 * This adds some complexity to both (5) and (8) but it reduces the total idle 7192 * time. 7193 * 7194 * [XXX more?] 7195 * 7196 * 7197 * CGROUPS 7198 * 7199 * Cgroups make a horror show out of (2), instead of a simple sum we get: 7200 * 7201 * s_k,i 7202 * W_i,0 = \Sum_j \Prod_k w_k * ----- (9) 7203 * S_k 7204 * 7205 * Where 7206 * 7207 * s_k,i = \Sum_j w_i,j,k and S_k = \Sum_i s_k,i (10) 7208 * 7209 * w_i,j,k is the weight of the j-th runnable task in the k-th cgroup on CPU i. 7210 * 7211 * The big problem is S_k, its a global sum needed to compute a local (W_i) 7212 * property. 7213 * 7214 * [XXX write more on how we solve this.. _after_ merging pjt's patches that 7215 * rewrite all of this once again.] 7216 */ 7217 7218 static unsigned long __read_mostly max_load_balance_interval = HZ/10; 7219 7220 enum fbq_type { regular, remote, all }; 7221 7222 enum group_type { 7223 group_other = 0, 7224 group_misfit_task, 7225 group_imbalanced, 7226 group_overloaded, 7227 }; 7228 7229 #define LBF_ALL_PINNED 0x01 7230 #define LBF_NEED_BREAK 0x02 7231 #define LBF_DST_PINNED 0x04 7232 #define LBF_SOME_PINNED 0x08 7233 #define LBF_NOHZ_STATS 0x10 7234 #define LBF_NOHZ_AGAIN 0x20 7235 7236 struct lb_env { 7237 struct sched_domain *sd; 7238 7239 struct rq *src_rq; 7240 int src_cpu; 7241 7242 int dst_cpu; 7243 struct rq *dst_rq; 7244 7245 struct cpumask *dst_grpmask; 7246 int new_dst_cpu; 7247 enum cpu_idle_type idle; 7248 long imbalance; 7249 /* The set of CPUs under consideration for load-balancing */ 7250 struct cpumask *cpus; 7251 7252 unsigned int flags; 7253 7254 unsigned int loop; 7255 unsigned int loop_break; 7256 unsigned int loop_max; 7257 7258 enum fbq_type fbq_type; 7259 enum group_type src_grp_type; 7260 struct list_head tasks; 7261 }; 7262 7263 /* 7264 * Is this task likely cache-hot: 7265 */ 7266 static int task_hot(struct task_struct *p, struct lb_env *env) 7267 { 7268 s64 delta; 7269 7270 lockdep_assert_held(&env->src_rq->lock); 7271 7272 if (p->sched_class != &fair_sched_class) 7273 return 0; 7274 7275 if (unlikely(task_has_idle_policy(p))) 7276 return 0; 7277 7278 /* 7279 * Buddy candidates are cache hot: 7280 */ 7281 if (sched_feat(CACHE_HOT_BUDDY) && env->dst_rq->nr_running && 7282 (&p->se == cfs_rq_of(&p->se)->next || 7283 &p->se == cfs_rq_of(&p->se)->last)) 7284 return 1; 7285 7286 if (sysctl_sched_migration_cost == -1) 7287 return 1; 7288 if (sysctl_sched_migration_cost == 0) 7289 return 0; 7290 7291 delta = rq_clock_task(env->src_rq) - p->se.exec_start; 7292 7293 return delta < (s64)sysctl_sched_migration_cost; 7294 } 7295 7296 #ifdef CONFIG_NUMA_BALANCING 7297 /* 7298 * Returns 1, if task migration degrades locality 7299 * Returns 0, if task migration improves locality i.e migration preferred. 7300 * Returns -1, if task migration is not affected by locality. 7301 */ 7302 static int migrate_degrades_locality(struct task_struct *p, struct lb_env *env) 7303 { 7304 struct numa_group *numa_group = rcu_dereference(p->numa_group); 7305 unsigned long src_weight, dst_weight; 7306 int src_nid, dst_nid, dist; 7307 7308 if (!static_branch_likely(&sched_numa_balancing)) 7309 return -1; 7310 7311 if (!p->numa_faults || !(env->sd->flags & SD_NUMA)) 7312 return -1; 7313 7314 src_nid = cpu_to_node(env->src_cpu); 7315 dst_nid = cpu_to_node(env->dst_cpu); 7316 7317 if (src_nid == dst_nid) 7318 return -1; 7319 7320 /* Migrating away from the preferred node is always bad. */ 7321 if (src_nid == p->numa_preferred_nid) { 7322 if (env->src_rq->nr_running > env->src_rq->nr_preferred_running) 7323 return 1; 7324 else 7325 return -1; 7326 } 7327 7328 /* Encourage migration to the preferred node. */ 7329 if (dst_nid == p->numa_preferred_nid) 7330 return 0; 7331 7332 /* Leaving a core idle is often worse than degrading locality. */ 7333 if (env->idle == CPU_IDLE) 7334 return -1; 7335 7336 dist = node_distance(src_nid, dst_nid); 7337 if (numa_group) { 7338 src_weight = group_weight(p, src_nid, dist); 7339 dst_weight = group_weight(p, dst_nid, dist); 7340 } else { 7341 src_weight = task_weight(p, src_nid, dist); 7342 dst_weight = task_weight(p, dst_nid, dist); 7343 } 7344 7345 return dst_weight < src_weight; 7346 } 7347 7348 #else 7349 static inline int migrate_degrades_locality(struct task_struct *p, 7350 struct lb_env *env) 7351 { 7352 return -1; 7353 } 7354 #endif 7355 7356 /* 7357 * can_migrate_task - may task p from runqueue rq be migrated to this_cpu? 7358 */ 7359 static 7360 int can_migrate_task(struct task_struct *p, struct lb_env *env) 7361 { 7362 int tsk_cache_hot; 7363 7364 lockdep_assert_held(&env->src_rq->lock); 7365 7366 /* 7367 * We do not migrate tasks that are: 7368 * 1) throttled_lb_pair, or 7369 * 2) cannot be migrated to this CPU due to cpus_allowed, or 7370 * 3) running (obviously), or 7371 * 4) are cache-hot on their current CPU. 7372 */ 7373 if (throttled_lb_pair(task_group(p), env->src_cpu, env->dst_cpu)) 7374 return 0; 7375 7376 if (!cpumask_test_cpu(env->dst_cpu, &p->cpus_allowed)) { 7377 int cpu; 7378 7379 schedstat_inc(p->se.statistics.nr_failed_migrations_affine); 7380 7381 env->flags |= LBF_SOME_PINNED; 7382 7383 /* 7384 * Remember if this task can be migrated to any other CPU in 7385 * our sched_group. We may want to revisit it if we couldn't 7386 * meet load balance goals by pulling other tasks on src_cpu. 7387 * 7388 * Avoid computing new_dst_cpu for NEWLY_IDLE or if we have 7389 * already computed one in current iteration. 7390 */ 7391 if (env->idle == CPU_NEWLY_IDLE || (env->flags & LBF_DST_PINNED)) 7392 return 0; 7393 7394 /* Prevent to re-select dst_cpu via env's CPUs: */ 7395 for_each_cpu_and(cpu, env->dst_grpmask, env->cpus) { 7396 if (cpumask_test_cpu(cpu, &p->cpus_allowed)) { 7397 env->flags |= LBF_DST_PINNED; 7398 env->new_dst_cpu = cpu; 7399 break; 7400 } 7401 } 7402 7403 return 0; 7404 } 7405 7406 /* Record that we found atleast one task that could run on dst_cpu */ 7407 env->flags &= ~LBF_ALL_PINNED; 7408 7409 if (task_running(env->src_rq, p)) { 7410 schedstat_inc(p->se.statistics.nr_failed_migrations_running); 7411 return 0; 7412 } 7413 7414 /* 7415 * Aggressive migration if: 7416 * 1) destination numa is preferred 7417 * 2) task is cache cold, or 7418 * 3) too many balance attempts have failed. 7419 */ 7420 tsk_cache_hot = migrate_degrades_locality(p, env); 7421 if (tsk_cache_hot == -1) 7422 tsk_cache_hot = task_hot(p, env); 7423 7424 if (tsk_cache_hot <= 0 || 7425 env->sd->nr_balance_failed > env->sd->cache_nice_tries) { 7426 if (tsk_cache_hot == 1) { 7427 schedstat_inc(env->sd->lb_hot_gained[env->idle]); 7428 schedstat_inc(p->se.statistics.nr_forced_migrations); 7429 } 7430 return 1; 7431 } 7432 7433 schedstat_inc(p->se.statistics.nr_failed_migrations_hot); 7434 return 0; 7435 } 7436 7437 /* 7438 * detach_task() -- detach the task for the migration specified in env 7439 */ 7440 static void detach_task(struct task_struct *p, struct lb_env *env) 7441 { 7442 lockdep_assert_held(&env->src_rq->lock); 7443 7444 p->on_rq = TASK_ON_RQ_MIGRATING; 7445 deactivate_task(env->src_rq, p, DEQUEUE_NOCLOCK); 7446 set_task_cpu(p, env->dst_cpu); 7447 } 7448 7449 /* 7450 * detach_one_task() -- tries to dequeue exactly one task from env->src_rq, as 7451 * part of active balancing operations within "domain". 7452 * 7453 * Returns a task if successful and NULL otherwise. 7454 */ 7455 static struct task_struct *detach_one_task(struct lb_env *env) 7456 { 7457 struct task_struct *p; 7458 7459 lockdep_assert_held(&env->src_rq->lock); 7460 7461 list_for_each_entry_reverse(p, 7462 &env->src_rq->cfs_tasks, se.group_node) { 7463 if (!can_migrate_task(p, env)) 7464 continue; 7465 7466 detach_task(p, env); 7467 7468 /* 7469 * Right now, this is only the second place where 7470 * lb_gained[env->idle] is updated (other is detach_tasks) 7471 * so we can safely collect stats here rather than 7472 * inside detach_tasks(). 7473 */ 7474 schedstat_inc(env->sd->lb_gained[env->idle]); 7475 return p; 7476 } 7477 return NULL; 7478 } 7479 7480 static const unsigned int sched_nr_migrate_break = 32; 7481 7482 /* 7483 * detach_tasks() -- tries to detach up to imbalance weighted load from 7484 * busiest_rq, as part of a balancing operation within domain "sd". 7485 * 7486 * Returns number of detached tasks if successful and 0 otherwise. 7487 */ 7488 static int detach_tasks(struct lb_env *env) 7489 { 7490 struct list_head *tasks = &env->src_rq->cfs_tasks; 7491 struct task_struct *p; 7492 unsigned long load; 7493 int detached = 0; 7494 7495 lockdep_assert_held(&env->src_rq->lock); 7496 7497 if (env->imbalance <= 0) 7498 return 0; 7499 7500 while (!list_empty(tasks)) { 7501 /* 7502 * We don't want to steal all, otherwise we may be treated likewise, 7503 * which could at worst lead to a livelock crash. 7504 */ 7505 if (env->idle != CPU_NOT_IDLE && env->src_rq->nr_running <= 1) 7506 break; 7507 7508 p = list_last_entry(tasks, struct task_struct, se.group_node); 7509 7510 env->loop++; 7511 /* We've more or less seen every task there is, call it quits */ 7512 if (env->loop > env->loop_max) 7513 break; 7514 7515 /* take a breather every nr_migrate tasks */ 7516 if (env->loop > env->loop_break) { 7517 env->loop_break += sched_nr_migrate_break; 7518 env->flags |= LBF_NEED_BREAK; 7519 break; 7520 } 7521 7522 if (!can_migrate_task(p, env)) 7523 goto next; 7524 7525 load = task_h_load(p); 7526 7527 if (sched_feat(LB_MIN) && load < 16 && !env->sd->nr_balance_failed) 7528 goto next; 7529 7530 if ((load / 2) > env->imbalance) 7531 goto next; 7532 7533 detach_task(p, env); 7534 list_add(&p->se.group_node, &env->tasks); 7535 7536 detached++; 7537 env->imbalance -= load; 7538 7539 #ifdef CONFIG_PREEMPT 7540 /* 7541 * NEWIDLE balancing is a source of latency, so preemptible 7542 * kernels will stop after the first task is detached to minimize 7543 * the critical section. 7544 */ 7545 if (env->idle == CPU_NEWLY_IDLE) 7546 break; 7547 #endif 7548 7549 /* 7550 * We only want to steal up to the prescribed amount of 7551 * weighted load. 7552 */ 7553 if (env->imbalance <= 0) 7554 break; 7555 7556 continue; 7557 next: 7558 list_move(&p->se.group_node, tasks); 7559 } 7560 7561 /* 7562 * Right now, this is one of only two places we collect this stat 7563 * so we can safely collect detach_one_task() stats here rather 7564 * than inside detach_one_task(). 7565 */ 7566 schedstat_add(env->sd->lb_gained[env->idle], detached); 7567 7568 return detached; 7569 } 7570 7571 /* 7572 * attach_task() -- attach the task detached by detach_task() to its new rq. 7573 */ 7574 static void attach_task(struct rq *rq, struct task_struct *p) 7575 { 7576 lockdep_assert_held(&rq->lock); 7577 7578 BUG_ON(task_rq(p) != rq); 7579 activate_task(rq, p, ENQUEUE_NOCLOCK); 7580 p->on_rq = TASK_ON_RQ_QUEUED; 7581 check_preempt_curr(rq, p, 0); 7582 } 7583 7584 /* 7585 * attach_one_task() -- attaches the task returned from detach_one_task() to 7586 * its new rq. 7587 */ 7588 static void attach_one_task(struct rq *rq, struct task_struct *p) 7589 { 7590 struct rq_flags rf; 7591 7592 rq_lock(rq, &rf); 7593 update_rq_clock(rq); 7594 attach_task(rq, p); 7595 rq_unlock(rq, &rf); 7596 } 7597 7598 /* 7599 * attach_tasks() -- attaches all tasks detached by detach_tasks() to their 7600 * new rq. 7601 */ 7602 static void attach_tasks(struct lb_env *env) 7603 { 7604 struct list_head *tasks = &env->tasks; 7605 struct task_struct *p; 7606 struct rq_flags rf; 7607 7608 rq_lock(env->dst_rq, &rf); 7609 update_rq_clock(env->dst_rq); 7610 7611 while (!list_empty(tasks)) { 7612 p = list_first_entry(tasks, struct task_struct, se.group_node); 7613 list_del_init(&p->se.group_node); 7614 7615 attach_task(env->dst_rq, p); 7616 } 7617 7618 rq_unlock(env->dst_rq, &rf); 7619 } 7620 7621 static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq) 7622 { 7623 if (cfs_rq->avg.load_avg) 7624 return true; 7625 7626 if (cfs_rq->avg.util_avg) 7627 return true; 7628 7629 return false; 7630 } 7631 7632 static inline bool others_have_blocked(struct rq *rq) 7633 { 7634 if (READ_ONCE(rq->avg_rt.util_avg)) 7635 return true; 7636 7637 if (READ_ONCE(rq->avg_dl.util_avg)) 7638 return true; 7639 7640 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ 7641 if (READ_ONCE(rq->avg_irq.util_avg)) 7642 return true; 7643 #endif 7644 7645 return false; 7646 } 7647 7648 #ifdef CONFIG_FAIR_GROUP_SCHED 7649 7650 static void update_blocked_averages(int cpu) 7651 { 7652 struct rq *rq = cpu_rq(cpu); 7653 struct cfs_rq *cfs_rq; 7654 const struct sched_class *curr_class; 7655 struct rq_flags rf; 7656 bool done = true; 7657 7658 rq_lock_irqsave(rq, &rf); 7659 update_rq_clock(rq); 7660 7661 /* 7662 * Iterates the task_group tree in a bottom up fashion, see 7663 * list_add_leaf_cfs_rq() for details. 7664 */ 7665 for_each_leaf_cfs_rq(rq, cfs_rq) { 7666 struct sched_entity *se; 7667 7668 /* throttled entities do not contribute to load */ 7669 if (throttled_hierarchy(cfs_rq)) 7670 continue; 7671 7672 if (update_cfs_rq_load_avg(cfs_rq_clock_task(cfs_rq), cfs_rq)) 7673 update_tg_load_avg(cfs_rq, 0); 7674 7675 /* Propagate pending load changes to the parent, if any: */ 7676 se = cfs_rq->tg->se[cpu]; 7677 if (se && !skip_blocked_update(se)) 7678 update_load_avg(cfs_rq_of(se), se, 0); 7679 7680 /* Don't need periodic decay once load/util_avg are null */ 7681 if (cfs_rq_has_blocked(cfs_rq)) 7682 done = false; 7683 } 7684 7685 curr_class = rq->curr->sched_class; 7686 update_rt_rq_load_avg(rq_clock_task(rq), rq, curr_class == &rt_sched_class); 7687 update_dl_rq_load_avg(rq_clock_task(rq), rq, curr_class == &dl_sched_class); 7688 update_irq_load_avg(rq, 0); 7689 /* Don't need periodic decay once load/util_avg are null */ 7690 if (others_have_blocked(rq)) 7691 done = false; 7692 7693 #ifdef CONFIG_NO_HZ_COMMON 7694 rq->last_blocked_load_update_tick = jiffies; 7695 if (done) 7696 rq->has_blocked_load = 0; 7697 #endif 7698 rq_unlock_irqrestore(rq, &rf); 7699 } 7700 7701 /* 7702 * Compute the hierarchical load factor for cfs_rq and all its ascendants. 7703 * This needs to be done in a top-down fashion because the load of a child 7704 * group is a fraction of its parents load. 7705 */ 7706 static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq) 7707 { 7708 struct rq *rq = rq_of(cfs_rq); 7709 struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)]; 7710 unsigned long now = jiffies; 7711 unsigned long load; 7712 7713 if (cfs_rq->last_h_load_update == now) 7714 return; 7715 7716 cfs_rq->h_load_next = NULL; 7717 for_each_sched_entity(se) { 7718 cfs_rq = cfs_rq_of(se); 7719 cfs_rq->h_load_next = se; 7720 if (cfs_rq->last_h_load_update == now) 7721 break; 7722 } 7723 7724 if (!se) { 7725 cfs_rq->h_load = cfs_rq_load_avg(cfs_rq); 7726 cfs_rq->last_h_load_update = now; 7727 } 7728 7729 while ((se = cfs_rq->h_load_next) != NULL) { 7730 load = cfs_rq->h_load; 7731 load = div64_ul(load * se->avg.load_avg, 7732 cfs_rq_load_avg(cfs_rq) + 1); 7733 cfs_rq = group_cfs_rq(se); 7734 cfs_rq->h_load = load; 7735 cfs_rq->last_h_load_update = now; 7736 } 7737 } 7738 7739 static unsigned long task_h_load(struct task_struct *p) 7740 { 7741 struct cfs_rq *cfs_rq = task_cfs_rq(p); 7742 7743 update_cfs_rq_h_load(cfs_rq); 7744 return div64_ul(p->se.avg.load_avg * cfs_rq->h_load, 7745 cfs_rq_load_avg(cfs_rq) + 1); 7746 } 7747 #else 7748 static inline void update_blocked_averages(int cpu) 7749 { 7750 struct rq *rq = cpu_rq(cpu); 7751 struct cfs_rq *cfs_rq = &rq->cfs; 7752 const struct sched_class *curr_class; 7753 struct rq_flags rf; 7754 7755 rq_lock_irqsave(rq, &rf); 7756 update_rq_clock(rq); 7757 update_cfs_rq_load_avg(cfs_rq_clock_task(cfs_rq), cfs_rq); 7758 7759 curr_class = rq->curr->sched_class; 7760 update_rt_rq_load_avg(rq_clock_task(rq), rq, curr_class == &rt_sched_class); 7761 update_dl_rq_load_avg(rq_clock_task(rq), rq, curr_class == &dl_sched_class); 7762 update_irq_load_avg(rq, 0); 7763 #ifdef CONFIG_NO_HZ_COMMON 7764 rq->last_blocked_load_update_tick = jiffies; 7765 if (!cfs_rq_has_blocked(cfs_rq) && !others_have_blocked(rq)) 7766 rq->has_blocked_load = 0; 7767 #endif 7768 rq_unlock_irqrestore(rq, &rf); 7769 } 7770 7771 static unsigned long task_h_load(struct task_struct *p) 7772 { 7773 return p->se.avg.load_avg; 7774 } 7775 #endif 7776 7777 /********** Helpers for find_busiest_group ************************/ 7778 7779 /* 7780 * sg_lb_stats - stats of a sched_group required for load_balancing 7781 */ 7782 struct sg_lb_stats { 7783 unsigned long avg_load; /*Avg load across the CPUs of the group */ 7784 unsigned long group_load; /* Total load over the CPUs of the group */ 7785 unsigned long sum_weighted_load; /* Weighted load of group's tasks */ 7786 unsigned long load_per_task; 7787 unsigned long group_capacity; 7788 unsigned long group_util; /* Total utilization of the group */ 7789 unsigned int sum_nr_running; /* Nr tasks running in the group */ 7790 unsigned int idle_cpus; 7791 unsigned int group_weight; 7792 enum group_type group_type; 7793 int group_no_capacity; 7794 unsigned long group_misfit_task_load; /* A CPU has a task too big for its capacity */ 7795 #ifdef CONFIG_NUMA_BALANCING 7796 unsigned int nr_numa_running; 7797 unsigned int nr_preferred_running; 7798 #endif 7799 }; 7800 7801 /* 7802 * sd_lb_stats - Structure to store the statistics of a sched_domain 7803 * during load balancing. 7804 */ 7805 struct sd_lb_stats { 7806 struct sched_group *busiest; /* Busiest group in this sd */ 7807 struct sched_group *local; /* Local group in this sd */ 7808 unsigned long total_running; 7809 unsigned long total_load; /* Total load of all groups in sd */ 7810 unsigned long total_capacity; /* Total capacity of all groups in sd */ 7811 unsigned long avg_load; /* Average load across all groups in sd */ 7812 7813 struct sg_lb_stats busiest_stat;/* Statistics of the busiest group */ 7814 struct sg_lb_stats local_stat; /* Statistics of the local group */ 7815 }; 7816 7817 static inline void init_sd_lb_stats(struct sd_lb_stats *sds) 7818 { 7819 /* 7820 * Skimp on the clearing to avoid duplicate work. We can avoid clearing 7821 * local_stat because update_sg_lb_stats() does a full clear/assignment. 7822 * We must however clear busiest_stat::avg_load because 7823 * update_sd_pick_busiest() reads this before assignment. 7824 */ 7825 *sds = (struct sd_lb_stats){ 7826 .busiest = NULL, 7827 .local = NULL, 7828 .total_running = 0UL, 7829 .total_load = 0UL, 7830 .total_capacity = 0UL, 7831 .busiest_stat = { 7832 .avg_load = 0UL, 7833 .sum_nr_running = 0, 7834 .group_type = group_other, 7835 }, 7836 }; 7837 } 7838 7839 /** 7840 * get_sd_load_idx - Obtain the load index for a given sched domain. 7841 * @sd: The sched_domain whose load_idx is to be obtained. 7842 * @idle: The idle status of the CPU for whose sd load_idx is obtained. 7843 * 7844 * Return: The load index. 7845 */ 7846 static inline int get_sd_load_idx(struct sched_domain *sd, 7847 enum cpu_idle_type idle) 7848 { 7849 int load_idx; 7850 7851 switch (idle) { 7852 case CPU_NOT_IDLE: 7853 load_idx = sd->busy_idx; 7854 break; 7855 7856 case CPU_NEWLY_IDLE: 7857 load_idx = sd->newidle_idx; 7858 break; 7859 default: 7860 load_idx = sd->idle_idx; 7861 break; 7862 } 7863 7864 return load_idx; 7865 } 7866 7867 static unsigned long scale_rt_capacity(struct sched_domain *sd, int cpu) 7868 { 7869 struct rq *rq = cpu_rq(cpu); 7870 unsigned long max = arch_scale_cpu_capacity(sd, cpu); 7871 unsigned long used, free; 7872 unsigned long irq; 7873 7874 irq = cpu_util_irq(rq); 7875 7876 if (unlikely(irq >= max)) 7877 return 1; 7878 7879 used = READ_ONCE(rq->avg_rt.util_avg); 7880 used += READ_ONCE(rq->avg_dl.util_avg); 7881 7882 if (unlikely(used >= max)) 7883 return 1; 7884 7885 free = max - used; 7886 7887 return scale_irq_capacity(free, irq, max); 7888 } 7889 7890 static void update_cpu_capacity(struct sched_domain *sd, int cpu) 7891 { 7892 unsigned long capacity = scale_rt_capacity(sd, cpu); 7893 struct sched_group *sdg = sd->groups; 7894 7895 cpu_rq(cpu)->cpu_capacity_orig = arch_scale_cpu_capacity(sd, cpu); 7896 7897 if (!capacity) 7898 capacity = 1; 7899 7900 cpu_rq(cpu)->cpu_capacity = capacity; 7901 sdg->sgc->capacity = capacity; 7902 sdg->sgc->min_capacity = capacity; 7903 sdg->sgc->max_capacity = capacity; 7904 } 7905 7906 void update_group_capacity(struct sched_domain *sd, int cpu) 7907 { 7908 struct sched_domain *child = sd->child; 7909 struct sched_group *group, *sdg = sd->groups; 7910 unsigned long capacity, min_capacity, max_capacity; 7911 unsigned long interval; 7912 7913 interval = msecs_to_jiffies(sd->balance_interval); 7914 interval = clamp(interval, 1UL, max_load_balance_interval); 7915 sdg->sgc->next_update = jiffies + interval; 7916 7917 if (!child) { 7918 update_cpu_capacity(sd, cpu); 7919 return; 7920 } 7921 7922 capacity = 0; 7923 min_capacity = ULONG_MAX; 7924 max_capacity = 0; 7925 7926 if (child->flags & SD_OVERLAP) { 7927 /* 7928 * SD_OVERLAP domains cannot assume that child groups 7929 * span the current group. 7930 */ 7931 7932 for_each_cpu(cpu, sched_group_span(sdg)) { 7933 struct sched_group_capacity *sgc; 7934 struct rq *rq = cpu_rq(cpu); 7935 7936 /* 7937 * build_sched_domains() -> init_sched_groups_capacity() 7938 * gets here before we've attached the domains to the 7939 * runqueues. 7940 * 7941 * Use capacity_of(), which is set irrespective of domains 7942 * in update_cpu_capacity(). 7943 * 7944 * This avoids capacity from being 0 and 7945 * causing divide-by-zero issues on boot. 7946 */ 7947 if (unlikely(!rq->sd)) { 7948 capacity += capacity_of(cpu); 7949 } else { 7950 sgc = rq->sd->groups->sgc; 7951 capacity += sgc->capacity; 7952 } 7953 7954 min_capacity = min(capacity, min_capacity); 7955 max_capacity = max(capacity, max_capacity); 7956 } 7957 } else { 7958 /* 7959 * !SD_OVERLAP domains can assume that child groups 7960 * span the current group. 7961 */ 7962 7963 group = child->groups; 7964 do { 7965 struct sched_group_capacity *sgc = group->sgc; 7966 7967 capacity += sgc->capacity; 7968 min_capacity = min(sgc->min_capacity, min_capacity); 7969 max_capacity = max(sgc->max_capacity, max_capacity); 7970 group = group->next; 7971 } while (group != child->groups); 7972 } 7973 7974 sdg->sgc->capacity = capacity; 7975 sdg->sgc->min_capacity = min_capacity; 7976 sdg->sgc->max_capacity = max_capacity; 7977 } 7978 7979 /* 7980 * Check whether the capacity of the rq has been noticeably reduced by side 7981 * activity. The imbalance_pct is used for the threshold. 7982 * Return true is the capacity is reduced 7983 */ 7984 static inline int 7985 check_cpu_capacity(struct rq *rq, struct sched_domain *sd) 7986 { 7987 return ((rq->cpu_capacity * sd->imbalance_pct) < 7988 (rq->cpu_capacity_orig * 100)); 7989 } 7990 7991 /* 7992 * Group imbalance indicates (and tries to solve) the problem where balancing 7993 * groups is inadequate due to ->cpus_allowed constraints. 7994 * 7995 * Imagine a situation of two groups of 4 CPUs each and 4 tasks each with a 7996 * cpumask covering 1 CPU of the first group and 3 CPUs of the second group. 7997 * Something like: 7998 * 7999 * { 0 1 2 3 } { 4 5 6 7 } 8000 * * * * * 8001 * 8002 * If we were to balance group-wise we'd place two tasks in the first group and 8003 * two tasks in the second group. Clearly this is undesired as it will overload 8004 * cpu 3 and leave one of the CPUs in the second group unused. 8005 * 8006 * The current solution to this issue is detecting the skew in the first group 8007 * by noticing the lower domain failed to reach balance and had difficulty 8008 * moving tasks due to affinity constraints. 8009 * 8010 * When this is so detected; this group becomes a candidate for busiest; see 8011 * update_sd_pick_busiest(). And calculate_imbalance() and 8012 * find_busiest_group() avoid some of the usual balance conditions to allow it 8013 * to create an effective group imbalance. 8014 * 8015 * This is a somewhat tricky proposition since the next run might not find the 8016 * group imbalance and decide the groups need to be balanced again. A most 8017 * subtle and fragile situation. 8018 */ 8019 8020 static inline int sg_imbalanced(struct sched_group *group) 8021 { 8022 return group->sgc->imbalance; 8023 } 8024 8025 /* 8026 * group_has_capacity returns true if the group has spare capacity that could 8027 * be used by some tasks. 8028 * We consider that a group has spare capacity if the * number of task is 8029 * smaller than the number of CPUs or if the utilization is lower than the 8030 * available capacity for CFS tasks. 8031 * For the latter, we use a threshold to stabilize the state, to take into 8032 * account the variance of the tasks' load and to return true if the available 8033 * capacity in meaningful for the load balancer. 8034 * As an example, an available capacity of 1% can appear but it doesn't make 8035 * any benefit for the load balance. 8036 */ 8037 static inline bool 8038 group_has_capacity(struct lb_env *env, struct sg_lb_stats *sgs) 8039 { 8040 if (sgs->sum_nr_running < sgs->group_weight) 8041 return true; 8042 8043 if ((sgs->group_capacity * 100) > 8044 (sgs->group_util * env->sd->imbalance_pct)) 8045 return true; 8046 8047 return false; 8048 } 8049 8050 /* 8051 * group_is_overloaded returns true if the group has more tasks than it can 8052 * handle. 8053 * group_is_overloaded is not equals to !group_has_capacity because a group 8054 * with the exact right number of tasks, has no more spare capacity but is not 8055 * overloaded so both group_has_capacity and group_is_overloaded return 8056 * false. 8057 */ 8058 static inline bool 8059 group_is_overloaded(struct lb_env *env, struct sg_lb_stats *sgs) 8060 { 8061 if (sgs->sum_nr_running <= sgs->group_weight) 8062 return false; 8063 8064 if ((sgs->group_capacity * 100) < 8065 (sgs->group_util * env->sd->imbalance_pct)) 8066 return true; 8067 8068 return false; 8069 } 8070 8071 /* 8072 * group_smaller_min_cpu_capacity: Returns true if sched_group sg has smaller 8073 * per-CPU capacity than sched_group ref. 8074 */ 8075 static inline bool 8076 group_smaller_min_cpu_capacity(struct sched_group *sg, struct sched_group *ref) 8077 { 8078 return sg->sgc->min_capacity * capacity_margin < 8079 ref->sgc->min_capacity * 1024; 8080 } 8081 8082 /* 8083 * group_smaller_max_cpu_capacity: Returns true if sched_group sg has smaller 8084 * per-CPU capacity_orig than sched_group ref. 8085 */ 8086 static inline bool 8087 group_smaller_max_cpu_capacity(struct sched_group *sg, struct sched_group *ref) 8088 { 8089 return sg->sgc->max_capacity * capacity_margin < 8090 ref->sgc->max_capacity * 1024; 8091 } 8092 8093 static inline enum 8094 group_type group_classify(struct sched_group *group, 8095 struct sg_lb_stats *sgs) 8096 { 8097 if (sgs->group_no_capacity) 8098 return group_overloaded; 8099 8100 if (sg_imbalanced(group)) 8101 return group_imbalanced; 8102 8103 if (sgs->group_misfit_task_load) 8104 return group_misfit_task; 8105 8106 return group_other; 8107 } 8108 8109 static bool update_nohz_stats(struct rq *rq, bool force) 8110 { 8111 #ifdef CONFIG_NO_HZ_COMMON 8112 unsigned int cpu = rq->cpu; 8113 8114 if (!rq->has_blocked_load) 8115 return false; 8116 8117 if (!cpumask_test_cpu(cpu, nohz.idle_cpus_mask)) 8118 return false; 8119 8120 if (!force && !time_after(jiffies, rq->last_blocked_load_update_tick)) 8121 return true; 8122 8123 update_blocked_averages(cpu); 8124 8125 return rq->has_blocked_load; 8126 #else 8127 return false; 8128 #endif 8129 } 8130 8131 /** 8132 * update_sg_lb_stats - Update sched_group's statistics for load balancing. 8133 * @env: The load balancing environment. 8134 * @group: sched_group whose statistics are to be updated. 8135 * @sgs: variable to hold the statistics for this group. 8136 * @sg_status: Holds flag indicating the status of the sched_group 8137 */ 8138 static inline void update_sg_lb_stats(struct lb_env *env, 8139 struct sched_group *group, 8140 struct sg_lb_stats *sgs, 8141 int *sg_status) 8142 { 8143 int local_group = cpumask_test_cpu(env->dst_cpu, sched_group_span(group)); 8144 int load_idx = get_sd_load_idx(env->sd, env->idle); 8145 unsigned long load; 8146 int i, nr_running; 8147 8148 memset(sgs, 0, sizeof(*sgs)); 8149 8150 for_each_cpu_and(i, sched_group_span(group), env->cpus) { 8151 struct rq *rq = cpu_rq(i); 8152 8153 if ((env->flags & LBF_NOHZ_STATS) && update_nohz_stats(rq, false)) 8154 env->flags |= LBF_NOHZ_AGAIN; 8155 8156 /* Bias balancing toward CPUs of our domain: */ 8157 if (local_group) 8158 load = target_load(i, load_idx); 8159 else 8160 load = source_load(i, load_idx); 8161 8162 sgs->group_load += load; 8163 sgs->group_util += cpu_util(i); 8164 sgs->sum_nr_running += rq->cfs.h_nr_running; 8165 8166 nr_running = rq->nr_running; 8167 if (nr_running > 1) 8168 *sg_status |= SG_OVERLOAD; 8169 8170 if (cpu_overutilized(i)) 8171 *sg_status |= SG_OVERUTILIZED; 8172 8173 #ifdef CONFIG_NUMA_BALANCING 8174 sgs->nr_numa_running += rq->nr_numa_running; 8175 sgs->nr_preferred_running += rq->nr_preferred_running; 8176 #endif 8177 sgs->sum_weighted_load += weighted_cpuload(rq); 8178 /* 8179 * No need to call idle_cpu() if nr_running is not 0 8180 */ 8181 if (!nr_running && idle_cpu(i)) 8182 sgs->idle_cpus++; 8183 8184 if (env->sd->flags & SD_ASYM_CPUCAPACITY && 8185 sgs->group_misfit_task_load < rq->misfit_task_load) { 8186 sgs->group_misfit_task_load = rq->misfit_task_load; 8187 *sg_status |= SG_OVERLOAD; 8188 } 8189 } 8190 8191 /* Adjust by relative CPU capacity of the group */ 8192 sgs->group_capacity = group->sgc->capacity; 8193 sgs->avg_load = (sgs->group_load*SCHED_CAPACITY_SCALE) / sgs->group_capacity; 8194 8195 if (sgs->sum_nr_running) 8196 sgs->load_per_task = sgs->sum_weighted_load / sgs->sum_nr_running; 8197 8198 sgs->group_weight = group->group_weight; 8199 8200 sgs->group_no_capacity = group_is_overloaded(env, sgs); 8201 sgs->group_type = group_classify(group, sgs); 8202 } 8203 8204 /** 8205 * update_sd_pick_busiest - return 1 on busiest group 8206 * @env: The load balancing environment. 8207 * @sds: sched_domain statistics 8208 * @sg: sched_group candidate to be checked for being the busiest 8209 * @sgs: sched_group statistics 8210 * 8211 * Determine if @sg is a busier group than the previously selected 8212 * busiest group. 8213 * 8214 * Return: %true if @sg is a busier group than the previously selected 8215 * busiest group. %false otherwise. 8216 */ 8217 static bool update_sd_pick_busiest(struct lb_env *env, 8218 struct sd_lb_stats *sds, 8219 struct sched_group *sg, 8220 struct sg_lb_stats *sgs) 8221 { 8222 struct sg_lb_stats *busiest = &sds->busiest_stat; 8223 8224 /* 8225 * Don't try to pull misfit tasks we can't help. 8226 * We can use max_capacity here as reduction in capacity on some 8227 * CPUs in the group should either be possible to resolve 8228 * internally or be covered by avg_load imbalance (eventually). 8229 */ 8230 if (sgs->group_type == group_misfit_task && 8231 (!group_smaller_max_cpu_capacity(sg, sds->local) || 8232 !group_has_capacity(env, &sds->local_stat))) 8233 return false; 8234 8235 if (sgs->group_type > busiest->group_type) 8236 return true; 8237 8238 if (sgs->group_type < busiest->group_type) 8239 return false; 8240 8241 if (sgs->avg_load <= busiest->avg_load) 8242 return false; 8243 8244 if (!(env->sd->flags & SD_ASYM_CPUCAPACITY)) 8245 goto asym_packing; 8246 8247 /* 8248 * Candidate sg has no more than one task per CPU and 8249 * has higher per-CPU capacity. Migrating tasks to less 8250 * capable CPUs may harm throughput. Maximize throughput, 8251 * power/energy consequences are not considered. 8252 */ 8253 if (sgs->sum_nr_running <= sgs->group_weight && 8254 group_smaller_min_cpu_capacity(sds->local, sg)) 8255 return false; 8256 8257 /* 8258 * If we have more than one misfit sg go with the biggest misfit. 8259 */ 8260 if (sgs->group_type == group_misfit_task && 8261 sgs->group_misfit_task_load < busiest->group_misfit_task_load) 8262 return false; 8263 8264 asym_packing: 8265 /* This is the busiest node in its class. */ 8266 if (!(env->sd->flags & SD_ASYM_PACKING)) 8267 return true; 8268 8269 /* No ASYM_PACKING if target CPU is already busy */ 8270 if (env->idle == CPU_NOT_IDLE) 8271 return true; 8272 /* 8273 * ASYM_PACKING needs to move all the work to the highest 8274 * prority CPUs in the group, therefore mark all groups 8275 * of lower priority than ourself as busy. 8276 */ 8277 if (sgs->sum_nr_running && 8278 sched_asym_prefer(env->dst_cpu, sg->asym_prefer_cpu)) { 8279 if (!sds->busiest) 8280 return true; 8281 8282 /* Prefer to move from lowest priority CPU's work */ 8283 if (sched_asym_prefer(sds->busiest->asym_prefer_cpu, 8284 sg->asym_prefer_cpu)) 8285 return true; 8286 } 8287 8288 return false; 8289 } 8290 8291 #ifdef CONFIG_NUMA_BALANCING 8292 static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs) 8293 { 8294 if (sgs->sum_nr_running > sgs->nr_numa_running) 8295 return regular; 8296 if (sgs->sum_nr_running > sgs->nr_preferred_running) 8297 return remote; 8298 return all; 8299 } 8300 8301 static inline enum fbq_type fbq_classify_rq(struct rq *rq) 8302 { 8303 if (rq->nr_running > rq->nr_numa_running) 8304 return regular; 8305 if (rq->nr_running > rq->nr_preferred_running) 8306 return remote; 8307 return all; 8308 } 8309 #else 8310 static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs) 8311 { 8312 return all; 8313 } 8314 8315 static inline enum fbq_type fbq_classify_rq(struct rq *rq) 8316 { 8317 return regular; 8318 } 8319 #endif /* CONFIG_NUMA_BALANCING */ 8320 8321 /** 8322 * update_sd_lb_stats - Update sched_domain's statistics for load balancing. 8323 * @env: The load balancing environment. 8324 * @sds: variable to hold the statistics for this sched_domain. 8325 */ 8326 static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sds) 8327 { 8328 struct sched_domain *child = env->sd->child; 8329 struct sched_group *sg = env->sd->groups; 8330 struct sg_lb_stats *local = &sds->local_stat; 8331 struct sg_lb_stats tmp_sgs; 8332 bool prefer_sibling = child && child->flags & SD_PREFER_SIBLING; 8333 int sg_status = 0; 8334 8335 #ifdef CONFIG_NO_HZ_COMMON 8336 if (env->idle == CPU_NEWLY_IDLE && READ_ONCE(nohz.has_blocked)) 8337 env->flags |= LBF_NOHZ_STATS; 8338 #endif 8339 8340 do { 8341 struct sg_lb_stats *sgs = &tmp_sgs; 8342 int local_group; 8343 8344 local_group = cpumask_test_cpu(env->dst_cpu, sched_group_span(sg)); 8345 if (local_group) { 8346 sds->local = sg; 8347 sgs = local; 8348 8349 if (env->idle != CPU_NEWLY_IDLE || 8350 time_after_eq(jiffies, sg->sgc->next_update)) 8351 update_group_capacity(env->sd, env->dst_cpu); 8352 } 8353 8354 update_sg_lb_stats(env, sg, sgs, &sg_status); 8355 8356 if (local_group) 8357 goto next_group; 8358 8359 /* 8360 * In case the child domain prefers tasks go to siblings 8361 * first, lower the sg capacity so that we'll try 8362 * and move all the excess tasks away. We lower the capacity 8363 * of a group only if the local group has the capacity to fit 8364 * these excess tasks. The extra check prevents the case where 8365 * you always pull from the heaviest group when it is already 8366 * under-utilized (possible with a large weight task outweighs 8367 * the tasks on the system). 8368 */ 8369 if (prefer_sibling && sds->local && 8370 group_has_capacity(env, local) && 8371 (sgs->sum_nr_running > local->sum_nr_running + 1)) { 8372 sgs->group_no_capacity = 1; 8373 sgs->group_type = group_classify(sg, sgs); 8374 } 8375 8376 if (update_sd_pick_busiest(env, sds, sg, sgs)) { 8377 sds->busiest = sg; 8378 sds->busiest_stat = *sgs; 8379 } 8380 8381 next_group: 8382 /* Now, start updating sd_lb_stats */ 8383 sds->total_running += sgs->sum_nr_running; 8384 sds->total_load += sgs->group_load; 8385 sds->total_capacity += sgs->group_capacity; 8386 8387 sg = sg->next; 8388 } while (sg != env->sd->groups); 8389 8390 #ifdef CONFIG_NO_HZ_COMMON 8391 if ((env->flags & LBF_NOHZ_AGAIN) && 8392 cpumask_subset(nohz.idle_cpus_mask, sched_domain_span(env->sd))) { 8393 8394 WRITE_ONCE(nohz.next_blocked, 8395 jiffies + msecs_to_jiffies(LOAD_AVG_PERIOD)); 8396 } 8397 #endif 8398 8399 if (env->sd->flags & SD_NUMA) 8400 env->fbq_type = fbq_classify_group(&sds->busiest_stat); 8401 8402 if (!env->sd->parent) { 8403 struct root_domain *rd = env->dst_rq->rd; 8404 8405 /* update overload indicator if we are at root domain */ 8406 WRITE_ONCE(rd->overload, sg_status & SG_OVERLOAD); 8407 8408 /* Update over-utilization (tipping point, U >= 0) indicator */ 8409 WRITE_ONCE(rd->overutilized, sg_status & SG_OVERUTILIZED); 8410 } else if (sg_status & SG_OVERUTILIZED) { 8411 WRITE_ONCE(env->dst_rq->rd->overutilized, SG_OVERUTILIZED); 8412 } 8413 } 8414 8415 /** 8416 * check_asym_packing - Check to see if the group is packed into the 8417 * sched domain. 8418 * 8419 * This is primarily intended to used at the sibling level. Some 8420 * cores like POWER7 prefer to use lower numbered SMT threads. In the 8421 * case of POWER7, it can move to lower SMT modes only when higher 8422 * threads are idle. When in lower SMT modes, the threads will 8423 * perform better since they share less core resources. Hence when we 8424 * have idle threads, we want them to be the higher ones. 8425 * 8426 * This packing function is run on idle threads. It checks to see if 8427 * the busiest CPU in this domain (core in the P7 case) has a higher 8428 * CPU number than the packing function is being run on. Here we are 8429 * assuming lower CPU number will be equivalent to lower a SMT thread 8430 * number. 8431 * 8432 * Return: 1 when packing is required and a task should be moved to 8433 * this CPU. The amount of the imbalance is returned in env->imbalance. 8434 * 8435 * @env: The load balancing environment. 8436 * @sds: Statistics of the sched_domain which is to be packed 8437 */ 8438 static int check_asym_packing(struct lb_env *env, struct sd_lb_stats *sds) 8439 { 8440 int busiest_cpu; 8441 8442 if (!(env->sd->flags & SD_ASYM_PACKING)) 8443 return 0; 8444 8445 if (env->idle == CPU_NOT_IDLE) 8446 return 0; 8447 8448 if (!sds->busiest) 8449 return 0; 8450 8451 busiest_cpu = sds->busiest->asym_prefer_cpu; 8452 if (sched_asym_prefer(busiest_cpu, env->dst_cpu)) 8453 return 0; 8454 8455 env->imbalance = DIV_ROUND_CLOSEST( 8456 sds->busiest_stat.avg_load * sds->busiest_stat.group_capacity, 8457 SCHED_CAPACITY_SCALE); 8458 8459 return 1; 8460 } 8461 8462 /** 8463 * fix_small_imbalance - Calculate the minor imbalance that exists 8464 * amongst the groups of a sched_domain, during 8465 * load balancing. 8466 * @env: The load balancing environment. 8467 * @sds: Statistics of the sched_domain whose imbalance is to be calculated. 8468 */ 8469 static inline 8470 void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds) 8471 { 8472 unsigned long tmp, capa_now = 0, capa_move = 0; 8473 unsigned int imbn = 2; 8474 unsigned long scaled_busy_load_per_task; 8475 struct sg_lb_stats *local, *busiest; 8476 8477 local = &sds->local_stat; 8478 busiest = &sds->busiest_stat; 8479 8480 if (!local->sum_nr_running) 8481 local->load_per_task = cpu_avg_load_per_task(env->dst_cpu); 8482 else if (busiest->load_per_task > local->load_per_task) 8483 imbn = 1; 8484 8485 scaled_busy_load_per_task = 8486 (busiest->load_per_task * SCHED_CAPACITY_SCALE) / 8487 busiest->group_capacity; 8488 8489 if (busiest->avg_load + scaled_busy_load_per_task >= 8490 local->avg_load + (scaled_busy_load_per_task * imbn)) { 8491 env->imbalance = busiest->load_per_task; 8492 return; 8493 } 8494 8495 /* 8496 * OK, we don't have enough imbalance to justify moving tasks, 8497 * however we may be able to increase total CPU capacity used by 8498 * moving them. 8499 */ 8500 8501 capa_now += busiest->group_capacity * 8502 min(busiest->load_per_task, busiest->avg_load); 8503 capa_now += local->group_capacity * 8504 min(local->load_per_task, local->avg_load); 8505 capa_now /= SCHED_CAPACITY_SCALE; 8506 8507 /* Amount of load we'd subtract */ 8508 if (busiest->avg_load > scaled_busy_load_per_task) { 8509 capa_move += busiest->group_capacity * 8510 min(busiest->load_per_task, 8511 busiest->avg_load - scaled_busy_load_per_task); 8512 } 8513 8514 /* Amount of load we'd add */ 8515 if (busiest->avg_load * busiest->group_capacity < 8516 busiest->load_per_task * SCHED_CAPACITY_SCALE) { 8517 tmp = (busiest->avg_load * busiest->group_capacity) / 8518 local->group_capacity; 8519 } else { 8520 tmp = (busiest->load_per_task * SCHED_CAPACITY_SCALE) / 8521 local->group_capacity; 8522 } 8523 capa_move += local->group_capacity * 8524 min(local->load_per_task, local->avg_load + tmp); 8525 capa_move /= SCHED_CAPACITY_SCALE; 8526 8527 /* Move if we gain throughput */ 8528 if (capa_move > capa_now) 8529 env->imbalance = busiest->load_per_task; 8530 } 8531 8532 /** 8533 * calculate_imbalance - Calculate the amount of imbalance present within the 8534 * groups of a given sched_domain during load balance. 8535 * @env: load balance environment 8536 * @sds: statistics of the sched_domain whose imbalance is to be calculated. 8537 */ 8538 static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds) 8539 { 8540 unsigned long max_pull, load_above_capacity = ~0UL; 8541 struct sg_lb_stats *local, *busiest; 8542 8543 local = &sds->local_stat; 8544 busiest = &sds->busiest_stat; 8545 8546 if (busiest->group_type == group_imbalanced) { 8547 /* 8548 * In the group_imb case we cannot rely on group-wide averages 8549 * to ensure CPU-load equilibrium, look at wider averages. XXX 8550 */ 8551 busiest->load_per_task = 8552 min(busiest->load_per_task, sds->avg_load); 8553 } 8554 8555 /* 8556 * Avg load of busiest sg can be less and avg load of local sg can 8557 * be greater than avg load across all sgs of sd because avg load 8558 * factors in sg capacity and sgs with smaller group_type are 8559 * skipped when updating the busiest sg: 8560 */ 8561 if (busiest->group_type != group_misfit_task && 8562 (busiest->avg_load <= sds->avg_load || 8563 local->avg_load >= sds->avg_load)) { 8564 env->imbalance = 0; 8565 return fix_small_imbalance(env, sds); 8566 } 8567 8568 /* 8569 * If there aren't any idle CPUs, avoid creating some. 8570 */ 8571 if (busiest->group_type == group_overloaded && 8572 local->group_type == group_overloaded) { 8573 load_above_capacity = busiest->sum_nr_running * SCHED_CAPACITY_SCALE; 8574 if (load_above_capacity > busiest->group_capacity) { 8575 load_above_capacity -= busiest->group_capacity; 8576 load_above_capacity *= scale_load_down(NICE_0_LOAD); 8577 load_above_capacity /= busiest->group_capacity; 8578 } else 8579 load_above_capacity = ~0UL; 8580 } 8581 8582 /* 8583 * We're trying to get all the CPUs to the average_load, so we don't 8584 * want to push ourselves above the average load, nor do we wish to 8585 * reduce the max loaded CPU below the average load. At the same time, 8586 * we also don't want to reduce the group load below the group 8587 * capacity. Thus we look for the minimum possible imbalance. 8588 */ 8589 max_pull = min(busiest->avg_load - sds->avg_load, load_above_capacity); 8590 8591 /* How much load to actually move to equalise the imbalance */ 8592 env->imbalance = min( 8593 max_pull * busiest->group_capacity, 8594 (sds->avg_load - local->avg_load) * local->group_capacity 8595 ) / SCHED_CAPACITY_SCALE; 8596 8597 /* Boost imbalance to allow misfit task to be balanced. */ 8598 if (busiest->group_type == group_misfit_task) { 8599 env->imbalance = max_t(long, env->imbalance, 8600 busiest->group_misfit_task_load); 8601 } 8602 8603 /* 8604 * if *imbalance is less than the average load per runnable task 8605 * there is no guarantee that any tasks will be moved so we'll have 8606 * a think about bumping its value to force at least one task to be 8607 * moved 8608 */ 8609 if (env->imbalance < busiest->load_per_task) 8610 return fix_small_imbalance(env, sds); 8611 } 8612 8613 /******* find_busiest_group() helpers end here *********************/ 8614 8615 /** 8616 * find_busiest_group - Returns the busiest group within the sched_domain 8617 * if there is an imbalance. 8618 * 8619 * Also calculates the amount of weighted load which should be moved 8620 * to restore balance. 8621 * 8622 * @env: The load balancing environment. 8623 * 8624 * Return: - The busiest group if imbalance exists. 8625 */ 8626 static struct sched_group *find_busiest_group(struct lb_env *env) 8627 { 8628 struct sg_lb_stats *local, *busiest; 8629 struct sd_lb_stats sds; 8630 8631 init_sd_lb_stats(&sds); 8632 8633 /* 8634 * Compute the various statistics relavent for load balancing at 8635 * this level. 8636 */ 8637 update_sd_lb_stats(env, &sds); 8638 8639 if (static_branch_unlikely(&sched_energy_present)) { 8640 struct root_domain *rd = env->dst_rq->rd; 8641 8642 if (rcu_dereference(rd->pd) && !READ_ONCE(rd->overutilized)) 8643 goto out_balanced; 8644 } 8645 8646 local = &sds.local_stat; 8647 busiest = &sds.busiest_stat; 8648 8649 /* ASYM feature bypasses nice load balance check */ 8650 if (check_asym_packing(env, &sds)) 8651 return sds.busiest; 8652 8653 /* There is no busy sibling group to pull tasks from */ 8654 if (!sds.busiest || busiest->sum_nr_running == 0) 8655 goto out_balanced; 8656 8657 /* XXX broken for overlapping NUMA groups */ 8658 sds.avg_load = (SCHED_CAPACITY_SCALE * sds.total_load) 8659 / sds.total_capacity; 8660 8661 /* 8662 * If the busiest group is imbalanced the below checks don't 8663 * work because they assume all things are equal, which typically 8664 * isn't true due to cpus_allowed constraints and the like. 8665 */ 8666 if (busiest->group_type == group_imbalanced) 8667 goto force_balance; 8668 8669 /* 8670 * When dst_cpu is idle, prevent SMP nice and/or asymmetric group 8671 * capacities from resulting in underutilization due to avg_load. 8672 */ 8673 if (env->idle != CPU_NOT_IDLE && group_has_capacity(env, local) && 8674 busiest->group_no_capacity) 8675 goto force_balance; 8676 8677 /* Misfit tasks should be dealt with regardless of the avg load */ 8678 if (busiest->group_type == group_misfit_task) 8679 goto force_balance; 8680 8681 /* 8682 * If the local group is busier than the selected busiest group 8683 * don't try and pull any tasks. 8684 */ 8685 if (local->avg_load >= busiest->avg_load) 8686 goto out_balanced; 8687 8688 /* 8689 * Don't pull any tasks if this group is already above the domain 8690 * average load. 8691 */ 8692 if (local->avg_load >= sds.avg_load) 8693 goto out_balanced; 8694 8695 if (env->idle == CPU_IDLE) { 8696 /* 8697 * This CPU is idle. If the busiest group is not overloaded 8698 * and there is no imbalance between this and busiest group 8699 * wrt idle CPUs, it is balanced. The imbalance becomes 8700 * significant if the diff is greater than 1 otherwise we 8701 * might end up to just move the imbalance on another group 8702 */ 8703 if ((busiest->group_type != group_overloaded) && 8704 (local->idle_cpus <= (busiest->idle_cpus + 1))) 8705 goto out_balanced; 8706 } else { 8707 /* 8708 * In the CPU_NEWLY_IDLE, CPU_NOT_IDLE cases, use 8709 * imbalance_pct to be conservative. 8710 */ 8711 if (100 * busiest->avg_load <= 8712 env->sd->imbalance_pct * local->avg_load) 8713 goto out_balanced; 8714 } 8715 8716 force_balance: 8717 /* Looks like there is an imbalance. Compute it */ 8718 env->src_grp_type = busiest->group_type; 8719 calculate_imbalance(env, &sds); 8720 return env->imbalance ? sds.busiest : NULL; 8721 8722 out_balanced: 8723 env->imbalance = 0; 8724 return NULL; 8725 } 8726 8727 /* 8728 * find_busiest_queue - find the busiest runqueue among the CPUs in the group. 8729 */ 8730 static struct rq *find_busiest_queue(struct lb_env *env, 8731 struct sched_group *group) 8732 { 8733 struct rq *busiest = NULL, *rq; 8734 unsigned long busiest_load = 0, busiest_capacity = 1; 8735 int i; 8736 8737 for_each_cpu_and(i, sched_group_span(group), env->cpus) { 8738 unsigned long capacity, wl; 8739 enum fbq_type rt; 8740 8741 rq = cpu_rq(i); 8742 rt = fbq_classify_rq(rq); 8743 8744 /* 8745 * We classify groups/runqueues into three groups: 8746 * - regular: there are !numa tasks 8747 * - remote: there are numa tasks that run on the 'wrong' node 8748 * - all: there is no distinction 8749 * 8750 * In order to avoid migrating ideally placed numa tasks, 8751 * ignore those when there's better options. 8752 * 8753 * If we ignore the actual busiest queue to migrate another 8754 * task, the next balance pass can still reduce the busiest 8755 * queue by moving tasks around inside the node. 8756 * 8757 * If we cannot move enough load due to this classification 8758 * the next pass will adjust the group classification and 8759 * allow migration of more tasks. 8760 * 8761 * Both cases only affect the total convergence complexity. 8762 */ 8763 if (rt > env->fbq_type) 8764 continue; 8765 8766 /* 8767 * For ASYM_CPUCAPACITY domains with misfit tasks we simply 8768 * seek the "biggest" misfit task. 8769 */ 8770 if (env->src_grp_type == group_misfit_task) { 8771 if (rq->misfit_task_load > busiest_load) { 8772 busiest_load = rq->misfit_task_load; 8773 busiest = rq; 8774 } 8775 8776 continue; 8777 } 8778 8779 capacity = capacity_of(i); 8780 8781 /* 8782 * For ASYM_CPUCAPACITY domains, don't pick a CPU that could 8783 * eventually lead to active_balancing high->low capacity. 8784 * Higher per-CPU capacity is considered better than balancing 8785 * average load. 8786 */ 8787 if (env->sd->flags & SD_ASYM_CPUCAPACITY && 8788 capacity_of(env->dst_cpu) < capacity && 8789 rq->nr_running == 1) 8790 continue; 8791 8792 wl = weighted_cpuload(rq); 8793 8794 /* 8795 * When comparing with imbalance, use weighted_cpuload() 8796 * which is not scaled with the CPU capacity. 8797 */ 8798 8799 if (rq->nr_running == 1 && wl > env->imbalance && 8800 !check_cpu_capacity(rq, env->sd)) 8801 continue; 8802 8803 /* 8804 * For the load comparisons with the other CPU's, consider 8805 * the weighted_cpuload() scaled with the CPU capacity, so 8806 * that the load can be moved away from the CPU that is 8807 * potentially running at a lower capacity. 8808 * 8809 * Thus we're looking for max(wl_i / capacity_i), crosswise 8810 * multiplication to rid ourselves of the division works out 8811 * to: wl_i * capacity_j > wl_j * capacity_i; where j is 8812 * our previous maximum. 8813 */ 8814 if (wl * busiest_capacity > busiest_load * capacity) { 8815 busiest_load = wl; 8816 busiest_capacity = capacity; 8817 busiest = rq; 8818 } 8819 } 8820 8821 return busiest; 8822 } 8823 8824 /* 8825 * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but 8826 * so long as it is large enough. 8827 */ 8828 #define MAX_PINNED_INTERVAL 512 8829 8830 static int need_active_balance(struct lb_env *env) 8831 { 8832 struct sched_domain *sd = env->sd; 8833 8834 if (env->idle == CPU_NEWLY_IDLE) { 8835 8836 /* 8837 * ASYM_PACKING needs to force migrate tasks from busy but 8838 * lower priority CPUs in order to pack all tasks in the 8839 * highest priority CPUs. 8840 */ 8841 if ((sd->flags & SD_ASYM_PACKING) && 8842 sched_asym_prefer(env->dst_cpu, env->src_cpu)) 8843 return 1; 8844 } 8845 8846 /* 8847 * The dst_cpu is idle and the src_cpu CPU has only 1 CFS task. 8848 * It's worth migrating the task if the src_cpu's capacity is reduced 8849 * because of other sched_class or IRQs if more capacity stays 8850 * available on dst_cpu. 8851 */ 8852 if ((env->idle != CPU_NOT_IDLE) && 8853 (env->src_rq->cfs.h_nr_running == 1)) { 8854 if ((check_cpu_capacity(env->src_rq, sd)) && 8855 (capacity_of(env->src_cpu)*sd->imbalance_pct < capacity_of(env->dst_cpu)*100)) 8856 return 1; 8857 } 8858 8859 if (env->src_grp_type == group_misfit_task) 8860 return 1; 8861 8862 return unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2); 8863 } 8864 8865 static int active_load_balance_cpu_stop(void *data); 8866 8867 static int should_we_balance(struct lb_env *env) 8868 { 8869 struct sched_group *sg = env->sd->groups; 8870 int cpu, balance_cpu = -1; 8871 8872 /* 8873 * Ensure the balancing environment is consistent; can happen 8874 * when the softirq triggers 'during' hotplug. 8875 */ 8876 if (!cpumask_test_cpu(env->dst_cpu, env->cpus)) 8877 return 0; 8878 8879 /* 8880 * In the newly idle case, we will allow all the CPUs 8881 * to do the newly idle load balance. 8882 */ 8883 if (env->idle == CPU_NEWLY_IDLE) 8884 return 1; 8885 8886 /* Try to find first idle CPU */ 8887 for_each_cpu_and(cpu, group_balance_mask(sg), env->cpus) { 8888 if (!idle_cpu(cpu)) 8889 continue; 8890 8891 balance_cpu = cpu; 8892 break; 8893 } 8894 8895 if (balance_cpu == -1) 8896 balance_cpu = group_balance_cpu(sg); 8897 8898 /* 8899 * First idle CPU or the first CPU(busiest) in this sched group 8900 * is eligible for doing load balancing at this and above domains. 8901 */ 8902 return balance_cpu == env->dst_cpu; 8903 } 8904 8905 /* 8906 * Check this_cpu to ensure it is balanced within domain. Attempt to move 8907 * tasks if there is an imbalance. 8908 */ 8909 static int load_balance(int this_cpu, struct rq *this_rq, 8910 struct sched_domain *sd, enum cpu_idle_type idle, 8911 int *continue_balancing) 8912 { 8913 int ld_moved, cur_ld_moved, active_balance = 0; 8914 struct sched_domain *sd_parent = sd->parent; 8915 struct sched_group *group; 8916 struct rq *busiest; 8917 struct rq_flags rf; 8918 struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask); 8919 8920 struct lb_env env = { 8921 .sd = sd, 8922 .dst_cpu = this_cpu, 8923 .dst_rq = this_rq, 8924 .dst_grpmask = sched_group_span(sd->groups), 8925 .idle = idle, 8926 .loop_break = sched_nr_migrate_break, 8927 .cpus = cpus, 8928 .fbq_type = all, 8929 .tasks = LIST_HEAD_INIT(env.tasks), 8930 }; 8931 8932 cpumask_and(cpus, sched_domain_span(sd), cpu_active_mask); 8933 8934 schedstat_inc(sd->lb_count[idle]); 8935 8936 redo: 8937 if (!should_we_balance(&env)) { 8938 *continue_balancing = 0; 8939 goto out_balanced; 8940 } 8941 8942 group = find_busiest_group(&env); 8943 if (!group) { 8944 schedstat_inc(sd->lb_nobusyg[idle]); 8945 goto out_balanced; 8946 } 8947 8948 busiest = find_busiest_queue(&env, group); 8949 if (!busiest) { 8950 schedstat_inc(sd->lb_nobusyq[idle]); 8951 goto out_balanced; 8952 } 8953 8954 BUG_ON(busiest == env.dst_rq); 8955 8956 schedstat_add(sd->lb_imbalance[idle], env.imbalance); 8957 8958 env.src_cpu = busiest->cpu; 8959 env.src_rq = busiest; 8960 8961 ld_moved = 0; 8962 if (busiest->nr_running > 1) { 8963 /* 8964 * Attempt to move tasks. If find_busiest_group has found 8965 * an imbalance but busiest->nr_running <= 1, the group is 8966 * still unbalanced. ld_moved simply stays zero, so it is 8967 * correctly treated as an imbalance. 8968 */ 8969 env.flags |= LBF_ALL_PINNED; 8970 env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running); 8971 8972 more_balance: 8973 rq_lock_irqsave(busiest, &rf); 8974 update_rq_clock(busiest); 8975 8976 /* 8977 * cur_ld_moved - load moved in current iteration 8978 * ld_moved - cumulative load moved across iterations 8979 */ 8980 cur_ld_moved = detach_tasks(&env); 8981 8982 /* 8983 * We've detached some tasks from busiest_rq. Every 8984 * task is masked "TASK_ON_RQ_MIGRATING", so we can safely 8985 * unlock busiest->lock, and we are able to be sure 8986 * that nobody can manipulate the tasks in parallel. 8987 * See task_rq_lock() family for the details. 8988 */ 8989 8990 rq_unlock(busiest, &rf); 8991 8992 if (cur_ld_moved) { 8993 attach_tasks(&env); 8994 ld_moved += cur_ld_moved; 8995 } 8996 8997 local_irq_restore(rf.flags); 8998 8999 if (env.flags & LBF_NEED_BREAK) { 9000 env.flags &= ~LBF_NEED_BREAK; 9001 goto more_balance; 9002 } 9003 9004 /* 9005 * Revisit (affine) tasks on src_cpu that couldn't be moved to 9006 * us and move them to an alternate dst_cpu in our sched_group 9007 * where they can run. The upper limit on how many times we 9008 * iterate on same src_cpu is dependent on number of CPUs in our 9009 * sched_group. 9010 * 9011 * This changes load balance semantics a bit on who can move 9012 * load to a given_cpu. In addition to the given_cpu itself 9013 * (or a ilb_cpu acting on its behalf where given_cpu is 9014 * nohz-idle), we now have balance_cpu in a position to move 9015 * load to given_cpu. In rare situations, this may cause 9016 * conflicts (balance_cpu and given_cpu/ilb_cpu deciding 9017 * _independently_ and at _same_ time to move some load to 9018 * given_cpu) causing exceess load to be moved to given_cpu. 9019 * This however should not happen so much in practice and 9020 * moreover subsequent load balance cycles should correct the 9021 * excess load moved. 9022 */ 9023 if ((env.flags & LBF_DST_PINNED) && env.imbalance > 0) { 9024 9025 /* Prevent to re-select dst_cpu via env's CPUs */ 9026 cpumask_clear_cpu(env.dst_cpu, env.cpus); 9027 9028 env.dst_rq = cpu_rq(env.new_dst_cpu); 9029 env.dst_cpu = env.new_dst_cpu; 9030 env.flags &= ~LBF_DST_PINNED; 9031 env.loop = 0; 9032 env.loop_break = sched_nr_migrate_break; 9033 9034 /* 9035 * Go back to "more_balance" rather than "redo" since we 9036 * need to continue with same src_cpu. 9037 */ 9038 goto more_balance; 9039 } 9040 9041 /* 9042 * We failed to reach balance because of affinity. 9043 */ 9044 if (sd_parent) { 9045 int *group_imbalance = &sd_parent->groups->sgc->imbalance; 9046 9047 if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0) 9048 *group_imbalance = 1; 9049 } 9050 9051 /* All tasks on this runqueue were pinned by CPU affinity */ 9052 if (unlikely(env.flags & LBF_ALL_PINNED)) { 9053 cpumask_clear_cpu(cpu_of(busiest), cpus); 9054 /* 9055 * Attempting to continue load balancing at the current 9056 * sched_domain level only makes sense if there are 9057 * active CPUs remaining as possible busiest CPUs to 9058 * pull load from which are not contained within the 9059 * destination group that is receiving any migrated 9060 * load. 9061 */ 9062 if (!cpumask_subset(cpus, env.dst_grpmask)) { 9063 env.loop = 0; 9064 env.loop_break = sched_nr_migrate_break; 9065 goto redo; 9066 } 9067 goto out_all_pinned; 9068 } 9069 } 9070 9071 if (!ld_moved) { 9072 schedstat_inc(sd->lb_failed[idle]); 9073 /* 9074 * Increment the failure counter only on periodic balance. 9075 * We do not want newidle balance, which can be very 9076 * frequent, pollute the failure counter causing 9077 * excessive cache_hot migrations and active balances. 9078 */ 9079 if (idle != CPU_NEWLY_IDLE) 9080 sd->nr_balance_failed++; 9081 9082 if (need_active_balance(&env)) { 9083 unsigned long flags; 9084 9085 raw_spin_lock_irqsave(&busiest->lock, flags); 9086 9087 /* 9088 * Don't kick the active_load_balance_cpu_stop, 9089 * if the curr task on busiest CPU can't be 9090 * moved to this_cpu: 9091 */ 9092 if (!cpumask_test_cpu(this_cpu, &busiest->curr->cpus_allowed)) { 9093 raw_spin_unlock_irqrestore(&busiest->lock, 9094 flags); 9095 env.flags |= LBF_ALL_PINNED; 9096 goto out_one_pinned; 9097 } 9098 9099 /* 9100 * ->active_balance synchronizes accesses to 9101 * ->active_balance_work. Once set, it's cleared 9102 * only after active load balance is finished. 9103 */ 9104 if (!busiest->active_balance) { 9105 busiest->active_balance = 1; 9106 busiest->push_cpu = this_cpu; 9107 active_balance = 1; 9108 } 9109 raw_spin_unlock_irqrestore(&busiest->lock, flags); 9110 9111 if (active_balance) { 9112 stop_one_cpu_nowait(cpu_of(busiest), 9113 active_load_balance_cpu_stop, busiest, 9114 &busiest->active_balance_work); 9115 } 9116 9117 /* We've kicked active balancing, force task migration. */ 9118 sd->nr_balance_failed = sd->cache_nice_tries+1; 9119 } 9120 } else 9121 sd->nr_balance_failed = 0; 9122 9123 if (likely(!active_balance)) { 9124 /* We were unbalanced, so reset the balancing interval */ 9125 sd->balance_interval = sd->min_interval; 9126 } else { 9127 /* 9128 * If we've begun active balancing, start to back off. This 9129 * case may not be covered by the all_pinned logic if there 9130 * is only 1 task on the busy runqueue (because we don't call 9131 * detach_tasks). 9132 */ 9133 if (sd->balance_interval < sd->max_interval) 9134 sd->balance_interval *= 2; 9135 } 9136 9137 goto out; 9138 9139 out_balanced: 9140 /* 9141 * We reach balance although we may have faced some affinity 9142 * constraints. Clear the imbalance flag if it was set. 9143 */ 9144 if (sd_parent) { 9145 int *group_imbalance = &sd_parent->groups->sgc->imbalance; 9146 9147 if (*group_imbalance) 9148 *group_imbalance = 0; 9149 } 9150 9151 out_all_pinned: 9152 /* 9153 * We reach balance because all tasks are pinned at this level so 9154 * we can't migrate them. Let the imbalance flag set so parent level 9155 * can try to migrate them. 9156 */ 9157 schedstat_inc(sd->lb_balanced[idle]); 9158 9159 sd->nr_balance_failed = 0; 9160 9161 out_one_pinned: 9162 ld_moved = 0; 9163 9164 /* 9165 * idle_balance() disregards balance intervals, so we could repeatedly 9166 * reach this code, which would lead to balance_interval skyrocketting 9167 * in a short amount of time. Skip the balance_interval increase logic 9168 * to avoid that. 9169 */ 9170 if (env.idle == CPU_NEWLY_IDLE) 9171 goto out; 9172 9173 /* tune up the balancing interval */ 9174 if ((env.flags & LBF_ALL_PINNED && 9175 sd->balance_interval < MAX_PINNED_INTERVAL) || 9176 sd->balance_interval < sd->max_interval) 9177 sd->balance_interval *= 2; 9178 out: 9179 return ld_moved; 9180 } 9181 9182 static inline unsigned long 9183 get_sd_balance_interval(struct sched_domain *sd, int cpu_busy) 9184 { 9185 unsigned long interval = sd->balance_interval; 9186 9187 if (cpu_busy) 9188 interval *= sd->busy_factor; 9189 9190 /* scale ms to jiffies */ 9191 interval = msecs_to_jiffies(interval); 9192 interval = clamp(interval, 1UL, max_load_balance_interval); 9193 9194 return interval; 9195 } 9196 9197 static inline void 9198 update_next_balance(struct sched_domain *sd, unsigned long *next_balance) 9199 { 9200 unsigned long interval, next; 9201 9202 /* used by idle balance, so cpu_busy = 0 */ 9203 interval = get_sd_balance_interval(sd, 0); 9204 next = sd->last_balance + interval; 9205 9206 if (time_after(*next_balance, next)) 9207 *next_balance = next; 9208 } 9209 9210 /* 9211 * active_load_balance_cpu_stop is run by the CPU stopper. It pushes 9212 * running tasks off the busiest CPU onto idle CPUs. It requires at 9213 * least 1 task to be running on each physical CPU where possible, and 9214 * avoids physical / logical imbalances. 9215 */ 9216 static int active_load_balance_cpu_stop(void *data) 9217 { 9218 struct rq *busiest_rq = data; 9219 int busiest_cpu = cpu_of(busiest_rq); 9220 int target_cpu = busiest_rq->push_cpu; 9221 struct rq *target_rq = cpu_rq(target_cpu); 9222 struct sched_domain *sd; 9223 struct task_struct *p = NULL; 9224 struct rq_flags rf; 9225 9226 rq_lock_irq(busiest_rq, &rf); 9227 /* 9228 * Between queueing the stop-work and running it is a hole in which 9229 * CPUs can become inactive. We should not move tasks from or to 9230 * inactive CPUs. 9231 */ 9232 if (!cpu_active(busiest_cpu) || !cpu_active(target_cpu)) 9233 goto out_unlock; 9234 9235 /* Make sure the requested CPU hasn't gone down in the meantime: */ 9236 if (unlikely(busiest_cpu != smp_processor_id() || 9237 !busiest_rq->active_balance)) 9238 goto out_unlock; 9239 9240 /* Is there any task to move? */ 9241 if (busiest_rq->nr_running <= 1) 9242 goto out_unlock; 9243 9244 /* 9245 * This condition is "impossible", if it occurs 9246 * we need to fix it. Originally reported by 9247 * Bjorn Helgaas on a 128-CPU setup. 9248 */ 9249 BUG_ON(busiest_rq == target_rq); 9250 9251 /* Search for an sd spanning us and the target CPU. */ 9252 rcu_read_lock(); 9253 for_each_domain(target_cpu, sd) { 9254 if ((sd->flags & SD_LOAD_BALANCE) && 9255 cpumask_test_cpu(busiest_cpu, sched_domain_span(sd))) 9256 break; 9257 } 9258 9259 if (likely(sd)) { 9260 struct lb_env env = { 9261 .sd = sd, 9262 .dst_cpu = target_cpu, 9263 .dst_rq = target_rq, 9264 .src_cpu = busiest_rq->cpu, 9265 .src_rq = busiest_rq, 9266 .idle = CPU_IDLE, 9267 /* 9268 * can_migrate_task() doesn't need to compute new_dst_cpu 9269 * for active balancing. Since we have CPU_IDLE, but no 9270 * @dst_grpmask we need to make that test go away with lying 9271 * about DST_PINNED. 9272 */ 9273 .flags = LBF_DST_PINNED, 9274 }; 9275 9276 schedstat_inc(sd->alb_count); 9277 update_rq_clock(busiest_rq); 9278 9279 p = detach_one_task(&env); 9280 if (p) { 9281 schedstat_inc(sd->alb_pushed); 9282 /* Active balancing done, reset the failure counter. */ 9283 sd->nr_balance_failed = 0; 9284 } else { 9285 schedstat_inc(sd->alb_failed); 9286 } 9287 } 9288 rcu_read_unlock(); 9289 out_unlock: 9290 busiest_rq->active_balance = 0; 9291 rq_unlock(busiest_rq, &rf); 9292 9293 if (p) 9294 attach_one_task(target_rq, p); 9295 9296 local_irq_enable(); 9297 9298 return 0; 9299 } 9300 9301 static DEFINE_SPINLOCK(balancing); 9302 9303 /* 9304 * Scale the max load_balance interval with the number of CPUs in the system. 9305 * This trades load-balance latency on larger machines for less cross talk. 9306 */ 9307 void update_max_interval(void) 9308 { 9309 max_load_balance_interval = HZ*num_online_cpus()/10; 9310 } 9311 9312 /* 9313 * It checks each scheduling domain to see if it is due to be balanced, 9314 * and initiates a balancing operation if so. 9315 * 9316 * Balancing parameters are set up in init_sched_domains. 9317 */ 9318 static void rebalance_domains(struct rq *rq, enum cpu_idle_type idle) 9319 { 9320 int continue_balancing = 1; 9321 int cpu = rq->cpu; 9322 unsigned long interval; 9323 struct sched_domain *sd; 9324 /* Earliest time when we have to do rebalance again */ 9325 unsigned long next_balance = jiffies + 60*HZ; 9326 int update_next_balance = 0; 9327 int need_serialize, need_decay = 0; 9328 u64 max_cost = 0; 9329 9330 rcu_read_lock(); 9331 for_each_domain(cpu, sd) { 9332 /* 9333 * Decay the newidle max times here because this is a regular 9334 * visit to all the domains. Decay ~1% per second. 9335 */ 9336 if (time_after(jiffies, sd->next_decay_max_lb_cost)) { 9337 sd->max_newidle_lb_cost = 9338 (sd->max_newidle_lb_cost * 253) / 256; 9339 sd->next_decay_max_lb_cost = jiffies + HZ; 9340 need_decay = 1; 9341 } 9342 max_cost += sd->max_newidle_lb_cost; 9343 9344 if (!(sd->flags & SD_LOAD_BALANCE)) 9345 continue; 9346 9347 /* 9348 * Stop the load balance at this level. There is another 9349 * CPU in our sched group which is doing load balancing more 9350 * actively. 9351 */ 9352 if (!continue_balancing) { 9353 if (need_decay) 9354 continue; 9355 break; 9356 } 9357 9358 interval = get_sd_balance_interval(sd, idle != CPU_IDLE); 9359 9360 need_serialize = sd->flags & SD_SERIALIZE; 9361 if (need_serialize) { 9362 if (!spin_trylock(&balancing)) 9363 goto out; 9364 } 9365 9366 if (time_after_eq(jiffies, sd->last_balance + interval)) { 9367 if (load_balance(cpu, rq, sd, idle, &continue_balancing)) { 9368 /* 9369 * The LBF_DST_PINNED logic could have changed 9370 * env->dst_cpu, so we can't know our idle 9371 * state even if we migrated tasks. Update it. 9372 */ 9373 idle = idle_cpu(cpu) ? CPU_IDLE : CPU_NOT_IDLE; 9374 } 9375 sd->last_balance = jiffies; 9376 interval = get_sd_balance_interval(sd, idle != CPU_IDLE); 9377 } 9378 if (need_serialize) 9379 spin_unlock(&balancing); 9380 out: 9381 if (time_after(next_balance, sd->last_balance + interval)) { 9382 next_balance = sd->last_balance + interval; 9383 update_next_balance = 1; 9384 } 9385 } 9386 if (need_decay) { 9387 /* 9388 * Ensure the rq-wide value also decays but keep it at a 9389 * reasonable floor to avoid funnies with rq->avg_idle. 9390 */ 9391 rq->max_idle_balance_cost = 9392 max((u64)sysctl_sched_migration_cost, max_cost); 9393 } 9394 rcu_read_unlock(); 9395 9396 /* 9397 * next_balance will be updated only when there is a need. 9398 * When the cpu is attached to null domain for ex, it will not be 9399 * updated. 9400 */ 9401 if (likely(update_next_balance)) { 9402 rq->next_balance = next_balance; 9403 9404 #ifdef CONFIG_NO_HZ_COMMON 9405 /* 9406 * If this CPU has been elected to perform the nohz idle 9407 * balance. Other idle CPUs have already rebalanced with 9408 * nohz_idle_balance() and nohz.next_balance has been 9409 * updated accordingly. This CPU is now running the idle load 9410 * balance for itself and we need to update the 9411 * nohz.next_balance accordingly. 9412 */ 9413 if ((idle == CPU_IDLE) && time_after(nohz.next_balance, rq->next_balance)) 9414 nohz.next_balance = rq->next_balance; 9415 #endif 9416 } 9417 } 9418 9419 static inline int on_null_domain(struct rq *rq) 9420 { 9421 return unlikely(!rcu_dereference_sched(rq->sd)); 9422 } 9423 9424 #ifdef CONFIG_NO_HZ_COMMON 9425 /* 9426 * idle load balancing details 9427 * - When one of the busy CPUs notice that there may be an idle rebalancing 9428 * needed, they will kick the idle load balancer, which then does idle 9429 * load balancing for all the idle CPUs. 9430 */ 9431 9432 static inline int find_new_ilb(void) 9433 { 9434 int ilb = cpumask_first(nohz.idle_cpus_mask); 9435 9436 if (ilb < nr_cpu_ids && idle_cpu(ilb)) 9437 return ilb; 9438 9439 return nr_cpu_ids; 9440 } 9441 9442 /* 9443 * Kick a CPU to do the nohz balancing, if it is time for it. We pick the 9444 * nohz_load_balancer CPU (if there is one) otherwise fallback to any idle 9445 * CPU (if there is one). 9446 */ 9447 static void kick_ilb(unsigned int flags) 9448 { 9449 int ilb_cpu; 9450 9451 nohz.next_balance++; 9452 9453 ilb_cpu = find_new_ilb(); 9454 9455 if (ilb_cpu >= nr_cpu_ids) 9456 return; 9457 9458 flags = atomic_fetch_or(flags, nohz_flags(ilb_cpu)); 9459 if (flags & NOHZ_KICK_MASK) 9460 return; 9461 9462 /* 9463 * Use smp_send_reschedule() instead of resched_cpu(). 9464 * This way we generate a sched IPI on the target CPU which 9465 * is idle. And the softirq performing nohz idle load balance 9466 * will be run before returning from the IPI. 9467 */ 9468 smp_send_reschedule(ilb_cpu); 9469 } 9470 9471 /* 9472 * Current heuristic for kicking the idle load balancer in the presence 9473 * of an idle cpu in the system. 9474 * - This rq has more than one task. 9475 * - This rq has at least one CFS task and the capacity of the CPU is 9476 * significantly reduced because of RT tasks or IRQs. 9477 * - At parent of LLC scheduler domain level, this cpu's scheduler group has 9478 * multiple busy cpu. 9479 * - For SD_ASYM_PACKING, if the lower numbered cpu's in the scheduler 9480 * domain span are idle. 9481 */ 9482 static void nohz_balancer_kick(struct rq *rq) 9483 { 9484 unsigned long now = jiffies; 9485 struct sched_domain_shared *sds; 9486 struct sched_domain *sd; 9487 int nr_busy, i, cpu = rq->cpu; 9488 unsigned int flags = 0; 9489 9490 if (unlikely(rq->idle_balance)) 9491 return; 9492 9493 /* 9494 * We may be recently in ticked or tickless idle mode. At the first 9495 * busy tick after returning from idle, we will update the busy stats. 9496 */ 9497 nohz_balance_exit_idle(rq); 9498 9499 /* 9500 * None are in tickless mode and hence no need for NOHZ idle load 9501 * balancing. 9502 */ 9503 if (likely(!atomic_read(&nohz.nr_cpus))) 9504 return; 9505 9506 if (READ_ONCE(nohz.has_blocked) && 9507 time_after(now, READ_ONCE(nohz.next_blocked))) 9508 flags = NOHZ_STATS_KICK; 9509 9510 if (time_before(now, nohz.next_balance)) 9511 goto out; 9512 9513 if (rq->nr_running >= 2 || rq->misfit_task_load) { 9514 flags = NOHZ_KICK_MASK; 9515 goto out; 9516 } 9517 9518 rcu_read_lock(); 9519 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); 9520 if (sds) { 9521 /* 9522 * XXX: write a coherent comment on why we do this. 9523 * See also: http://lkml.kernel.org/r/20111202010832.602203411@sbsiddha-desk.sc.intel.com 9524 */ 9525 nr_busy = atomic_read(&sds->nr_busy_cpus); 9526 if (nr_busy > 1) { 9527 flags = NOHZ_KICK_MASK; 9528 goto unlock; 9529 } 9530 9531 } 9532 9533 sd = rcu_dereference(rq->sd); 9534 if (sd) { 9535 if ((rq->cfs.h_nr_running >= 1) && 9536 check_cpu_capacity(rq, sd)) { 9537 flags = NOHZ_KICK_MASK; 9538 goto unlock; 9539 } 9540 } 9541 9542 sd = rcu_dereference(per_cpu(sd_asym_packing, cpu)); 9543 if (sd) { 9544 for_each_cpu(i, sched_domain_span(sd)) { 9545 if (i == cpu || 9546 !cpumask_test_cpu(i, nohz.idle_cpus_mask)) 9547 continue; 9548 9549 if (sched_asym_prefer(i, cpu)) { 9550 flags = NOHZ_KICK_MASK; 9551 goto unlock; 9552 } 9553 } 9554 } 9555 unlock: 9556 rcu_read_unlock(); 9557 out: 9558 if (flags) 9559 kick_ilb(flags); 9560 } 9561 9562 static void set_cpu_sd_state_busy(int cpu) 9563 { 9564 struct sched_domain *sd; 9565 9566 rcu_read_lock(); 9567 sd = rcu_dereference(per_cpu(sd_llc, cpu)); 9568 9569 if (!sd || !sd->nohz_idle) 9570 goto unlock; 9571 sd->nohz_idle = 0; 9572 9573 atomic_inc(&sd->shared->nr_busy_cpus); 9574 unlock: 9575 rcu_read_unlock(); 9576 } 9577 9578 void nohz_balance_exit_idle(struct rq *rq) 9579 { 9580 SCHED_WARN_ON(rq != this_rq()); 9581 9582 if (likely(!rq->nohz_tick_stopped)) 9583 return; 9584 9585 rq->nohz_tick_stopped = 0; 9586 cpumask_clear_cpu(rq->cpu, nohz.idle_cpus_mask); 9587 atomic_dec(&nohz.nr_cpus); 9588 9589 set_cpu_sd_state_busy(rq->cpu); 9590 } 9591 9592 static void set_cpu_sd_state_idle(int cpu) 9593 { 9594 struct sched_domain *sd; 9595 9596 rcu_read_lock(); 9597 sd = rcu_dereference(per_cpu(sd_llc, cpu)); 9598 9599 if (!sd || sd->nohz_idle) 9600 goto unlock; 9601 sd->nohz_idle = 1; 9602 9603 atomic_dec(&sd->shared->nr_busy_cpus); 9604 unlock: 9605 rcu_read_unlock(); 9606 } 9607 9608 /* 9609 * This routine will record that the CPU is going idle with tick stopped. 9610 * This info will be used in performing idle load balancing in the future. 9611 */ 9612 void nohz_balance_enter_idle(int cpu) 9613 { 9614 struct rq *rq = cpu_rq(cpu); 9615 9616 SCHED_WARN_ON(cpu != smp_processor_id()); 9617 9618 /* If this CPU is going down, then nothing needs to be done: */ 9619 if (!cpu_active(cpu)) 9620 return; 9621 9622 /* Spare idle load balancing on CPUs that don't want to be disturbed: */ 9623 if (!housekeeping_cpu(cpu, HK_FLAG_SCHED)) 9624 return; 9625 9626 /* 9627 * Can be set safely without rq->lock held 9628 * If a clear happens, it will have evaluated last additions because 9629 * rq->lock is held during the check and the clear 9630 */ 9631 rq->has_blocked_load = 1; 9632 9633 /* 9634 * The tick is still stopped but load could have been added in the 9635 * meantime. We set the nohz.has_blocked flag to trig a check of the 9636 * *_avg. The CPU is already part of nohz.idle_cpus_mask so the clear 9637 * of nohz.has_blocked can only happen after checking the new load 9638 */ 9639 if (rq->nohz_tick_stopped) 9640 goto out; 9641 9642 /* If we're a completely isolated CPU, we don't play: */ 9643 if (on_null_domain(rq)) 9644 return; 9645 9646 rq->nohz_tick_stopped = 1; 9647 9648 cpumask_set_cpu(cpu, nohz.idle_cpus_mask); 9649 atomic_inc(&nohz.nr_cpus); 9650 9651 /* 9652 * Ensures that if nohz_idle_balance() fails to observe our 9653 * @idle_cpus_mask store, it must observe the @has_blocked 9654 * store. 9655 */ 9656 smp_mb__after_atomic(); 9657 9658 set_cpu_sd_state_idle(cpu); 9659 9660 out: 9661 /* 9662 * Each time a cpu enter idle, we assume that it has blocked load and 9663 * enable the periodic update of the load of idle cpus 9664 */ 9665 WRITE_ONCE(nohz.has_blocked, 1); 9666 } 9667 9668 /* 9669 * Internal function that runs load balance for all idle cpus. The load balance 9670 * can be a simple update of blocked load or a complete load balance with 9671 * tasks movement depending of flags. 9672 * The function returns false if the loop has stopped before running 9673 * through all idle CPUs. 9674 */ 9675 static bool _nohz_idle_balance(struct rq *this_rq, unsigned int flags, 9676 enum cpu_idle_type idle) 9677 { 9678 /* Earliest time when we have to do rebalance again */ 9679 unsigned long now = jiffies; 9680 unsigned long next_balance = now + 60*HZ; 9681 bool has_blocked_load = false; 9682 int update_next_balance = 0; 9683 int this_cpu = this_rq->cpu; 9684 int balance_cpu; 9685 int ret = false; 9686 struct rq *rq; 9687 9688 SCHED_WARN_ON((flags & NOHZ_KICK_MASK) == NOHZ_BALANCE_KICK); 9689 9690 /* 9691 * We assume there will be no idle load after this update and clear 9692 * the has_blocked flag. If a cpu enters idle in the mean time, it will 9693 * set the has_blocked flag and trig another update of idle load. 9694 * Because a cpu that becomes idle, is added to idle_cpus_mask before 9695 * setting the flag, we are sure to not clear the state and not 9696 * check the load of an idle cpu. 9697 */ 9698 WRITE_ONCE(nohz.has_blocked, 0); 9699 9700 /* 9701 * Ensures that if we miss the CPU, we must see the has_blocked 9702 * store from nohz_balance_enter_idle(). 9703 */ 9704 smp_mb(); 9705 9706 for_each_cpu(balance_cpu, nohz.idle_cpus_mask) { 9707 if (balance_cpu == this_cpu || !idle_cpu(balance_cpu)) 9708 continue; 9709 9710 /* 9711 * If this CPU gets work to do, stop the load balancing 9712 * work being done for other CPUs. Next load 9713 * balancing owner will pick it up. 9714 */ 9715 if (need_resched()) { 9716 has_blocked_load = true; 9717 goto abort; 9718 } 9719 9720 rq = cpu_rq(balance_cpu); 9721 9722 has_blocked_load |= update_nohz_stats(rq, true); 9723 9724 /* 9725 * If time for next balance is due, 9726 * do the balance. 9727 */ 9728 if (time_after_eq(jiffies, rq->next_balance)) { 9729 struct rq_flags rf; 9730 9731 rq_lock_irqsave(rq, &rf); 9732 update_rq_clock(rq); 9733 cpu_load_update_idle(rq); 9734 rq_unlock_irqrestore(rq, &rf); 9735 9736 if (flags & NOHZ_BALANCE_KICK) 9737 rebalance_domains(rq, CPU_IDLE); 9738 } 9739 9740 if (time_after(next_balance, rq->next_balance)) { 9741 next_balance = rq->next_balance; 9742 update_next_balance = 1; 9743 } 9744 } 9745 9746 /* Newly idle CPU doesn't need an update */ 9747 if (idle != CPU_NEWLY_IDLE) { 9748 update_blocked_averages(this_cpu); 9749 has_blocked_load |= this_rq->has_blocked_load; 9750 } 9751 9752 if (flags & NOHZ_BALANCE_KICK) 9753 rebalance_domains(this_rq, CPU_IDLE); 9754 9755 WRITE_ONCE(nohz.next_blocked, 9756 now + msecs_to_jiffies(LOAD_AVG_PERIOD)); 9757 9758 /* The full idle balance loop has been done */ 9759 ret = true; 9760 9761 abort: 9762 /* There is still blocked load, enable periodic update */ 9763 if (has_blocked_load) 9764 WRITE_ONCE(nohz.has_blocked, 1); 9765 9766 /* 9767 * next_balance will be updated only when there is a need. 9768 * When the CPU is attached to null domain for ex, it will not be 9769 * updated. 9770 */ 9771 if (likely(update_next_balance)) 9772 nohz.next_balance = next_balance; 9773 9774 return ret; 9775 } 9776 9777 /* 9778 * In CONFIG_NO_HZ_COMMON case, the idle balance kickee will do the 9779 * rebalancing for all the cpus for whom scheduler ticks are stopped. 9780 */ 9781 static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) 9782 { 9783 int this_cpu = this_rq->cpu; 9784 unsigned int flags; 9785 9786 if (!(atomic_read(nohz_flags(this_cpu)) & NOHZ_KICK_MASK)) 9787 return false; 9788 9789 if (idle != CPU_IDLE) { 9790 atomic_andnot(NOHZ_KICK_MASK, nohz_flags(this_cpu)); 9791 return false; 9792 } 9793 9794 /* could be _relaxed() */ 9795 flags = atomic_fetch_andnot(NOHZ_KICK_MASK, nohz_flags(this_cpu)); 9796 if (!(flags & NOHZ_KICK_MASK)) 9797 return false; 9798 9799 _nohz_idle_balance(this_rq, flags, idle); 9800 9801 return true; 9802 } 9803 9804 static void nohz_newidle_balance(struct rq *this_rq) 9805 { 9806 int this_cpu = this_rq->cpu; 9807 9808 /* 9809 * This CPU doesn't want to be disturbed by scheduler 9810 * housekeeping 9811 */ 9812 if (!housekeeping_cpu(this_cpu, HK_FLAG_SCHED)) 9813 return; 9814 9815 /* Will wake up very soon. No time for doing anything else*/ 9816 if (this_rq->avg_idle < sysctl_sched_migration_cost) 9817 return; 9818 9819 /* Don't need to update blocked load of idle CPUs*/ 9820 if (!READ_ONCE(nohz.has_blocked) || 9821 time_before(jiffies, READ_ONCE(nohz.next_blocked))) 9822 return; 9823 9824 raw_spin_unlock(&this_rq->lock); 9825 /* 9826 * This CPU is going to be idle and blocked load of idle CPUs 9827 * need to be updated. Run the ilb locally as it is a good 9828 * candidate for ilb instead of waking up another idle CPU. 9829 * Kick an normal ilb if we failed to do the update. 9830 */ 9831 if (!_nohz_idle_balance(this_rq, NOHZ_STATS_KICK, CPU_NEWLY_IDLE)) 9832 kick_ilb(NOHZ_STATS_KICK); 9833 raw_spin_lock(&this_rq->lock); 9834 } 9835 9836 #else /* !CONFIG_NO_HZ_COMMON */ 9837 static inline void nohz_balancer_kick(struct rq *rq) { } 9838 9839 static inline bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) 9840 { 9841 return false; 9842 } 9843 9844 static inline void nohz_newidle_balance(struct rq *this_rq) { } 9845 #endif /* CONFIG_NO_HZ_COMMON */ 9846 9847 /* 9848 * idle_balance is called by schedule() if this_cpu is about to become 9849 * idle. Attempts to pull tasks from other CPUs. 9850 */ 9851 static int idle_balance(struct rq *this_rq, struct rq_flags *rf) 9852 { 9853 unsigned long next_balance = jiffies + HZ; 9854 int this_cpu = this_rq->cpu; 9855 struct sched_domain *sd; 9856 int pulled_task = 0; 9857 u64 curr_cost = 0; 9858 9859 /* 9860 * We must set idle_stamp _before_ calling idle_balance(), such that we 9861 * measure the duration of idle_balance() as idle time. 9862 */ 9863 this_rq->idle_stamp = rq_clock(this_rq); 9864 9865 /* 9866 * Do not pull tasks towards !active CPUs... 9867 */ 9868 if (!cpu_active(this_cpu)) 9869 return 0; 9870 9871 /* 9872 * This is OK, because current is on_cpu, which avoids it being picked 9873 * for load-balance and preemption/IRQs are still disabled avoiding 9874 * further scheduler activity on it and we're being very careful to 9875 * re-start the picking loop. 9876 */ 9877 rq_unpin_lock(this_rq, rf); 9878 9879 if (this_rq->avg_idle < sysctl_sched_migration_cost || 9880 !READ_ONCE(this_rq->rd->overload)) { 9881 9882 rcu_read_lock(); 9883 sd = rcu_dereference_check_sched_domain(this_rq->sd); 9884 if (sd) 9885 update_next_balance(sd, &next_balance); 9886 rcu_read_unlock(); 9887 9888 nohz_newidle_balance(this_rq); 9889 9890 goto out; 9891 } 9892 9893 raw_spin_unlock(&this_rq->lock); 9894 9895 update_blocked_averages(this_cpu); 9896 rcu_read_lock(); 9897 for_each_domain(this_cpu, sd) { 9898 int continue_balancing = 1; 9899 u64 t0, domain_cost; 9900 9901 if (!(sd->flags & SD_LOAD_BALANCE)) 9902 continue; 9903 9904 if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost) { 9905 update_next_balance(sd, &next_balance); 9906 break; 9907 } 9908 9909 if (sd->flags & SD_BALANCE_NEWIDLE) { 9910 t0 = sched_clock_cpu(this_cpu); 9911 9912 pulled_task = load_balance(this_cpu, this_rq, 9913 sd, CPU_NEWLY_IDLE, 9914 &continue_balancing); 9915 9916 domain_cost = sched_clock_cpu(this_cpu) - t0; 9917 if (domain_cost > sd->max_newidle_lb_cost) 9918 sd->max_newidle_lb_cost = domain_cost; 9919 9920 curr_cost += domain_cost; 9921 } 9922 9923 update_next_balance(sd, &next_balance); 9924 9925 /* 9926 * Stop searching for tasks to pull if there are 9927 * now runnable tasks on this rq. 9928 */ 9929 if (pulled_task || this_rq->nr_running > 0) 9930 break; 9931 } 9932 rcu_read_unlock(); 9933 9934 raw_spin_lock(&this_rq->lock); 9935 9936 if (curr_cost > this_rq->max_idle_balance_cost) 9937 this_rq->max_idle_balance_cost = curr_cost; 9938 9939 out: 9940 /* 9941 * While browsing the domains, we released the rq lock, a task could 9942 * have been enqueued in the meantime. Since we're not going idle, 9943 * pretend we pulled a task. 9944 */ 9945 if (this_rq->cfs.h_nr_running && !pulled_task) 9946 pulled_task = 1; 9947 9948 /* Move the next balance forward */ 9949 if (time_after(this_rq->next_balance, next_balance)) 9950 this_rq->next_balance = next_balance; 9951 9952 /* Is there a task of a high priority class? */ 9953 if (this_rq->nr_running != this_rq->cfs.h_nr_running) 9954 pulled_task = -1; 9955 9956 if (pulled_task) 9957 this_rq->idle_stamp = 0; 9958 9959 rq_repin_lock(this_rq, rf); 9960 9961 return pulled_task; 9962 } 9963 9964 /* 9965 * run_rebalance_domains is triggered when needed from the scheduler tick. 9966 * Also triggered for nohz idle balancing (with nohz_balancing_kick set). 9967 */ 9968 static __latent_entropy void run_rebalance_domains(struct softirq_action *h) 9969 { 9970 struct rq *this_rq = this_rq(); 9971 enum cpu_idle_type idle = this_rq->idle_balance ? 9972 CPU_IDLE : CPU_NOT_IDLE; 9973 9974 /* 9975 * If this CPU has a pending nohz_balance_kick, then do the 9976 * balancing on behalf of the other idle CPUs whose ticks are 9977 * stopped. Do nohz_idle_balance *before* rebalance_domains to 9978 * give the idle CPUs a chance to load balance. Else we may 9979 * load balance only within the local sched_domain hierarchy 9980 * and abort nohz_idle_balance altogether if we pull some load. 9981 */ 9982 if (nohz_idle_balance(this_rq, idle)) 9983 return; 9984 9985 /* normal load balance */ 9986 update_blocked_averages(this_rq->cpu); 9987 rebalance_domains(this_rq, idle); 9988 } 9989 9990 /* 9991 * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing. 9992 */ 9993 void trigger_load_balance(struct rq *rq) 9994 { 9995 /* Don't need to rebalance while attached to NULL domain */ 9996 if (unlikely(on_null_domain(rq))) 9997 return; 9998 9999 if (time_after_eq(jiffies, rq->next_balance)) 10000 raise_softirq(SCHED_SOFTIRQ); 10001 10002 nohz_balancer_kick(rq); 10003 } 10004 10005 static void rq_online_fair(struct rq *rq) 10006 { 10007 update_sysctl(); 10008 10009 update_runtime_enabled(rq); 10010 } 10011 10012 static void rq_offline_fair(struct rq *rq) 10013 { 10014 update_sysctl(); 10015 10016 /* Ensure any throttled groups are reachable by pick_next_task */ 10017 unthrottle_offline_cfs_rqs(rq); 10018 } 10019 10020 #endif /* CONFIG_SMP */ 10021 10022 /* 10023 * scheduler tick hitting a task of our scheduling class. 10024 * 10025 * NOTE: This function can be called remotely by the tick offload that 10026 * goes along full dynticks. Therefore no local assumption can be made 10027 * and everything must be accessed through the @rq and @curr passed in 10028 * parameters. 10029 */ 10030 static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) 10031 { 10032 struct cfs_rq *cfs_rq; 10033 struct sched_entity *se = &curr->se; 10034 10035 for_each_sched_entity(se) { 10036 cfs_rq = cfs_rq_of(se); 10037 entity_tick(cfs_rq, se, queued); 10038 } 10039 10040 if (static_branch_unlikely(&sched_numa_balancing)) 10041 task_tick_numa(rq, curr); 10042 10043 update_misfit_status(curr, rq); 10044 update_overutilized_status(task_rq(curr)); 10045 } 10046 10047 /* 10048 * called on fork with the child task as argument from the parent's context 10049 * - child not yet on the tasklist 10050 * - preemption disabled 10051 */ 10052 static void task_fork_fair(struct task_struct *p) 10053 { 10054 struct cfs_rq *cfs_rq; 10055 struct sched_entity *se = &p->se, *curr; 10056 struct rq *rq = this_rq(); 10057 struct rq_flags rf; 10058 10059 rq_lock(rq, &rf); 10060 update_rq_clock(rq); 10061 10062 cfs_rq = task_cfs_rq(current); 10063 curr = cfs_rq->curr; 10064 if (curr) { 10065 update_curr(cfs_rq); 10066 se->vruntime = curr->vruntime; 10067 } 10068 place_entity(cfs_rq, se, 1); 10069 10070 if (sysctl_sched_child_runs_first && curr && entity_before(curr, se)) { 10071 /* 10072 * Upon rescheduling, sched_class::put_prev_task() will place 10073 * 'current' within the tree based on its new key value. 10074 */ 10075 swap(curr->vruntime, se->vruntime); 10076 resched_curr(rq); 10077 } 10078 10079 se->vruntime -= cfs_rq->min_vruntime; 10080 rq_unlock(rq, &rf); 10081 } 10082 10083 /* 10084 * Priority of the task has changed. Check to see if we preempt 10085 * the current task. 10086 */ 10087 static void 10088 prio_changed_fair(struct rq *rq, struct task_struct *p, int oldprio) 10089 { 10090 if (!task_on_rq_queued(p)) 10091 return; 10092 10093 /* 10094 * Reschedule if we are currently running on this runqueue and 10095 * our priority decreased, or if we are not currently running on 10096 * this runqueue and our priority is higher than the current's 10097 */ 10098 if (rq->curr == p) { 10099 if (p->prio > oldprio) 10100 resched_curr(rq); 10101 } else 10102 check_preempt_curr(rq, p, 0); 10103 } 10104 10105 static inline bool vruntime_normalized(struct task_struct *p) 10106 { 10107 struct sched_entity *se = &p->se; 10108 10109 /* 10110 * In both the TASK_ON_RQ_QUEUED and TASK_ON_RQ_MIGRATING cases, 10111 * the dequeue_entity(.flags=0) will already have normalized the 10112 * vruntime. 10113 */ 10114 if (p->on_rq) 10115 return true; 10116 10117 /* 10118 * When !on_rq, vruntime of the task has usually NOT been normalized. 10119 * But there are some cases where it has already been normalized: 10120 * 10121 * - A forked child which is waiting for being woken up by 10122 * wake_up_new_task(). 10123 * - A task which has been woken up by try_to_wake_up() and 10124 * waiting for actually being woken up by sched_ttwu_pending(). 10125 */ 10126 if (!se->sum_exec_runtime || 10127 (p->state == TASK_WAKING && p->sched_remote_wakeup)) 10128 return true; 10129 10130 return false; 10131 } 10132 10133 #ifdef CONFIG_FAIR_GROUP_SCHED 10134 /* 10135 * Propagate the changes of the sched_entity across the tg tree to make it 10136 * visible to the root 10137 */ 10138 static void propagate_entity_cfs_rq(struct sched_entity *se) 10139 { 10140 struct cfs_rq *cfs_rq; 10141 10142 /* Start to propagate at parent */ 10143 se = se->parent; 10144 10145 for_each_sched_entity(se) { 10146 cfs_rq = cfs_rq_of(se); 10147 10148 if (cfs_rq_throttled(cfs_rq)) 10149 break; 10150 10151 update_load_avg(cfs_rq, se, UPDATE_TG); 10152 } 10153 } 10154 #else 10155 static void propagate_entity_cfs_rq(struct sched_entity *se) { } 10156 #endif 10157 10158 static void detach_entity_cfs_rq(struct sched_entity *se) 10159 { 10160 struct cfs_rq *cfs_rq = cfs_rq_of(se); 10161 10162 /* Catch up with the cfs_rq and remove our load when we leave */ 10163 update_load_avg(cfs_rq, se, 0); 10164 detach_entity_load_avg(cfs_rq, se); 10165 update_tg_load_avg(cfs_rq, false); 10166 propagate_entity_cfs_rq(se); 10167 } 10168 10169 static void attach_entity_cfs_rq(struct sched_entity *se) 10170 { 10171 struct cfs_rq *cfs_rq = cfs_rq_of(se); 10172 10173 #ifdef CONFIG_FAIR_GROUP_SCHED 10174 /* 10175 * Since the real-depth could have been changed (only FAIR 10176 * class maintain depth value), reset depth properly. 10177 */ 10178 se->depth = se->parent ? se->parent->depth + 1 : 0; 10179 #endif 10180 10181 /* Synchronize entity with its cfs_rq */ 10182 update_load_avg(cfs_rq, se, sched_feat(ATTACH_AGE_LOAD) ? 0 : SKIP_AGE_LOAD); 10183 attach_entity_load_avg(cfs_rq, se, 0); 10184 update_tg_load_avg(cfs_rq, false); 10185 propagate_entity_cfs_rq(se); 10186 } 10187 10188 static void detach_task_cfs_rq(struct task_struct *p) 10189 { 10190 struct sched_entity *se = &p->se; 10191 struct cfs_rq *cfs_rq = cfs_rq_of(se); 10192 10193 if (!vruntime_normalized(p)) { 10194 /* 10195 * Fix up our vruntime so that the current sleep doesn't 10196 * cause 'unlimited' sleep bonus. 10197 */ 10198 place_entity(cfs_rq, se, 0); 10199 se->vruntime -= cfs_rq->min_vruntime; 10200 } 10201 10202 detach_entity_cfs_rq(se); 10203 } 10204 10205 static void attach_task_cfs_rq(struct task_struct *p) 10206 { 10207 struct sched_entity *se = &p->se; 10208 struct cfs_rq *cfs_rq = cfs_rq_of(se); 10209 10210 attach_entity_cfs_rq(se); 10211 10212 if (!vruntime_normalized(p)) 10213 se->vruntime += cfs_rq->min_vruntime; 10214 } 10215 10216 static void switched_from_fair(struct rq *rq, struct task_struct *p) 10217 { 10218 detach_task_cfs_rq(p); 10219 } 10220 10221 static void switched_to_fair(struct rq *rq, struct task_struct *p) 10222 { 10223 attach_task_cfs_rq(p); 10224 10225 if (task_on_rq_queued(p)) { 10226 /* 10227 * We were most likely switched from sched_rt, so 10228 * kick off the schedule if running, otherwise just see 10229 * if we can still preempt the current task. 10230 */ 10231 if (rq->curr == p) 10232 resched_curr(rq); 10233 else 10234 check_preempt_curr(rq, p, 0); 10235 } 10236 } 10237 10238 /* Account for a task changing its policy or group. 10239 * 10240 * This routine is mostly called to set cfs_rq->curr field when a task 10241 * migrates between groups/classes. 10242 */ 10243 static void set_curr_task_fair(struct rq *rq) 10244 { 10245 struct sched_entity *se = &rq->curr->se; 10246 10247 for_each_sched_entity(se) { 10248 struct cfs_rq *cfs_rq = cfs_rq_of(se); 10249 10250 set_next_entity(cfs_rq, se); 10251 /* ensure bandwidth has been allocated on our new cfs_rq */ 10252 account_cfs_rq_runtime(cfs_rq, 0); 10253 } 10254 } 10255 10256 void init_cfs_rq(struct cfs_rq *cfs_rq) 10257 { 10258 cfs_rq->tasks_timeline = RB_ROOT_CACHED; 10259 cfs_rq->min_vruntime = (u64)(-(1LL << 20)); 10260 #ifndef CONFIG_64BIT 10261 cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime; 10262 #endif 10263 #ifdef CONFIG_SMP 10264 raw_spin_lock_init(&cfs_rq->removed.lock); 10265 #endif 10266 } 10267 10268 #ifdef CONFIG_FAIR_GROUP_SCHED 10269 static void task_set_group_fair(struct task_struct *p) 10270 { 10271 struct sched_entity *se = &p->se; 10272 10273 set_task_rq(p, task_cpu(p)); 10274 se->depth = se->parent ? se->parent->depth + 1 : 0; 10275 } 10276 10277 static void task_move_group_fair(struct task_struct *p) 10278 { 10279 detach_task_cfs_rq(p); 10280 set_task_rq(p, task_cpu(p)); 10281 10282 #ifdef CONFIG_SMP 10283 /* Tell se's cfs_rq has been changed -- migrated */ 10284 p->se.avg.last_update_time = 0; 10285 #endif 10286 attach_task_cfs_rq(p); 10287 } 10288 10289 static void task_change_group_fair(struct task_struct *p, int type) 10290 { 10291 switch (type) { 10292 case TASK_SET_GROUP: 10293 task_set_group_fair(p); 10294 break; 10295 10296 case TASK_MOVE_GROUP: 10297 task_move_group_fair(p); 10298 break; 10299 } 10300 } 10301 10302 void free_fair_sched_group(struct task_group *tg) 10303 { 10304 int i; 10305 10306 destroy_cfs_bandwidth(tg_cfs_bandwidth(tg)); 10307 10308 for_each_possible_cpu(i) { 10309 if (tg->cfs_rq) 10310 kfree(tg->cfs_rq[i]); 10311 if (tg->se) 10312 kfree(tg->se[i]); 10313 } 10314 10315 kfree(tg->cfs_rq); 10316 kfree(tg->se); 10317 } 10318 10319 int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) 10320 { 10321 struct sched_entity *se; 10322 struct cfs_rq *cfs_rq; 10323 int i; 10324 10325 tg->cfs_rq = kcalloc(nr_cpu_ids, sizeof(cfs_rq), GFP_KERNEL); 10326 if (!tg->cfs_rq) 10327 goto err; 10328 tg->se = kcalloc(nr_cpu_ids, sizeof(se), GFP_KERNEL); 10329 if (!tg->se) 10330 goto err; 10331 10332 tg->shares = NICE_0_LOAD; 10333 10334 init_cfs_bandwidth(tg_cfs_bandwidth(tg)); 10335 10336 for_each_possible_cpu(i) { 10337 cfs_rq = kzalloc_node(sizeof(struct cfs_rq), 10338 GFP_KERNEL, cpu_to_node(i)); 10339 if (!cfs_rq) 10340 goto err; 10341 10342 se = kzalloc_node(sizeof(struct sched_entity), 10343 GFP_KERNEL, cpu_to_node(i)); 10344 if (!se) 10345 goto err_free_rq; 10346 10347 init_cfs_rq(cfs_rq); 10348 init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]); 10349 init_entity_runnable_average(se); 10350 } 10351 10352 return 1; 10353 10354 err_free_rq: 10355 kfree(cfs_rq); 10356 err: 10357 return 0; 10358 } 10359 10360 void online_fair_sched_group(struct task_group *tg) 10361 { 10362 struct sched_entity *se; 10363 struct rq *rq; 10364 int i; 10365 10366 for_each_possible_cpu(i) { 10367 rq = cpu_rq(i); 10368 se = tg->se[i]; 10369 10370 raw_spin_lock_irq(&rq->lock); 10371 update_rq_clock(rq); 10372 attach_entity_cfs_rq(se); 10373 sync_throttle(tg, i); 10374 raw_spin_unlock_irq(&rq->lock); 10375 } 10376 } 10377 10378 void unregister_fair_sched_group(struct task_group *tg) 10379 { 10380 unsigned long flags; 10381 struct rq *rq; 10382 int cpu; 10383 10384 for_each_possible_cpu(cpu) { 10385 if (tg->se[cpu]) 10386 remove_entity_load_avg(tg->se[cpu]); 10387 10388 /* 10389 * Only empty task groups can be destroyed; so we can speculatively 10390 * check on_list without danger of it being re-added. 10391 */ 10392 if (!tg->cfs_rq[cpu]->on_list) 10393 continue; 10394 10395 rq = cpu_rq(cpu); 10396 10397 raw_spin_lock_irqsave(&rq->lock, flags); 10398 list_del_leaf_cfs_rq(tg->cfs_rq[cpu]); 10399 raw_spin_unlock_irqrestore(&rq->lock, flags); 10400 } 10401 } 10402 10403 void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq, 10404 struct sched_entity *se, int cpu, 10405 struct sched_entity *parent) 10406 { 10407 struct rq *rq = cpu_rq(cpu); 10408 10409 cfs_rq->tg = tg; 10410 cfs_rq->rq = rq; 10411 init_cfs_rq_runtime(cfs_rq); 10412 10413 tg->cfs_rq[cpu] = cfs_rq; 10414 tg->se[cpu] = se; 10415 10416 /* se could be NULL for root_task_group */ 10417 if (!se) 10418 return; 10419 10420 if (!parent) { 10421 se->cfs_rq = &rq->cfs; 10422 se->depth = 0; 10423 } else { 10424 se->cfs_rq = parent->my_q; 10425 se->depth = parent->depth + 1; 10426 } 10427 10428 se->my_q = cfs_rq; 10429 /* guarantee group entities always have weight */ 10430 update_load_set(&se->load, NICE_0_LOAD); 10431 se->parent = parent; 10432 } 10433 10434 static DEFINE_MUTEX(shares_mutex); 10435 10436 int sched_group_set_shares(struct task_group *tg, unsigned long shares) 10437 { 10438 int i; 10439 10440 /* 10441 * We can't change the weight of the root cgroup. 10442 */ 10443 if (!tg->se[0]) 10444 return -EINVAL; 10445 10446 shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES)); 10447 10448 mutex_lock(&shares_mutex); 10449 if (tg->shares == shares) 10450 goto done; 10451 10452 tg->shares = shares; 10453 for_each_possible_cpu(i) { 10454 struct rq *rq = cpu_rq(i); 10455 struct sched_entity *se = tg->se[i]; 10456 struct rq_flags rf; 10457 10458 /* Propagate contribution to hierarchy */ 10459 rq_lock_irqsave(rq, &rf); 10460 update_rq_clock(rq); 10461 for_each_sched_entity(se) { 10462 update_load_avg(cfs_rq_of(se), se, UPDATE_TG); 10463 update_cfs_group(se); 10464 } 10465 rq_unlock_irqrestore(rq, &rf); 10466 } 10467 10468 done: 10469 mutex_unlock(&shares_mutex); 10470 return 0; 10471 } 10472 #else /* CONFIG_FAIR_GROUP_SCHED */ 10473 10474 void free_fair_sched_group(struct task_group *tg) { } 10475 10476 int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) 10477 { 10478 return 1; 10479 } 10480 10481 void online_fair_sched_group(struct task_group *tg) { } 10482 10483 void unregister_fair_sched_group(struct task_group *tg) { } 10484 10485 #endif /* CONFIG_FAIR_GROUP_SCHED */ 10486 10487 10488 static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task) 10489 { 10490 struct sched_entity *se = &task->se; 10491 unsigned int rr_interval = 0; 10492 10493 /* 10494 * Time slice is 0 for SCHED_OTHER tasks that are on an otherwise 10495 * idle runqueue: 10496 */ 10497 if (rq->cfs.load.weight) 10498 rr_interval = NS_TO_JIFFIES(sched_slice(cfs_rq_of(se), se)); 10499 10500 return rr_interval; 10501 } 10502 10503 /* 10504 * All the scheduling class methods: 10505 */ 10506 const struct sched_class fair_sched_class = { 10507 .next = &idle_sched_class, 10508 .enqueue_task = enqueue_task_fair, 10509 .dequeue_task = dequeue_task_fair, 10510 .yield_task = yield_task_fair, 10511 .yield_to_task = yield_to_task_fair, 10512 10513 .check_preempt_curr = check_preempt_wakeup, 10514 10515 .pick_next_task = pick_next_task_fair, 10516 .put_prev_task = put_prev_task_fair, 10517 10518 #ifdef CONFIG_SMP 10519 .select_task_rq = select_task_rq_fair, 10520 .migrate_task_rq = migrate_task_rq_fair, 10521 10522 .rq_online = rq_online_fair, 10523 .rq_offline = rq_offline_fair, 10524 10525 .task_dead = task_dead_fair, 10526 .set_cpus_allowed = set_cpus_allowed_common, 10527 #endif 10528 10529 .set_curr_task = set_curr_task_fair, 10530 .task_tick = task_tick_fair, 10531 .task_fork = task_fork_fair, 10532 10533 .prio_changed = prio_changed_fair, 10534 .switched_from = switched_from_fair, 10535 .switched_to = switched_to_fair, 10536 10537 .get_rr_interval = get_rr_interval_fair, 10538 10539 .update_curr = update_curr_fair, 10540 10541 #ifdef CONFIG_FAIR_GROUP_SCHED 10542 .task_change_group = task_change_group_fair, 10543 #endif 10544 }; 10545 10546 #ifdef CONFIG_SCHED_DEBUG 10547 void print_cfs_stats(struct seq_file *m, int cpu) 10548 { 10549 struct cfs_rq *cfs_rq; 10550 10551 rcu_read_lock(); 10552 for_each_leaf_cfs_rq(cpu_rq(cpu), cfs_rq) 10553 print_cfs_rq(m, cpu, cfs_rq); 10554 rcu_read_unlock(); 10555 } 10556 10557 #ifdef CONFIG_NUMA_BALANCING 10558 void show_numa_stats(struct task_struct *p, struct seq_file *m) 10559 { 10560 int node; 10561 unsigned long tsf = 0, tpf = 0, gsf = 0, gpf = 0; 10562 10563 for_each_online_node(node) { 10564 if (p->numa_faults) { 10565 tsf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 0)]; 10566 tpf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 1)]; 10567 } 10568 if (p->numa_group) { 10569 gsf = p->numa_group->faults[task_faults_idx(NUMA_MEM, node, 0)], 10570 gpf = p->numa_group->faults[task_faults_idx(NUMA_MEM, node, 1)]; 10571 } 10572 print_numa_stats(m, node, tsf, tpf, gsf, gpf); 10573 } 10574 } 10575 #endif /* CONFIG_NUMA_BALANCING */ 10576 #endif /* CONFIG_SCHED_DEBUG */ 10577 10578 __init void init_sched_fair_class(void) 10579 { 10580 #ifdef CONFIG_SMP 10581 open_softirq(SCHED_SOFTIRQ, run_rebalance_domains); 10582 10583 #ifdef CONFIG_NO_HZ_COMMON 10584 nohz.next_balance = jiffies; 10585 nohz.next_blocked = jiffies; 10586 zalloc_cpumask_var(&nohz.idle_cpus_mask, GFP_NOWAIT); 10587 #endif 10588 #endif /* SMP */ 10589 10590 } 10591