1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Scheduler internal types and methods: 4 */ 5 #ifndef _KERNEL_SCHED_SCHED_H 6 #define _KERNEL_SCHED_SCHED_H 7 8 #include <linux/sched/affinity.h> 9 #include <linux/sched/autogroup.h> 10 #include <linux/sched/cpufreq.h> 11 #include <linux/sched/deadline.h> 12 #include <linux/sched.h> 13 #include <linux/sched/loadavg.h> 14 #include <linux/sched/mm.h> 15 #include <linux/sched/rseq_api.h> 16 #include <linux/sched/signal.h> 17 #include <linux/sched/smt.h> 18 #include <linux/sched/stat.h> 19 #include <linux/sched/sysctl.h> 20 #include <linux/sched/task_flags.h> 21 #include <linux/sched/task.h> 22 #include <linux/sched/topology.h> 23 24 #include <linux/atomic.h> 25 #include <linux/bitmap.h> 26 #include <linux/bug.h> 27 #include <linux/capability.h> 28 #include <linux/cgroup_api.h> 29 #include <linux/cgroup.h> 30 #include <linux/context_tracking.h> 31 #include <linux/cpufreq.h> 32 #include <linux/cpumask_api.h> 33 #include <linux/ctype.h> 34 #include <linux/file.h> 35 #include <linux/fs_api.h> 36 #include <linux/hrtimer_api.h> 37 #include <linux/interrupt.h> 38 #include <linux/irq_work.h> 39 #include <linux/jiffies.h> 40 #include <linux/kref_api.h> 41 #include <linux/kthread.h> 42 #include <linux/ktime_api.h> 43 #include <linux/lockdep_api.h> 44 #include <linux/lockdep.h> 45 #include <linux/minmax.h> 46 #include <linux/mm.h> 47 #include <linux/module.h> 48 #include <linux/mutex_api.h> 49 #include <linux/plist.h> 50 #include <linux/poll.h> 51 #include <linux/proc_fs.h> 52 #include <linux/profile.h> 53 #include <linux/psi.h> 54 #include <linux/rcupdate.h> 55 #include <linux/seq_file.h> 56 #include <linux/seqlock.h> 57 #include <linux/softirq.h> 58 #include <linux/spinlock_api.h> 59 #include <linux/static_key.h> 60 #include <linux/stop_machine.h> 61 #include <linux/syscalls_api.h> 62 #include <linux/syscalls.h> 63 #include <linux/tick.h> 64 #include <linux/topology.h> 65 #include <linux/types.h> 66 #include <linux/u64_stats_sync_api.h> 67 #include <linux/uaccess.h> 68 #include <linux/wait_api.h> 69 #include <linux/wait_bit.h> 70 #include <linux/workqueue_api.h> 71 72 #include <trace/events/power.h> 73 #include <trace/events/sched.h> 74 75 #include "../workqueue_internal.h" 76 77 #ifdef CONFIG_CGROUP_SCHED 78 #include <linux/cgroup.h> 79 #include <linux/psi.h> 80 #endif 81 82 #ifdef CONFIG_SCHED_DEBUG 83 # include <linux/static_key.h> 84 #endif 85 86 #ifdef CONFIG_PARAVIRT 87 # include <asm/paravirt.h> 88 # include <asm/paravirt_api_clock.h> 89 #endif 90 91 #include "cpupri.h" 92 #include "cpudeadline.h" 93 94 #ifdef CONFIG_SCHED_DEBUG 95 # define SCHED_WARN_ON(x) WARN_ONCE(x, #x) 96 #else 97 # define SCHED_WARN_ON(x) ({ (void)(x), 0; }) 98 #endif 99 100 struct rq; 101 struct cpuidle_state; 102 103 /* task_struct::on_rq states: */ 104 #define TASK_ON_RQ_QUEUED 1 105 #define TASK_ON_RQ_MIGRATING 2 106 107 extern __read_mostly int scheduler_running; 108 109 extern unsigned long calc_load_update; 110 extern atomic_long_t calc_load_tasks; 111 112 extern unsigned int sysctl_sched_child_runs_first; 113 114 extern void calc_global_load_tick(struct rq *this_rq); 115 extern long calc_load_fold_active(struct rq *this_rq, long adjust); 116 117 extern void call_trace_sched_update_nr_running(struct rq *rq, int count); 118 119 extern unsigned int sysctl_sched_rt_period; 120 extern int sysctl_sched_rt_runtime; 121 extern int sched_rr_timeslice; 122 123 /* 124 * Helpers for converting nanosecond timing to jiffy resolution 125 */ 126 #define NS_TO_JIFFIES(TIME) ((unsigned long)(TIME) / (NSEC_PER_SEC / HZ)) 127 128 /* 129 * Increase resolution of nice-level calculations for 64-bit architectures. 130 * The extra resolution improves shares distribution and load balancing of 131 * low-weight task groups (eg. nice +19 on an autogroup), deeper taskgroup 132 * hierarchies, especially on larger systems. This is not a user-visible change 133 * and does not change the user-interface for setting shares/weights. 134 * 135 * We increase resolution only if we have enough bits to allow this increased 136 * resolution (i.e. 64-bit). The costs for increasing resolution when 32-bit 137 * are pretty high and the returns do not justify the increased costs. 138 * 139 * Really only required when CONFIG_FAIR_GROUP_SCHED=y is also set, but to 140 * increase coverage and consistency always enable it on 64-bit platforms. 141 */ 142 #ifdef CONFIG_64BIT 143 # define NICE_0_LOAD_SHIFT (SCHED_FIXEDPOINT_SHIFT + SCHED_FIXEDPOINT_SHIFT) 144 # define scale_load(w) ((w) << SCHED_FIXEDPOINT_SHIFT) 145 # define scale_load_down(w) \ 146 ({ \ 147 unsigned long __w = (w); \ 148 if (__w) \ 149 __w = max(2UL, __w >> SCHED_FIXEDPOINT_SHIFT); \ 150 __w; \ 151 }) 152 #else 153 # define NICE_0_LOAD_SHIFT (SCHED_FIXEDPOINT_SHIFT) 154 # define scale_load(w) (w) 155 # define scale_load_down(w) (w) 156 #endif 157 158 /* 159 * Task weight (visible to users) and its load (invisible to users) have 160 * independent resolution, but they should be well calibrated. We use 161 * scale_load() and scale_load_down(w) to convert between them. The 162 * following must be true: 163 * 164 * scale_load(sched_prio_to_weight[NICE_TO_PRIO(0)-MAX_RT_PRIO]) == NICE_0_LOAD 165 * 166 */ 167 #define NICE_0_LOAD (1L << NICE_0_LOAD_SHIFT) 168 169 /* 170 * Single value that decides SCHED_DEADLINE internal math precision. 171 * 10 -> just above 1us 172 * 9 -> just above 0.5us 173 */ 174 #define DL_SCALE 10 175 176 /* 177 * Single value that denotes runtime == period, ie unlimited time. 178 */ 179 #define RUNTIME_INF ((u64)~0ULL) 180 181 static inline int idle_policy(int policy) 182 { 183 return policy == SCHED_IDLE; 184 } 185 static inline int fair_policy(int policy) 186 { 187 return policy == SCHED_NORMAL || policy == SCHED_BATCH; 188 } 189 190 static inline int rt_policy(int policy) 191 { 192 return policy == SCHED_FIFO || policy == SCHED_RR; 193 } 194 195 static inline int dl_policy(int policy) 196 { 197 return policy == SCHED_DEADLINE; 198 } 199 static inline bool valid_policy(int policy) 200 { 201 return idle_policy(policy) || fair_policy(policy) || 202 rt_policy(policy) || dl_policy(policy); 203 } 204 205 static inline int task_has_idle_policy(struct task_struct *p) 206 { 207 return idle_policy(p->policy); 208 } 209 210 static inline int task_has_rt_policy(struct task_struct *p) 211 { 212 return rt_policy(p->policy); 213 } 214 215 static inline int task_has_dl_policy(struct task_struct *p) 216 { 217 return dl_policy(p->policy); 218 } 219 220 #define cap_scale(v, s) ((v)*(s) >> SCHED_CAPACITY_SHIFT) 221 222 static inline void update_avg(u64 *avg, u64 sample) 223 { 224 s64 diff = sample - *avg; 225 *avg += diff / 8; 226 } 227 228 /* 229 * Shifting a value by an exponent greater *or equal* to the size of said value 230 * is UB; cap at size-1. 231 */ 232 #define shr_bound(val, shift) \ 233 (val >> min_t(typeof(shift), shift, BITS_PER_TYPE(typeof(val)) - 1)) 234 235 /* 236 * !! For sched_setattr_nocheck() (kernel) only !! 237 * 238 * This is actually gross. :( 239 * 240 * It is used to make schedutil kworker(s) higher priority than SCHED_DEADLINE 241 * tasks, but still be able to sleep. We need this on platforms that cannot 242 * atomically change clock frequency. Remove once fast switching will be 243 * available on such platforms. 244 * 245 * SUGOV stands for SchedUtil GOVernor. 246 */ 247 #define SCHED_FLAG_SUGOV 0x10000000 248 249 #define SCHED_DL_FLAGS (SCHED_FLAG_RECLAIM | SCHED_FLAG_DL_OVERRUN | SCHED_FLAG_SUGOV) 250 251 static inline bool dl_entity_is_special(const struct sched_dl_entity *dl_se) 252 { 253 #ifdef CONFIG_CPU_FREQ_GOV_SCHEDUTIL 254 return unlikely(dl_se->flags & SCHED_FLAG_SUGOV); 255 #else 256 return false; 257 #endif 258 } 259 260 /* 261 * Tells if entity @a should preempt entity @b. 262 */ 263 static inline bool dl_entity_preempt(const struct sched_dl_entity *a, 264 const struct sched_dl_entity *b) 265 { 266 return dl_entity_is_special(a) || 267 dl_time_before(a->deadline, b->deadline); 268 } 269 270 /* 271 * This is the priority-queue data structure of the RT scheduling class: 272 */ 273 struct rt_prio_array { 274 DECLARE_BITMAP(bitmap, MAX_RT_PRIO+1); /* include 1 bit for delimiter */ 275 struct list_head queue[MAX_RT_PRIO]; 276 }; 277 278 struct rt_bandwidth { 279 /* nests inside the rq lock: */ 280 raw_spinlock_t rt_runtime_lock; 281 ktime_t rt_period; 282 u64 rt_runtime; 283 struct hrtimer rt_period_timer; 284 unsigned int rt_period_active; 285 }; 286 287 void __dl_clear_params(struct task_struct *p); 288 289 static inline int dl_bandwidth_enabled(void) 290 { 291 return sysctl_sched_rt_runtime >= 0; 292 } 293 294 /* 295 * To keep the bandwidth of -deadline tasks under control 296 * we need some place where: 297 * - store the maximum -deadline bandwidth of each cpu; 298 * - cache the fraction of bandwidth that is currently allocated in 299 * each root domain; 300 * 301 * This is all done in the data structure below. It is similar to the 302 * one used for RT-throttling (rt_bandwidth), with the main difference 303 * that, since here we are only interested in admission control, we 304 * do not decrease any runtime while the group "executes", neither we 305 * need a timer to replenish it. 306 * 307 * With respect to SMP, bandwidth is given on a per root domain basis, 308 * meaning that: 309 * - bw (< 100%) is the deadline bandwidth of each CPU; 310 * - total_bw is the currently allocated bandwidth in each root domain; 311 */ 312 struct dl_bw { 313 raw_spinlock_t lock; 314 u64 bw; 315 u64 total_bw; 316 }; 317 318 extern void init_dl_bw(struct dl_bw *dl_b); 319 extern int sched_dl_global_validate(void); 320 extern void sched_dl_do_global(void); 321 extern int sched_dl_overflow(struct task_struct *p, int policy, const struct sched_attr *attr); 322 extern void __setparam_dl(struct task_struct *p, const struct sched_attr *attr); 323 extern void __getparam_dl(struct task_struct *p, struct sched_attr *attr); 324 extern bool __checkparam_dl(const struct sched_attr *attr); 325 extern bool dl_param_changed(struct task_struct *p, const struct sched_attr *attr); 326 extern int dl_cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial); 327 extern int dl_bw_check_overflow(int cpu); 328 329 #ifdef CONFIG_CGROUP_SCHED 330 331 struct cfs_rq; 332 struct rt_rq; 333 334 extern struct list_head task_groups; 335 336 struct cfs_bandwidth { 337 #ifdef CONFIG_CFS_BANDWIDTH 338 raw_spinlock_t lock; 339 ktime_t period; 340 u64 quota; 341 u64 runtime; 342 u64 burst; 343 u64 runtime_snap; 344 s64 hierarchical_quota; 345 346 u8 idle; 347 u8 period_active; 348 u8 slack_started; 349 struct hrtimer period_timer; 350 struct hrtimer slack_timer; 351 struct list_head throttled_cfs_rq; 352 353 /* Statistics: */ 354 int nr_periods; 355 int nr_throttled; 356 int nr_burst; 357 u64 throttled_time; 358 u64 burst_time; 359 #endif 360 }; 361 362 /* Task group related information */ 363 struct task_group { 364 struct cgroup_subsys_state css; 365 366 #ifdef CONFIG_FAIR_GROUP_SCHED 367 /* schedulable entities of this group on each CPU */ 368 struct sched_entity **se; 369 /* runqueue "owned" by this group on each CPU */ 370 struct cfs_rq **cfs_rq; 371 unsigned long shares; 372 373 /* A positive value indicates that this is a SCHED_IDLE group. */ 374 int idle; 375 376 #ifdef CONFIG_SMP 377 /* 378 * load_avg can be heavily contended at clock tick time, so put 379 * it in its own cacheline separated from the fields above which 380 * will also be accessed at each tick. 381 */ 382 atomic_long_t load_avg ____cacheline_aligned; 383 #endif 384 #endif 385 386 #ifdef CONFIG_RT_GROUP_SCHED 387 struct sched_rt_entity **rt_se; 388 struct rt_rq **rt_rq; 389 390 struct rt_bandwidth rt_bandwidth; 391 #endif 392 393 struct rcu_head rcu; 394 struct list_head list; 395 396 struct task_group *parent; 397 struct list_head siblings; 398 struct list_head children; 399 400 #ifdef CONFIG_SCHED_AUTOGROUP 401 struct autogroup *autogroup; 402 #endif 403 404 struct cfs_bandwidth cfs_bandwidth; 405 406 #ifdef CONFIG_UCLAMP_TASK_GROUP 407 /* The two decimal precision [%] value requested from user-space */ 408 unsigned int uclamp_pct[UCLAMP_CNT]; 409 /* Clamp values requested for a task group */ 410 struct uclamp_se uclamp_req[UCLAMP_CNT]; 411 /* Effective clamp values used for a task group */ 412 struct uclamp_se uclamp[UCLAMP_CNT]; 413 #endif 414 415 }; 416 417 #ifdef CONFIG_FAIR_GROUP_SCHED 418 #define ROOT_TASK_GROUP_LOAD NICE_0_LOAD 419 420 /* 421 * A weight of 0 or 1 can cause arithmetics problems. 422 * A weight of a cfs_rq is the sum of weights of which entities 423 * are queued on this cfs_rq, so a weight of a entity should not be 424 * too large, so as the shares value of a task group. 425 * (The default weight is 1024 - so there's no practical 426 * limitation from this.) 427 */ 428 #define MIN_SHARES (1UL << 1) 429 #define MAX_SHARES (1UL << 18) 430 #endif 431 432 typedef int (*tg_visitor)(struct task_group *, void *); 433 434 extern int walk_tg_tree_from(struct task_group *from, 435 tg_visitor down, tg_visitor up, void *data); 436 437 /* 438 * Iterate the full tree, calling @down when first entering a node and @up when 439 * leaving it for the final time. 440 * 441 * Caller must hold rcu_lock or sufficient equivalent. 442 */ 443 static inline int walk_tg_tree(tg_visitor down, tg_visitor up, void *data) 444 { 445 return walk_tg_tree_from(&root_task_group, down, up, data); 446 } 447 448 extern int tg_nop(struct task_group *tg, void *data); 449 450 extern void free_fair_sched_group(struct task_group *tg); 451 extern int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent); 452 extern void online_fair_sched_group(struct task_group *tg); 453 extern void unregister_fair_sched_group(struct task_group *tg); 454 extern void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq, 455 struct sched_entity *se, int cpu, 456 struct sched_entity *parent); 457 extern void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b); 458 459 extern void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b); 460 extern void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b); 461 extern void unthrottle_cfs_rq(struct cfs_rq *cfs_rq); 462 463 extern void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq, 464 struct sched_rt_entity *rt_se, int cpu, 465 struct sched_rt_entity *parent); 466 extern int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us); 467 extern int sched_group_set_rt_period(struct task_group *tg, u64 rt_period_us); 468 extern long sched_group_rt_runtime(struct task_group *tg); 469 extern long sched_group_rt_period(struct task_group *tg); 470 extern int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk); 471 472 extern struct task_group *sched_create_group(struct task_group *parent); 473 extern void sched_online_group(struct task_group *tg, 474 struct task_group *parent); 475 extern void sched_destroy_group(struct task_group *tg); 476 extern void sched_release_group(struct task_group *tg); 477 478 extern void sched_move_task(struct task_struct *tsk); 479 480 #ifdef CONFIG_FAIR_GROUP_SCHED 481 extern int sched_group_set_shares(struct task_group *tg, unsigned long shares); 482 483 extern int sched_group_set_idle(struct task_group *tg, long idle); 484 485 #ifdef CONFIG_SMP 486 extern void set_task_rq_fair(struct sched_entity *se, 487 struct cfs_rq *prev, struct cfs_rq *next); 488 #else /* !CONFIG_SMP */ 489 static inline void set_task_rq_fair(struct sched_entity *se, 490 struct cfs_rq *prev, struct cfs_rq *next) { } 491 #endif /* CONFIG_SMP */ 492 #endif /* CONFIG_FAIR_GROUP_SCHED */ 493 494 #else /* CONFIG_CGROUP_SCHED */ 495 496 struct cfs_bandwidth { }; 497 498 #endif /* CONFIG_CGROUP_SCHED */ 499 500 extern void unregister_rt_sched_group(struct task_group *tg); 501 extern void free_rt_sched_group(struct task_group *tg); 502 extern int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent); 503 504 /* 505 * u64_u32_load/u64_u32_store 506 * 507 * Use a copy of a u64 value to protect against data race. This is only 508 * applicable for 32-bits architectures. 509 */ 510 #ifdef CONFIG_64BIT 511 # define u64_u32_load_copy(var, copy) var 512 # define u64_u32_store_copy(var, copy, val) (var = val) 513 #else 514 # define u64_u32_load_copy(var, copy) \ 515 ({ \ 516 u64 __val, __val_copy; \ 517 do { \ 518 __val_copy = copy; \ 519 /* \ 520 * paired with u64_u32_store_copy(), ordering access \ 521 * to var and copy. \ 522 */ \ 523 smp_rmb(); \ 524 __val = var; \ 525 } while (__val != __val_copy); \ 526 __val; \ 527 }) 528 # define u64_u32_store_copy(var, copy, val) \ 529 do { \ 530 typeof(val) __val = (val); \ 531 var = __val; \ 532 /* \ 533 * paired with u64_u32_load_copy(), ordering access to var and \ 534 * copy. \ 535 */ \ 536 smp_wmb(); \ 537 copy = __val; \ 538 } while (0) 539 #endif 540 # define u64_u32_load(var) u64_u32_load_copy(var, var##_copy) 541 # define u64_u32_store(var, val) u64_u32_store_copy(var, var##_copy, val) 542 543 /* CFS-related fields in a runqueue */ 544 struct cfs_rq { 545 struct load_weight load; 546 unsigned int nr_running; 547 unsigned int h_nr_running; /* SCHED_{NORMAL,BATCH,IDLE} */ 548 unsigned int idle_nr_running; /* SCHED_IDLE */ 549 unsigned int idle_h_nr_running; /* SCHED_IDLE */ 550 551 s64 avg_vruntime; 552 u64 avg_load; 553 554 u64 exec_clock; 555 u64 min_vruntime; 556 #ifdef CONFIG_SCHED_CORE 557 unsigned int forceidle_seq; 558 u64 min_vruntime_fi; 559 #endif 560 561 #ifndef CONFIG_64BIT 562 u64 min_vruntime_copy; 563 #endif 564 565 struct rb_root_cached tasks_timeline; 566 567 /* 568 * 'curr' points to currently running entity on this cfs_rq. 569 * It is set to NULL otherwise (i.e when none are currently running). 570 */ 571 struct sched_entity *curr; 572 struct sched_entity *next; 573 574 #ifdef CONFIG_SCHED_DEBUG 575 unsigned int nr_spread_over; 576 #endif 577 578 #ifdef CONFIG_SMP 579 /* 580 * CFS load tracking 581 */ 582 struct sched_avg avg; 583 #ifndef CONFIG_64BIT 584 u64 last_update_time_copy; 585 #endif 586 struct { 587 raw_spinlock_t lock ____cacheline_aligned; 588 int nr; 589 unsigned long load_avg; 590 unsigned long util_avg; 591 unsigned long runnable_avg; 592 } removed; 593 594 #ifdef CONFIG_FAIR_GROUP_SCHED 595 unsigned long tg_load_avg_contrib; 596 long propagate; 597 long prop_runnable_sum; 598 599 /* 600 * h_load = weight * f(tg) 601 * 602 * Where f(tg) is the recursive weight fraction assigned to 603 * this group. 604 */ 605 unsigned long h_load; 606 u64 last_h_load_update; 607 struct sched_entity *h_load_next; 608 #endif /* CONFIG_FAIR_GROUP_SCHED */ 609 #endif /* CONFIG_SMP */ 610 611 #ifdef CONFIG_FAIR_GROUP_SCHED 612 struct rq *rq; /* CPU runqueue to which this cfs_rq is attached */ 613 614 /* 615 * leaf cfs_rqs are those that hold tasks (lowest schedulable entity in 616 * a hierarchy). Non-leaf lrqs hold other higher schedulable entities 617 * (like users, containers etc.) 618 * 619 * leaf_cfs_rq_list ties together list of leaf cfs_rq's in a CPU. 620 * This list is used during load balance. 621 */ 622 int on_list; 623 struct list_head leaf_cfs_rq_list; 624 struct task_group *tg; /* group that "owns" this runqueue */ 625 626 /* Locally cached copy of our task_group's idle value */ 627 int idle; 628 629 #ifdef CONFIG_CFS_BANDWIDTH 630 int runtime_enabled; 631 s64 runtime_remaining; 632 633 u64 throttled_pelt_idle; 634 #ifndef CONFIG_64BIT 635 u64 throttled_pelt_idle_copy; 636 #endif 637 u64 throttled_clock; 638 u64 throttled_clock_pelt; 639 u64 throttled_clock_pelt_time; 640 u64 throttled_clock_self; 641 u64 throttled_clock_self_time; 642 int throttled; 643 int throttle_count; 644 struct list_head throttled_list; 645 #ifdef CONFIG_SMP 646 struct list_head throttled_csd_list; 647 #endif 648 #endif /* CONFIG_CFS_BANDWIDTH */ 649 #endif /* CONFIG_FAIR_GROUP_SCHED */ 650 }; 651 652 static inline int rt_bandwidth_enabled(void) 653 { 654 return sysctl_sched_rt_runtime >= 0; 655 } 656 657 /* RT IPI pull logic requires IRQ_WORK */ 658 #if defined(CONFIG_IRQ_WORK) && defined(CONFIG_SMP) 659 # define HAVE_RT_PUSH_IPI 660 #endif 661 662 /* Real-Time classes' related field in a runqueue: */ 663 struct rt_rq { 664 struct rt_prio_array active; 665 unsigned int rt_nr_running; 666 unsigned int rr_nr_running; 667 #if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED 668 struct { 669 int curr; /* highest queued rt task prio */ 670 #ifdef CONFIG_SMP 671 int next; /* next highest */ 672 #endif 673 } highest_prio; 674 #endif 675 #ifdef CONFIG_SMP 676 unsigned int rt_nr_migratory; 677 unsigned int rt_nr_total; 678 int overloaded; 679 struct plist_head pushable_tasks; 680 681 #endif /* CONFIG_SMP */ 682 int rt_queued; 683 684 int rt_throttled; 685 u64 rt_time; 686 u64 rt_runtime; 687 /* Nests inside the rq lock: */ 688 raw_spinlock_t rt_runtime_lock; 689 690 #ifdef CONFIG_RT_GROUP_SCHED 691 unsigned int rt_nr_boosted; 692 693 struct rq *rq; 694 struct task_group *tg; 695 #endif 696 }; 697 698 static inline bool rt_rq_is_runnable(struct rt_rq *rt_rq) 699 { 700 return rt_rq->rt_queued && rt_rq->rt_nr_running; 701 } 702 703 /* Deadline class' related fields in a runqueue */ 704 struct dl_rq { 705 /* runqueue is an rbtree, ordered by deadline */ 706 struct rb_root_cached root; 707 708 unsigned int dl_nr_running; 709 710 #ifdef CONFIG_SMP 711 /* 712 * Deadline values of the currently executing and the 713 * earliest ready task on this rq. Caching these facilitates 714 * the decision whether or not a ready but not running task 715 * should migrate somewhere else. 716 */ 717 struct { 718 u64 curr; 719 u64 next; 720 } earliest_dl; 721 722 unsigned int dl_nr_migratory; 723 int overloaded; 724 725 /* 726 * Tasks on this rq that can be pushed away. They are kept in 727 * an rb-tree, ordered by tasks' deadlines, with caching 728 * of the leftmost (earliest deadline) element. 729 */ 730 struct rb_root_cached pushable_dl_tasks_root; 731 #else 732 struct dl_bw dl_bw; 733 #endif 734 /* 735 * "Active utilization" for this runqueue: increased when a 736 * task wakes up (becomes TASK_RUNNING) and decreased when a 737 * task blocks 738 */ 739 u64 running_bw; 740 741 /* 742 * Utilization of the tasks "assigned" to this runqueue (including 743 * the tasks that are in runqueue and the tasks that executed on this 744 * CPU and blocked). Increased when a task moves to this runqueue, and 745 * decreased when the task moves away (migrates, changes scheduling 746 * policy, or terminates). 747 * This is needed to compute the "inactive utilization" for the 748 * runqueue (inactive utilization = this_bw - running_bw). 749 */ 750 u64 this_bw; 751 u64 extra_bw; 752 753 /* 754 * Maximum available bandwidth for reclaiming by SCHED_FLAG_RECLAIM 755 * tasks of this rq. Used in calculation of reclaimable bandwidth(GRUB). 756 */ 757 u64 max_bw; 758 759 /* 760 * Inverse of the fraction of CPU utilization that can be reclaimed 761 * by the GRUB algorithm. 762 */ 763 u64 bw_ratio; 764 }; 765 766 #ifdef CONFIG_FAIR_GROUP_SCHED 767 /* An entity is a task if it doesn't "own" a runqueue */ 768 #define entity_is_task(se) (!se->my_q) 769 770 static inline void se_update_runnable(struct sched_entity *se) 771 { 772 if (!entity_is_task(se)) 773 se->runnable_weight = se->my_q->h_nr_running; 774 } 775 776 static inline long se_runnable(struct sched_entity *se) 777 { 778 if (entity_is_task(se)) 779 return !!se->on_rq; 780 else 781 return se->runnable_weight; 782 } 783 784 #else 785 #define entity_is_task(se) 1 786 787 static inline void se_update_runnable(struct sched_entity *se) {} 788 789 static inline long se_runnable(struct sched_entity *se) 790 { 791 return !!se->on_rq; 792 } 793 #endif 794 795 #ifdef CONFIG_SMP 796 /* 797 * XXX we want to get rid of these helpers and use the full load resolution. 798 */ 799 static inline long se_weight(struct sched_entity *se) 800 { 801 return scale_load_down(se->load.weight); 802 } 803 804 805 static inline bool sched_asym_prefer(int a, int b) 806 { 807 return arch_asym_cpu_priority(a) > arch_asym_cpu_priority(b); 808 } 809 810 struct perf_domain { 811 struct em_perf_domain *em_pd; 812 struct perf_domain *next; 813 struct rcu_head rcu; 814 }; 815 816 /* Scheduling group status flags */ 817 #define SG_OVERLOAD 0x1 /* More than one runnable task on a CPU. */ 818 #define SG_OVERUTILIZED 0x2 /* One or more CPUs are over-utilized. */ 819 820 /* 821 * We add the notion of a root-domain which will be used to define per-domain 822 * variables. Each exclusive cpuset essentially defines an island domain by 823 * fully partitioning the member CPUs from any other cpuset. Whenever a new 824 * exclusive cpuset is created, we also create and attach a new root-domain 825 * object. 826 * 827 */ 828 struct root_domain { 829 atomic_t refcount; 830 atomic_t rto_count; 831 struct rcu_head rcu; 832 cpumask_var_t span; 833 cpumask_var_t online; 834 835 /* 836 * Indicate pullable load on at least one CPU, e.g: 837 * - More than one runnable task 838 * - Running task is misfit 839 */ 840 int overload; 841 842 /* Indicate one or more cpus over-utilized (tipping point) */ 843 int overutilized; 844 845 /* 846 * The bit corresponding to a CPU gets set here if such CPU has more 847 * than one runnable -deadline task (as it is below for RT tasks). 848 */ 849 cpumask_var_t dlo_mask; 850 atomic_t dlo_count; 851 struct dl_bw dl_bw; 852 struct cpudl cpudl; 853 854 /* 855 * Indicate whether a root_domain's dl_bw has been checked or 856 * updated. It's monotonously increasing value. 857 * 858 * Also, some corner cases, like 'wrap around' is dangerous, but given 859 * that u64 is 'big enough'. So that shouldn't be a concern. 860 */ 861 u64 visit_gen; 862 863 #ifdef HAVE_RT_PUSH_IPI 864 /* 865 * For IPI pull requests, loop across the rto_mask. 866 */ 867 struct irq_work rto_push_work; 868 raw_spinlock_t rto_lock; 869 /* These are only updated and read within rto_lock */ 870 int rto_loop; 871 int rto_cpu; 872 /* These atomics are updated outside of a lock */ 873 atomic_t rto_loop_next; 874 atomic_t rto_loop_start; 875 #endif 876 /* 877 * The "RT overload" flag: it gets set if a CPU has more than 878 * one runnable RT task. 879 */ 880 cpumask_var_t rto_mask; 881 struct cpupri cpupri; 882 883 unsigned long max_cpu_capacity; 884 885 /* 886 * NULL-terminated list of performance domains intersecting with the 887 * CPUs of the rd. Protected by RCU. 888 */ 889 struct perf_domain __rcu *pd; 890 }; 891 892 extern void init_defrootdomain(void); 893 extern int sched_init_domains(const struct cpumask *cpu_map); 894 extern void rq_attach_root(struct rq *rq, struct root_domain *rd); 895 extern void sched_get_rd(struct root_domain *rd); 896 extern void sched_put_rd(struct root_domain *rd); 897 898 #ifdef HAVE_RT_PUSH_IPI 899 extern void rto_push_irq_work_func(struct irq_work *work); 900 #endif 901 #endif /* CONFIG_SMP */ 902 903 #ifdef CONFIG_UCLAMP_TASK 904 /* 905 * struct uclamp_bucket - Utilization clamp bucket 906 * @value: utilization clamp value for tasks on this clamp bucket 907 * @tasks: number of RUNNABLE tasks on this clamp bucket 908 * 909 * Keep track of how many tasks are RUNNABLE for a given utilization 910 * clamp value. 911 */ 912 struct uclamp_bucket { 913 unsigned long value : bits_per(SCHED_CAPACITY_SCALE); 914 unsigned long tasks : BITS_PER_LONG - bits_per(SCHED_CAPACITY_SCALE); 915 }; 916 917 /* 918 * struct uclamp_rq - rq's utilization clamp 919 * @value: currently active clamp values for a rq 920 * @bucket: utilization clamp buckets affecting a rq 921 * 922 * Keep track of RUNNABLE tasks on a rq to aggregate their clamp values. 923 * A clamp value is affecting a rq when there is at least one task RUNNABLE 924 * (or actually running) with that value. 925 * 926 * There are up to UCLAMP_CNT possible different clamp values, currently there 927 * are only two: minimum utilization and maximum utilization. 928 * 929 * All utilization clamping values are MAX aggregated, since: 930 * - for util_min: we want to run the CPU at least at the max of the minimum 931 * utilization required by its currently RUNNABLE tasks. 932 * - for util_max: we want to allow the CPU to run up to the max of the 933 * maximum utilization allowed by its currently RUNNABLE tasks. 934 * 935 * Since on each system we expect only a limited number of different 936 * utilization clamp values (UCLAMP_BUCKETS), use a simple array to track 937 * the metrics required to compute all the per-rq utilization clamp values. 938 */ 939 struct uclamp_rq { 940 unsigned int value; 941 struct uclamp_bucket bucket[UCLAMP_BUCKETS]; 942 }; 943 944 DECLARE_STATIC_KEY_FALSE(sched_uclamp_used); 945 #endif /* CONFIG_UCLAMP_TASK */ 946 947 struct rq; 948 struct balance_callback { 949 struct balance_callback *next; 950 void (*func)(struct rq *rq); 951 }; 952 953 /* 954 * This is the main, per-CPU runqueue data structure. 955 * 956 * Locking rule: those places that want to lock multiple runqueues 957 * (such as the load balancing or the thread migration code), lock 958 * acquire operations must be ordered by ascending &runqueue. 959 */ 960 struct rq { 961 /* runqueue lock: */ 962 raw_spinlock_t __lock; 963 964 /* 965 * nr_running and cpu_load should be in the same cacheline because 966 * remote CPUs use both these fields when doing load calculation. 967 */ 968 unsigned int nr_running; 969 #ifdef CONFIG_NUMA_BALANCING 970 unsigned int nr_numa_running; 971 unsigned int nr_preferred_running; 972 unsigned int numa_migrate_on; 973 #endif 974 #ifdef CONFIG_NO_HZ_COMMON 975 #ifdef CONFIG_SMP 976 unsigned long last_blocked_load_update_tick; 977 unsigned int has_blocked_load; 978 call_single_data_t nohz_csd; 979 #endif /* CONFIG_SMP */ 980 unsigned int nohz_tick_stopped; 981 atomic_t nohz_flags; 982 #endif /* CONFIG_NO_HZ_COMMON */ 983 984 #ifdef CONFIG_SMP 985 unsigned int ttwu_pending; 986 #endif 987 u64 nr_switches; 988 989 #ifdef CONFIG_UCLAMP_TASK 990 /* Utilization clamp values based on CPU's RUNNABLE tasks */ 991 struct uclamp_rq uclamp[UCLAMP_CNT] ____cacheline_aligned; 992 unsigned int uclamp_flags; 993 #define UCLAMP_FLAG_IDLE 0x01 994 #endif 995 996 struct cfs_rq cfs; 997 struct rt_rq rt; 998 struct dl_rq dl; 999 1000 #ifdef CONFIG_FAIR_GROUP_SCHED 1001 /* list of leaf cfs_rq on this CPU: */ 1002 struct list_head leaf_cfs_rq_list; 1003 struct list_head *tmp_alone_branch; 1004 #endif /* CONFIG_FAIR_GROUP_SCHED */ 1005 1006 /* 1007 * This is part of a global counter where only the total sum 1008 * over all CPUs matters. A task can increase this counter on 1009 * one CPU and if it got migrated afterwards it may decrease 1010 * it on another CPU. Always updated under the runqueue lock: 1011 */ 1012 unsigned int nr_uninterruptible; 1013 1014 struct task_struct __rcu *curr; 1015 struct task_struct *idle; 1016 struct task_struct *stop; 1017 unsigned long next_balance; 1018 struct mm_struct *prev_mm; 1019 1020 unsigned int clock_update_flags; 1021 u64 clock; 1022 /* Ensure that all clocks are in the same cache line */ 1023 u64 clock_task ____cacheline_aligned; 1024 u64 clock_pelt; 1025 unsigned long lost_idle_time; 1026 u64 clock_pelt_idle; 1027 u64 clock_idle; 1028 #ifndef CONFIG_64BIT 1029 u64 clock_pelt_idle_copy; 1030 u64 clock_idle_copy; 1031 #endif 1032 1033 atomic_t nr_iowait; 1034 1035 #ifdef CONFIG_SCHED_DEBUG 1036 u64 last_seen_need_resched_ns; 1037 int ticks_without_resched; 1038 #endif 1039 1040 #ifdef CONFIG_MEMBARRIER 1041 int membarrier_state; 1042 #endif 1043 1044 #ifdef CONFIG_SMP 1045 struct root_domain *rd; 1046 struct sched_domain __rcu *sd; 1047 1048 unsigned long cpu_capacity; 1049 unsigned long cpu_capacity_orig; 1050 1051 struct balance_callback *balance_callback; 1052 1053 unsigned char nohz_idle_balance; 1054 unsigned char idle_balance; 1055 1056 unsigned long misfit_task_load; 1057 1058 /* For active balancing */ 1059 int active_balance; 1060 int push_cpu; 1061 struct cpu_stop_work active_balance_work; 1062 1063 /* CPU of this runqueue: */ 1064 int cpu; 1065 int online; 1066 1067 struct list_head cfs_tasks; 1068 1069 struct sched_avg avg_rt; 1070 struct sched_avg avg_dl; 1071 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ 1072 struct sched_avg avg_irq; 1073 #endif 1074 #ifdef CONFIG_SCHED_THERMAL_PRESSURE 1075 struct sched_avg avg_thermal; 1076 #endif 1077 u64 idle_stamp; 1078 u64 avg_idle; 1079 1080 unsigned long wake_stamp; 1081 u64 wake_avg_idle; 1082 1083 /* This is used to determine avg_idle's max value */ 1084 u64 max_idle_balance_cost; 1085 1086 #ifdef CONFIG_HOTPLUG_CPU 1087 struct rcuwait hotplug_wait; 1088 #endif 1089 #endif /* CONFIG_SMP */ 1090 1091 #ifdef CONFIG_IRQ_TIME_ACCOUNTING 1092 u64 prev_irq_time; 1093 #endif 1094 #ifdef CONFIG_PARAVIRT 1095 u64 prev_steal_time; 1096 #endif 1097 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING 1098 u64 prev_steal_time_rq; 1099 #endif 1100 1101 /* calc_load related fields */ 1102 unsigned long calc_load_update; 1103 long calc_load_active; 1104 1105 #ifdef CONFIG_SCHED_HRTICK 1106 #ifdef CONFIG_SMP 1107 call_single_data_t hrtick_csd; 1108 #endif 1109 struct hrtimer hrtick_timer; 1110 ktime_t hrtick_time; 1111 #endif 1112 1113 #ifdef CONFIG_SCHEDSTATS 1114 /* latency stats */ 1115 struct sched_info rq_sched_info; 1116 unsigned long long rq_cpu_time; 1117 /* could above be rq->cfs_rq.exec_clock + rq->rt_rq.rt_runtime ? */ 1118 1119 /* sys_sched_yield() stats */ 1120 unsigned int yld_count; 1121 1122 /* schedule() stats */ 1123 unsigned int sched_count; 1124 unsigned int sched_goidle; 1125 1126 /* try_to_wake_up() stats */ 1127 unsigned int ttwu_count; 1128 unsigned int ttwu_local; 1129 #endif 1130 1131 #ifdef CONFIG_CPU_IDLE 1132 /* Must be inspected within a rcu lock section */ 1133 struct cpuidle_state *idle_state; 1134 #endif 1135 1136 #ifdef CONFIG_SMP 1137 unsigned int nr_pinned; 1138 #endif 1139 unsigned int push_busy; 1140 struct cpu_stop_work push_work; 1141 1142 #ifdef CONFIG_SCHED_CORE 1143 /* per rq */ 1144 struct rq *core; 1145 struct task_struct *core_pick; 1146 unsigned int core_enabled; 1147 unsigned int core_sched_seq; 1148 struct rb_root core_tree; 1149 1150 /* shared state -- careful with sched_core_cpu_deactivate() */ 1151 unsigned int core_task_seq; 1152 unsigned int core_pick_seq; 1153 unsigned long core_cookie; 1154 unsigned int core_forceidle_count; 1155 unsigned int core_forceidle_seq; 1156 unsigned int core_forceidle_occupation; 1157 u64 core_forceidle_start; 1158 #endif 1159 1160 /* Scratch cpumask to be temporarily used under rq_lock */ 1161 cpumask_var_t scratch_mask; 1162 1163 #if defined(CONFIG_CFS_BANDWIDTH) && defined(CONFIG_SMP) 1164 call_single_data_t cfsb_csd; 1165 struct list_head cfsb_csd_list; 1166 #endif 1167 }; 1168 1169 #ifdef CONFIG_FAIR_GROUP_SCHED 1170 1171 /* CPU runqueue to which this cfs_rq is attached */ 1172 static inline struct rq *rq_of(struct cfs_rq *cfs_rq) 1173 { 1174 return cfs_rq->rq; 1175 } 1176 1177 #else 1178 1179 static inline struct rq *rq_of(struct cfs_rq *cfs_rq) 1180 { 1181 return container_of(cfs_rq, struct rq, cfs); 1182 } 1183 #endif 1184 1185 static inline int cpu_of(struct rq *rq) 1186 { 1187 #ifdef CONFIG_SMP 1188 return rq->cpu; 1189 #else 1190 return 0; 1191 #endif 1192 } 1193 1194 #define MDF_PUSH 0x01 1195 1196 static inline bool is_migration_disabled(struct task_struct *p) 1197 { 1198 #ifdef CONFIG_SMP 1199 return p->migration_disabled; 1200 #else 1201 return false; 1202 #endif 1203 } 1204 1205 DECLARE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); 1206 1207 #define cpu_rq(cpu) (&per_cpu(runqueues, (cpu))) 1208 #define this_rq() this_cpu_ptr(&runqueues) 1209 #define task_rq(p) cpu_rq(task_cpu(p)) 1210 #define cpu_curr(cpu) (cpu_rq(cpu)->curr) 1211 #define raw_rq() raw_cpu_ptr(&runqueues) 1212 1213 struct sched_group; 1214 #ifdef CONFIG_SCHED_CORE 1215 static inline struct cpumask *sched_group_span(struct sched_group *sg); 1216 1217 DECLARE_STATIC_KEY_FALSE(__sched_core_enabled); 1218 1219 static inline bool sched_core_enabled(struct rq *rq) 1220 { 1221 return static_branch_unlikely(&__sched_core_enabled) && rq->core_enabled; 1222 } 1223 1224 static inline bool sched_core_disabled(void) 1225 { 1226 return !static_branch_unlikely(&__sched_core_enabled); 1227 } 1228 1229 /* 1230 * Be careful with this function; not for general use. The return value isn't 1231 * stable unless you actually hold a relevant rq->__lock. 1232 */ 1233 static inline raw_spinlock_t *rq_lockp(struct rq *rq) 1234 { 1235 if (sched_core_enabled(rq)) 1236 return &rq->core->__lock; 1237 1238 return &rq->__lock; 1239 } 1240 1241 static inline raw_spinlock_t *__rq_lockp(struct rq *rq) 1242 { 1243 if (rq->core_enabled) 1244 return &rq->core->__lock; 1245 1246 return &rq->__lock; 1247 } 1248 1249 bool cfs_prio_less(const struct task_struct *a, const struct task_struct *b, 1250 bool fi); 1251 1252 /* 1253 * Helpers to check if the CPU's core cookie matches with the task's cookie 1254 * when core scheduling is enabled. 1255 * A special case is that the task's cookie always matches with CPU's core 1256 * cookie if the CPU is in an idle core. 1257 */ 1258 static inline bool sched_cpu_cookie_match(struct rq *rq, struct task_struct *p) 1259 { 1260 /* Ignore cookie match if core scheduler is not enabled on the CPU. */ 1261 if (!sched_core_enabled(rq)) 1262 return true; 1263 1264 return rq->core->core_cookie == p->core_cookie; 1265 } 1266 1267 static inline bool sched_core_cookie_match(struct rq *rq, struct task_struct *p) 1268 { 1269 bool idle_core = true; 1270 int cpu; 1271 1272 /* Ignore cookie match if core scheduler is not enabled on the CPU. */ 1273 if (!sched_core_enabled(rq)) 1274 return true; 1275 1276 for_each_cpu(cpu, cpu_smt_mask(cpu_of(rq))) { 1277 if (!available_idle_cpu(cpu)) { 1278 idle_core = false; 1279 break; 1280 } 1281 } 1282 1283 /* 1284 * A CPU in an idle core is always the best choice for tasks with 1285 * cookies. 1286 */ 1287 return idle_core || rq->core->core_cookie == p->core_cookie; 1288 } 1289 1290 static inline bool sched_group_cookie_match(struct rq *rq, 1291 struct task_struct *p, 1292 struct sched_group *group) 1293 { 1294 int cpu; 1295 1296 /* Ignore cookie match if core scheduler is not enabled on the CPU. */ 1297 if (!sched_core_enabled(rq)) 1298 return true; 1299 1300 for_each_cpu_and(cpu, sched_group_span(group), p->cpus_ptr) { 1301 if (sched_core_cookie_match(cpu_rq(cpu), p)) 1302 return true; 1303 } 1304 return false; 1305 } 1306 1307 static inline bool sched_core_enqueued(struct task_struct *p) 1308 { 1309 return !RB_EMPTY_NODE(&p->core_node); 1310 } 1311 1312 extern void sched_core_enqueue(struct rq *rq, struct task_struct *p); 1313 extern void sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags); 1314 1315 extern void sched_core_get(void); 1316 extern void sched_core_put(void); 1317 1318 #else /* !CONFIG_SCHED_CORE */ 1319 1320 static inline bool sched_core_enabled(struct rq *rq) 1321 { 1322 return false; 1323 } 1324 1325 static inline bool sched_core_disabled(void) 1326 { 1327 return true; 1328 } 1329 1330 static inline raw_spinlock_t *rq_lockp(struct rq *rq) 1331 { 1332 return &rq->__lock; 1333 } 1334 1335 static inline raw_spinlock_t *__rq_lockp(struct rq *rq) 1336 { 1337 return &rq->__lock; 1338 } 1339 1340 static inline bool sched_cpu_cookie_match(struct rq *rq, struct task_struct *p) 1341 { 1342 return true; 1343 } 1344 1345 static inline bool sched_core_cookie_match(struct rq *rq, struct task_struct *p) 1346 { 1347 return true; 1348 } 1349 1350 static inline bool sched_group_cookie_match(struct rq *rq, 1351 struct task_struct *p, 1352 struct sched_group *group) 1353 { 1354 return true; 1355 } 1356 #endif /* CONFIG_SCHED_CORE */ 1357 1358 static inline void lockdep_assert_rq_held(struct rq *rq) 1359 { 1360 lockdep_assert_held(__rq_lockp(rq)); 1361 } 1362 1363 extern void raw_spin_rq_lock_nested(struct rq *rq, int subclass); 1364 extern bool raw_spin_rq_trylock(struct rq *rq); 1365 extern void raw_spin_rq_unlock(struct rq *rq); 1366 1367 static inline void raw_spin_rq_lock(struct rq *rq) 1368 { 1369 raw_spin_rq_lock_nested(rq, 0); 1370 } 1371 1372 static inline void raw_spin_rq_lock_irq(struct rq *rq) 1373 { 1374 local_irq_disable(); 1375 raw_spin_rq_lock(rq); 1376 } 1377 1378 static inline void raw_spin_rq_unlock_irq(struct rq *rq) 1379 { 1380 raw_spin_rq_unlock(rq); 1381 local_irq_enable(); 1382 } 1383 1384 static inline unsigned long _raw_spin_rq_lock_irqsave(struct rq *rq) 1385 { 1386 unsigned long flags; 1387 local_irq_save(flags); 1388 raw_spin_rq_lock(rq); 1389 return flags; 1390 } 1391 1392 static inline void raw_spin_rq_unlock_irqrestore(struct rq *rq, unsigned long flags) 1393 { 1394 raw_spin_rq_unlock(rq); 1395 local_irq_restore(flags); 1396 } 1397 1398 #define raw_spin_rq_lock_irqsave(rq, flags) \ 1399 do { \ 1400 flags = _raw_spin_rq_lock_irqsave(rq); \ 1401 } while (0) 1402 1403 #ifdef CONFIG_SCHED_SMT 1404 extern void __update_idle_core(struct rq *rq); 1405 1406 static inline void update_idle_core(struct rq *rq) 1407 { 1408 if (static_branch_unlikely(&sched_smt_present)) 1409 __update_idle_core(rq); 1410 } 1411 1412 #else 1413 static inline void update_idle_core(struct rq *rq) { } 1414 #endif 1415 1416 #ifdef CONFIG_FAIR_GROUP_SCHED 1417 static inline struct task_struct *task_of(struct sched_entity *se) 1418 { 1419 SCHED_WARN_ON(!entity_is_task(se)); 1420 return container_of(se, struct task_struct, se); 1421 } 1422 1423 static inline struct cfs_rq *task_cfs_rq(struct task_struct *p) 1424 { 1425 return p->se.cfs_rq; 1426 } 1427 1428 /* runqueue on which this entity is (to be) queued */ 1429 static inline struct cfs_rq *cfs_rq_of(const struct sched_entity *se) 1430 { 1431 return se->cfs_rq; 1432 } 1433 1434 /* runqueue "owned" by this group */ 1435 static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp) 1436 { 1437 return grp->my_q; 1438 } 1439 1440 #else 1441 1442 #define task_of(_se) container_of(_se, struct task_struct, se) 1443 1444 static inline struct cfs_rq *task_cfs_rq(const struct task_struct *p) 1445 { 1446 return &task_rq(p)->cfs; 1447 } 1448 1449 static inline struct cfs_rq *cfs_rq_of(const struct sched_entity *se) 1450 { 1451 const struct task_struct *p = task_of(se); 1452 struct rq *rq = task_rq(p); 1453 1454 return &rq->cfs; 1455 } 1456 1457 /* runqueue "owned" by this group */ 1458 static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp) 1459 { 1460 return NULL; 1461 } 1462 #endif 1463 1464 extern void update_rq_clock(struct rq *rq); 1465 1466 /* 1467 * rq::clock_update_flags bits 1468 * 1469 * %RQCF_REQ_SKIP - will request skipping of clock update on the next 1470 * call to __schedule(). This is an optimisation to avoid 1471 * neighbouring rq clock updates. 1472 * 1473 * %RQCF_ACT_SKIP - is set from inside of __schedule() when skipping is 1474 * in effect and calls to update_rq_clock() are being ignored. 1475 * 1476 * %RQCF_UPDATED - is a debug flag that indicates whether a call has been 1477 * made to update_rq_clock() since the last time rq::lock was pinned. 1478 * 1479 * If inside of __schedule(), clock_update_flags will have been 1480 * shifted left (a left shift is a cheap operation for the fast path 1481 * to promote %RQCF_REQ_SKIP to %RQCF_ACT_SKIP), so you must use, 1482 * 1483 * if (rq-clock_update_flags >= RQCF_UPDATED) 1484 * 1485 * to check if %RQCF_UPDATED is set. It'll never be shifted more than 1486 * one position though, because the next rq_unpin_lock() will shift it 1487 * back. 1488 */ 1489 #define RQCF_REQ_SKIP 0x01 1490 #define RQCF_ACT_SKIP 0x02 1491 #define RQCF_UPDATED 0x04 1492 1493 static inline void assert_clock_updated(struct rq *rq) 1494 { 1495 /* 1496 * The only reason for not seeing a clock update since the 1497 * last rq_pin_lock() is if we're currently skipping updates. 1498 */ 1499 SCHED_WARN_ON(rq->clock_update_flags < RQCF_ACT_SKIP); 1500 } 1501 1502 static inline u64 rq_clock(struct rq *rq) 1503 { 1504 lockdep_assert_rq_held(rq); 1505 assert_clock_updated(rq); 1506 1507 return rq->clock; 1508 } 1509 1510 static inline u64 rq_clock_task(struct rq *rq) 1511 { 1512 lockdep_assert_rq_held(rq); 1513 assert_clock_updated(rq); 1514 1515 return rq->clock_task; 1516 } 1517 1518 /** 1519 * By default the decay is the default pelt decay period. 1520 * The decay shift can change the decay period in 1521 * multiples of 32. 1522 * Decay shift Decay period(ms) 1523 * 0 32 1524 * 1 64 1525 * 2 128 1526 * 3 256 1527 * 4 512 1528 */ 1529 extern int sched_thermal_decay_shift; 1530 1531 static inline u64 rq_clock_thermal(struct rq *rq) 1532 { 1533 return rq_clock_task(rq) >> sched_thermal_decay_shift; 1534 } 1535 1536 static inline void rq_clock_skip_update(struct rq *rq) 1537 { 1538 lockdep_assert_rq_held(rq); 1539 rq->clock_update_flags |= RQCF_REQ_SKIP; 1540 } 1541 1542 /* 1543 * See rt task throttling, which is the only time a skip 1544 * request is canceled. 1545 */ 1546 static inline void rq_clock_cancel_skipupdate(struct rq *rq) 1547 { 1548 lockdep_assert_rq_held(rq); 1549 rq->clock_update_flags &= ~RQCF_REQ_SKIP; 1550 } 1551 1552 /* 1553 * During cpu offlining and rq wide unthrottling, we can trigger 1554 * an update_rq_clock() for several cfs and rt runqueues (Typically 1555 * when using list_for_each_entry_*) 1556 * rq_clock_start_loop_update() can be called after updating the clock 1557 * once and before iterating over the list to prevent multiple update. 1558 * After the iterative traversal, we need to call rq_clock_stop_loop_update() 1559 * to clear RQCF_ACT_SKIP of rq->clock_update_flags. 1560 */ 1561 static inline void rq_clock_start_loop_update(struct rq *rq) 1562 { 1563 lockdep_assert_rq_held(rq); 1564 SCHED_WARN_ON(rq->clock_update_flags & RQCF_ACT_SKIP); 1565 rq->clock_update_flags |= RQCF_ACT_SKIP; 1566 } 1567 1568 static inline void rq_clock_stop_loop_update(struct rq *rq) 1569 { 1570 lockdep_assert_rq_held(rq); 1571 rq->clock_update_flags &= ~RQCF_ACT_SKIP; 1572 } 1573 1574 struct rq_flags { 1575 unsigned long flags; 1576 struct pin_cookie cookie; 1577 #ifdef CONFIG_SCHED_DEBUG 1578 /* 1579 * A copy of (rq::clock_update_flags & RQCF_UPDATED) for the 1580 * current pin context is stashed here in case it needs to be 1581 * restored in rq_repin_lock(). 1582 */ 1583 unsigned int clock_update_flags; 1584 #endif 1585 }; 1586 1587 extern struct balance_callback balance_push_callback; 1588 1589 /* 1590 * Lockdep annotation that avoids accidental unlocks; it's like a 1591 * sticky/continuous lockdep_assert_held(). 1592 * 1593 * This avoids code that has access to 'struct rq *rq' (basically everything in 1594 * the scheduler) from accidentally unlocking the rq if they do not also have a 1595 * copy of the (on-stack) 'struct rq_flags rf'. 1596 * 1597 * Also see Documentation/locking/lockdep-design.rst. 1598 */ 1599 static inline void rq_pin_lock(struct rq *rq, struct rq_flags *rf) 1600 { 1601 rf->cookie = lockdep_pin_lock(__rq_lockp(rq)); 1602 1603 #ifdef CONFIG_SCHED_DEBUG 1604 rq->clock_update_flags &= (RQCF_REQ_SKIP|RQCF_ACT_SKIP); 1605 rf->clock_update_flags = 0; 1606 #ifdef CONFIG_SMP 1607 SCHED_WARN_ON(rq->balance_callback && rq->balance_callback != &balance_push_callback); 1608 #endif 1609 #endif 1610 } 1611 1612 static inline void rq_unpin_lock(struct rq *rq, struct rq_flags *rf) 1613 { 1614 #ifdef CONFIG_SCHED_DEBUG 1615 if (rq->clock_update_flags > RQCF_ACT_SKIP) 1616 rf->clock_update_flags = RQCF_UPDATED; 1617 #endif 1618 1619 lockdep_unpin_lock(__rq_lockp(rq), rf->cookie); 1620 } 1621 1622 static inline void rq_repin_lock(struct rq *rq, struct rq_flags *rf) 1623 { 1624 lockdep_repin_lock(__rq_lockp(rq), rf->cookie); 1625 1626 #ifdef CONFIG_SCHED_DEBUG 1627 /* 1628 * Restore the value we stashed in @rf for this pin context. 1629 */ 1630 rq->clock_update_flags |= rf->clock_update_flags; 1631 #endif 1632 } 1633 1634 struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf) 1635 __acquires(rq->lock); 1636 1637 struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf) 1638 __acquires(p->pi_lock) 1639 __acquires(rq->lock); 1640 1641 static inline void __task_rq_unlock(struct rq *rq, struct rq_flags *rf) 1642 __releases(rq->lock) 1643 { 1644 rq_unpin_lock(rq, rf); 1645 raw_spin_rq_unlock(rq); 1646 } 1647 1648 static inline void 1649 task_rq_unlock(struct rq *rq, struct task_struct *p, struct rq_flags *rf) 1650 __releases(rq->lock) 1651 __releases(p->pi_lock) 1652 { 1653 rq_unpin_lock(rq, rf); 1654 raw_spin_rq_unlock(rq); 1655 raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags); 1656 } 1657 1658 static inline void 1659 rq_lock_irqsave(struct rq *rq, struct rq_flags *rf) 1660 __acquires(rq->lock) 1661 { 1662 raw_spin_rq_lock_irqsave(rq, rf->flags); 1663 rq_pin_lock(rq, rf); 1664 } 1665 1666 static inline void 1667 rq_lock_irq(struct rq *rq, struct rq_flags *rf) 1668 __acquires(rq->lock) 1669 { 1670 raw_spin_rq_lock_irq(rq); 1671 rq_pin_lock(rq, rf); 1672 } 1673 1674 static inline void 1675 rq_lock(struct rq *rq, struct rq_flags *rf) 1676 __acquires(rq->lock) 1677 { 1678 raw_spin_rq_lock(rq); 1679 rq_pin_lock(rq, rf); 1680 } 1681 1682 static inline void 1683 rq_unlock_irqrestore(struct rq *rq, struct rq_flags *rf) 1684 __releases(rq->lock) 1685 { 1686 rq_unpin_lock(rq, rf); 1687 raw_spin_rq_unlock_irqrestore(rq, rf->flags); 1688 } 1689 1690 static inline void 1691 rq_unlock_irq(struct rq *rq, struct rq_flags *rf) 1692 __releases(rq->lock) 1693 { 1694 rq_unpin_lock(rq, rf); 1695 raw_spin_rq_unlock_irq(rq); 1696 } 1697 1698 static inline void 1699 rq_unlock(struct rq *rq, struct rq_flags *rf) 1700 __releases(rq->lock) 1701 { 1702 rq_unpin_lock(rq, rf); 1703 raw_spin_rq_unlock(rq); 1704 } 1705 1706 static inline struct rq * 1707 this_rq_lock_irq(struct rq_flags *rf) 1708 __acquires(rq->lock) 1709 { 1710 struct rq *rq; 1711 1712 local_irq_disable(); 1713 rq = this_rq(); 1714 rq_lock(rq, rf); 1715 return rq; 1716 } 1717 1718 #ifdef CONFIG_NUMA 1719 enum numa_topology_type { 1720 NUMA_DIRECT, 1721 NUMA_GLUELESS_MESH, 1722 NUMA_BACKPLANE, 1723 }; 1724 extern enum numa_topology_type sched_numa_topology_type; 1725 extern int sched_max_numa_distance; 1726 extern bool find_numa_distance(int distance); 1727 extern void sched_init_numa(int offline_node); 1728 extern void sched_update_numa(int cpu, bool online); 1729 extern void sched_domains_numa_masks_set(unsigned int cpu); 1730 extern void sched_domains_numa_masks_clear(unsigned int cpu); 1731 extern int sched_numa_find_closest(const struct cpumask *cpus, int cpu); 1732 #else 1733 static inline void sched_init_numa(int offline_node) { } 1734 static inline void sched_update_numa(int cpu, bool online) { } 1735 static inline void sched_domains_numa_masks_set(unsigned int cpu) { } 1736 static inline void sched_domains_numa_masks_clear(unsigned int cpu) { } 1737 static inline int sched_numa_find_closest(const struct cpumask *cpus, int cpu) 1738 { 1739 return nr_cpu_ids; 1740 } 1741 #endif 1742 1743 #ifdef CONFIG_NUMA_BALANCING 1744 /* The regions in numa_faults array from task_struct */ 1745 enum numa_faults_stats { 1746 NUMA_MEM = 0, 1747 NUMA_CPU, 1748 NUMA_MEMBUF, 1749 NUMA_CPUBUF 1750 }; 1751 extern void sched_setnuma(struct task_struct *p, int node); 1752 extern int migrate_task_to(struct task_struct *p, int cpu); 1753 extern int migrate_swap(struct task_struct *p, struct task_struct *t, 1754 int cpu, int scpu); 1755 extern void init_numa_balancing(unsigned long clone_flags, struct task_struct *p); 1756 #else 1757 static inline void 1758 init_numa_balancing(unsigned long clone_flags, struct task_struct *p) 1759 { 1760 } 1761 #endif /* CONFIG_NUMA_BALANCING */ 1762 1763 #ifdef CONFIG_SMP 1764 1765 static inline void 1766 queue_balance_callback(struct rq *rq, 1767 struct balance_callback *head, 1768 void (*func)(struct rq *rq)) 1769 { 1770 lockdep_assert_rq_held(rq); 1771 1772 /* 1773 * Don't (re)queue an already queued item; nor queue anything when 1774 * balance_push() is active, see the comment with 1775 * balance_push_callback. 1776 */ 1777 if (unlikely(head->next || rq->balance_callback == &balance_push_callback)) 1778 return; 1779 1780 head->func = func; 1781 head->next = rq->balance_callback; 1782 rq->balance_callback = head; 1783 } 1784 1785 #define rcu_dereference_check_sched_domain(p) \ 1786 rcu_dereference_check((p), \ 1787 lockdep_is_held(&sched_domains_mutex)) 1788 1789 /* 1790 * The domain tree (rq->sd) is protected by RCU's quiescent state transition. 1791 * See destroy_sched_domains: call_rcu for details. 1792 * 1793 * The domain tree of any CPU may only be accessed from within 1794 * preempt-disabled sections. 1795 */ 1796 #define for_each_domain(cpu, __sd) \ 1797 for (__sd = rcu_dereference_check_sched_domain(cpu_rq(cpu)->sd); \ 1798 __sd; __sd = __sd->parent) 1799 1800 /* A mask of all the SD flags that have the SDF_SHARED_CHILD metaflag */ 1801 #define SD_FLAG(name, mflags) (name * !!((mflags) & SDF_SHARED_CHILD)) | 1802 static const unsigned int SD_SHARED_CHILD_MASK = 1803 #include <linux/sched/sd_flags.h> 1804 0; 1805 #undef SD_FLAG 1806 1807 /** 1808 * highest_flag_domain - Return highest sched_domain containing flag. 1809 * @cpu: The CPU whose highest level of sched domain is to 1810 * be returned. 1811 * @flag: The flag to check for the highest sched_domain 1812 * for the given CPU. 1813 * 1814 * Returns the highest sched_domain of a CPU which contains @flag. If @flag has 1815 * the SDF_SHARED_CHILD metaflag, all the children domains also have @flag. 1816 */ 1817 static inline struct sched_domain *highest_flag_domain(int cpu, int flag) 1818 { 1819 struct sched_domain *sd, *hsd = NULL; 1820 1821 for_each_domain(cpu, sd) { 1822 if (sd->flags & flag) { 1823 hsd = sd; 1824 continue; 1825 } 1826 1827 /* 1828 * Stop the search if @flag is known to be shared at lower 1829 * levels. It will not be found further up. 1830 */ 1831 if (flag & SD_SHARED_CHILD_MASK) 1832 break; 1833 } 1834 1835 return hsd; 1836 } 1837 1838 static inline struct sched_domain *lowest_flag_domain(int cpu, int flag) 1839 { 1840 struct sched_domain *sd; 1841 1842 for_each_domain(cpu, sd) { 1843 if (sd->flags & flag) 1844 break; 1845 } 1846 1847 return sd; 1848 } 1849 1850 DECLARE_PER_CPU(struct sched_domain __rcu *, sd_llc); 1851 DECLARE_PER_CPU(int, sd_llc_size); 1852 DECLARE_PER_CPU(int, sd_llc_id); 1853 DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared); 1854 DECLARE_PER_CPU(struct sched_domain __rcu *, sd_numa); 1855 DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing); 1856 DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity); 1857 extern struct static_key_false sched_asym_cpucapacity; 1858 1859 static __always_inline bool sched_asym_cpucap_active(void) 1860 { 1861 return static_branch_unlikely(&sched_asym_cpucapacity); 1862 } 1863 1864 struct sched_group_capacity { 1865 atomic_t ref; 1866 /* 1867 * CPU capacity of this group, SCHED_CAPACITY_SCALE being max capacity 1868 * for a single CPU. 1869 */ 1870 unsigned long capacity; 1871 unsigned long min_capacity; /* Min per-CPU capacity in group */ 1872 unsigned long max_capacity; /* Max per-CPU capacity in group */ 1873 unsigned long next_update; 1874 int imbalance; /* XXX unrelated to capacity but shared group state */ 1875 1876 #ifdef CONFIG_SCHED_DEBUG 1877 int id; 1878 #endif 1879 1880 unsigned long cpumask[]; /* Balance mask */ 1881 }; 1882 1883 struct sched_group { 1884 struct sched_group *next; /* Must be a circular list */ 1885 atomic_t ref; 1886 1887 unsigned int group_weight; 1888 unsigned int cores; 1889 struct sched_group_capacity *sgc; 1890 int asym_prefer_cpu; /* CPU of highest priority in group */ 1891 int flags; 1892 1893 /* 1894 * The CPUs this group covers. 1895 * 1896 * NOTE: this field is variable length. (Allocated dynamically 1897 * by attaching extra space to the end of the structure, 1898 * depending on how many CPUs the kernel has booted up with) 1899 */ 1900 unsigned long cpumask[]; 1901 }; 1902 1903 static inline struct cpumask *sched_group_span(struct sched_group *sg) 1904 { 1905 return to_cpumask(sg->cpumask); 1906 } 1907 1908 /* 1909 * See build_balance_mask(). 1910 */ 1911 static inline struct cpumask *group_balance_mask(struct sched_group *sg) 1912 { 1913 return to_cpumask(sg->sgc->cpumask); 1914 } 1915 1916 extern int group_balance_cpu(struct sched_group *sg); 1917 1918 #ifdef CONFIG_SCHED_DEBUG 1919 void update_sched_domain_debugfs(void); 1920 void dirty_sched_domain_sysctl(int cpu); 1921 #else 1922 static inline void update_sched_domain_debugfs(void) 1923 { 1924 } 1925 static inline void dirty_sched_domain_sysctl(int cpu) 1926 { 1927 } 1928 #endif 1929 1930 extern int sched_update_scaling(void); 1931 1932 static inline const struct cpumask *task_user_cpus(struct task_struct *p) 1933 { 1934 if (!p->user_cpus_ptr) 1935 return cpu_possible_mask; /* &init_task.cpus_mask */ 1936 return p->user_cpus_ptr; 1937 } 1938 #endif /* CONFIG_SMP */ 1939 1940 #include "stats.h" 1941 1942 #if defined(CONFIG_SCHED_CORE) && defined(CONFIG_SCHEDSTATS) 1943 1944 extern void __sched_core_account_forceidle(struct rq *rq); 1945 1946 static inline void sched_core_account_forceidle(struct rq *rq) 1947 { 1948 if (schedstat_enabled()) 1949 __sched_core_account_forceidle(rq); 1950 } 1951 1952 extern void __sched_core_tick(struct rq *rq); 1953 1954 static inline void sched_core_tick(struct rq *rq) 1955 { 1956 if (sched_core_enabled(rq) && schedstat_enabled()) 1957 __sched_core_tick(rq); 1958 } 1959 1960 #else 1961 1962 static inline void sched_core_account_forceidle(struct rq *rq) {} 1963 1964 static inline void sched_core_tick(struct rq *rq) {} 1965 1966 #endif /* CONFIG_SCHED_CORE && CONFIG_SCHEDSTATS */ 1967 1968 #ifdef CONFIG_CGROUP_SCHED 1969 1970 /* 1971 * Return the group to which this tasks belongs. 1972 * 1973 * We cannot use task_css() and friends because the cgroup subsystem 1974 * changes that value before the cgroup_subsys::attach() method is called, 1975 * therefore we cannot pin it and might observe the wrong value. 1976 * 1977 * The same is true for autogroup's p->signal->autogroup->tg, the autogroup 1978 * core changes this before calling sched_move_task(). 1979 * 1980 * Instead we use a 'copy' which is updated from sched_move_task() while 1981 * holding both task_struct::pi_lock and rq::lock. 1982 */ 1983 static inline struct task_group *task_group(struct task_struct *p) 1984 { 1985 return p->sched_task_group; 1986 } 1987 1988 /* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */ 1989 static inline void set_task_rq(struct task_struct *p, unsigned int cpu) 1990 { 1991 #if defined(CONFIG_FAIR_GROUP_SCHED) || defined(CONFIG_RT_GROUP_SCHED) 1992 struct task_group *tg = task_group(p); 1993 #endif 1994 1995 #ifdef CONFIG_FAIR_GROUP_SCHED 1996 set_task_rq_fair(&p->se, p->se.cfs_rq, tg->cfs_rq[cpu]); 1997 p->se.cfs_rq = tg->cfs_rq[cpu]; 1998 p->se.parent = tg->se[cpu]; 1999 p->se.depth = tg->se[cpu] ? tg->se[cpu]->depth + 1 : 0; 2000 #endif 2001 2002 #ifdef CONFIG_RT_GROUP_SCHED 2003 p->rt.rt_rq = tg->rt_rq[cpu]; 2004 p->rt.parent = tg->rt_se[cpu]; 2005 #endif 2006 } 2007 2008 #else /* CONFIG_CGROUP_SCHED */ 2009 2010 static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { } 2011 static inline struct task_group *task_group(struct task_struct *p) 2012 { 2013 return NULL; 2014 } 2015 2016 #endif /* CONFIG_CGROUP_SCHED */ 2017 2018 static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu) 2019 { 2020 set_task_rq(p, cpu); 2021 #ifdef CONFIG_SMP 2022 /* 2023 * After ->cpu is set up to a new value, task_rq_lock(p, ...) can be 2024 * successfully executed on another CPU. We must ensure that updates of 2025 * per-task data have been completed by this moment. 2026 */ 2027 smp_wmb(); 2028 WRITE_ONCE(task_thread_info(p)->cpu, cpu); 2029 p->wake_cpu = cpu; 2030 #endif 2031 } 2032 2033 /* 2034 * Tunables that become constants when CONFIG_SCHED_DEBUG is off: 2035 */ 2036 #ifdef CONFIG_SCHED_DEBUG 2037 # define const_debug __read_mostly 2038 #else 2039 # define const_debug const 2040 #endif 2041 2042 #define SCHED_FEAT(name, enabled) \ 2043 __SCHED_FEAT_##name , 2044 2045 enum { 2046 #include "features.h" 2047 __SCHED_FEAT_NR, 2048 }; 2049 2050 #undef SCHED_FEAT 2051 2052 #ifdef CONFIG_SCHED_DEBUG 2053 2054 /* 2055 * To support run-time toggling of sched features, all the translation units 2056 * (but core.c) reference the sysctl_sched_features defined in core.c. 2057 */ 2058 extern const_debug unsigned int sysctl_sched_features; 2059 2060 #ifdef CONFIG_JUMP_LABEL 2061 #define SCHED_FEAT(name, enabled) \ 2062 static __always_inline bool static_branch_##name(struct static_key *key) \ 2063 { \ 2064 return static_key_##enabled(key); \ 2065 } 2066 2067 #include "features.h" 2068 #undef SCHED_FEAT 2069 2070 extern struct static_key sched_feat_keys[__SCHED_FEAT_NR]; 2071 #define sched_feat(x) (static_branch_##x(&sched_feat_keys[__SCHED_FEAT_##x])) 2072 2073 #else /* !CONFIG_JUMP_LABEL */ 2074 2075 #define sched_feat(x) (sysctl_sched_features & (1UL << __SCHED_FEAT_##x)) 2076 2077 #endif /* CONFIG_JUMP_LABEL */ 2078 2079 #else /* !SCHED_DEBUG */ 2080 2081 /* 2082 * Each translation unit has its own copy of sysctl_sched_features to allow 2083 * constants propagation at compile time and compiler optimization based on 2084 * features default. 2085 */ 2086 #define SCHED_FEAT(name, enabled) \ 2087 (1UL << __SCHED_FEAT_##name) * enabled | 2088 static const_debug __maybe_unused unsigned int sysctl_sched_features = 2089 #include "features.h" 2090 0; 2091 #undef SCHED_FEAT 2092 2093 #define sched_feat(x) !!(sysctl_sched_features & (1UL << __SCHED_FEAT_##x)) 2094 2095 #endif /* SCHED_DEBUG */ 2096 2097 extern struct static_key_false sched_numa_balancing; 2098 extern struct static_key_false sched_schedstats; 2099 2100 static inline u64 global_rt_period(void) 2101 { 2102 return (u64)sysctl_sched_rt_period * NSEC_PER_USEC; 2103 } 2104 2105 static inline u64 global_rt_runtime(void) 2106 { 2107 if (sysctl_sched_rt_runtime < 0) 2108 return RUNTIME_INF; 2109 2110 return (u64)sysctl_sched_rt_runtime * NSEC_PER_USEC; 2111 } 2112 2113 static inline int task_current(struct rq *rq, struct task_struct *p) 2114 { 2115 return rq->curr == p; 2116 } 2117 2118 static inline int task_on_cpu(struct rq *rq, struct task_struct *p) 2119 { 2120 #ifdef CONFIG_SMP 2121 return p->on_cpu; 2122 #else 2123 return task_current(rq, p); 2124 #endif 2125 } 2126 2127 static inline int task_on_rq_queued(struct task_struct *p) 2128 { 2129 return p->on_rq == TASK_ON_RQ_QUEUED; 2130 } 2131 2132 static inline int task_on_rq_migrating(struct task_struct *p) 2133 { 2134 return READ_ONCE(p->on_rq) == TASK_ON_RQ_MIGRATING; 2135 } 2136 2137 /* Wake flags. The first three directly map to some SD flag value */ 2138 #define WF_EXEC 0x02 /* Wakeup after exec; maps to SD_BALANCE_EXEC */ 2139 #define WF_FORK 0x04 /* Wakeup after fork; maps to SD_BALANCE_FORK */ 2140 #define WF_TTWU 0x08 /* Wakeup; maps to SD_BALANCE_WAKE */ 2141 2142 #define WF_SYNC 0x10 /* Waker goes to sleep after wakeup */ 2143 #define WF_MIGRATED 0x20 /* Internal use, task got migrated */ 2144 2145 #ifdef CONFIG_SMP 2146 static_assert(WF_EXEC == SD_BALANCE_EXEC); 2147 static_assert(WF_FORK == SD_BALANCE_FORK); 2148 static_assert(WF_TTWU == SD_BALANCE_WAKE); 2149 #endif 2150 2151 /* 2152 * To aid in avoiding the subversion of "niceness" due to uneven distribution 2153 * of tasks with abnormal "nice" values across CPUs the contribution that 2154 * each task makes to its run queue's load is weighted according to its 2155 * scheduling class and "nice" value. For SCHED_NORMAL tasks this is just a 2156 * scaled version of the new time slice allocation that they receive on time 2157 * slice expiry etc. 2158 */ 2159 2160 #define WEIGHT_IDLEPRIO 3 2161 #define WMULT_IDLEPRIO 1431655765 2162 2163 extern const int sched_prio_to_weight[40]; 2164 extern const u32 sched_prio_to_wmult[40]; 2165 2166 /* 2167 * {de,en}queue flags: 2168 * 2169 * DEQUEUE_SLEEP - task is no longer runnable 2170 * ENQUEUE_WAKEUP - task just became runnable 2171 * 2172 * SAVE/RESTORE - an otherwise spurious dequeue/enqueue, done to ensure tasks 2173 * are in a known state which allows modification. Such pairs 2174 * should preserve as much state as possible. 2175 * 2176 * MOVE - paired with SAVE/RESTORE, explicitly does not preserve the location 2177 * in the runqueue. 2178 * 2179 * ENQUEUE_HEAD - place at front of runqueue (tail if not specified) 2180 * ENQUEUE_REPLENISH - CBS (replenish runtime and postpone deadline) 2181 * ENQUEUE_MIGRATED - the task was migrated during wakeup 2182 * 2183 */ 2184 2185 #define DEQUEUE_SLEEP 0x01 2186 #define DEQUEUE_SAVE 0x02 /* Matches ENQUEUE_RESTORE */ 2187 #define DEQUEUE_MOVE 0x04 /* Matches ENQUEUE_MOVE */ 2188 #define DEQUEUE_NOCLOCK 0x08 /* Matches ENQUEUE_NOCLOCK */ 2189 2190 #define ENQUEUE_WAKEUP 0x01 2191 #define ENQUEUE_RESTORE 0x02 2192 #define ENQUEUE_MOVE 0x04 2193 #define ENQUEUE_NOCLOCK 0x08 2194 2195 #define ENQUEUE_HEAD 0x10 2196 #define ENQUEUE_REPLENISH 0x20 2197 #ifdef CONFIG_SMP 2198 #define ENQUEUE_MIGRATED 0x40 2199 #else 2200 #define ENQUEUE_MIGRATED 0x00 2201 #endif 2202 #define ENQUEUE_INITIAL 0x80 2203 2204 #define RETRY_TASK ((void *)-1UL) 2205 2206 struct affinity_context { 2207 const struct cpumask *new_mask; 2208 struct cpumask *user_mask; 2209 unsigned int flags; 2210 }; 2211 2212 struct sched_class { 2213 2214 #ifdef CONFIG_UCLAMP_TASK 2215 int uclamp_enabled; 2216 #endif 2217 2218 void (*enqueue_task) (struct rq *rq, struct task_struct *p, int flags); 2219 void (*dequeue_task) (struct rq *rq, struct task_struct *p, int flags); 2220 void (*yield_task) (struct rq *rq); 2221 bool (*yield_to_task)(struct rq *rq, struct task_struct *p); 2222 2223 void (*check_preempt_curr)(struct rq *rq, struct task_struct *p, int flags); 2224 2225 struct task_struct *(*pick_next_task)(struct rq *rq); 2226 2227 void (*put_prev_task)(struct rq *rq, struct task_struct *p); 2228 void (*set_next_task)(struct rq *rq, struct task_struct *p, bool first); 2229 2230 #ifdef CONFIG_SMP 2231 int (*balance)(struct rq *rq, struct task_struct *prev, struct rq_flags *rf); 2232 int (*select_task_rq)(struct task_struct *p, int task_cpu, int flags); 2233 2234 struct task_struct * (*pick_task)(struct rq *rq); 2235 2236 void (*migrate_task_rq)(struct task_struct *p, int new_cpu); 2237 2238 void (*task_woken)(struct rq *this_rq, struct task_struct *task); 2239 2240 void (*set_cpus_allowed)(struct task_struct *p, struct affinity_context *ctx); 2241 2242 void (*rq_online)(struct rq *rq); 2243 void (*rq_offline)(struct rq *rq); 2244 2245 struct rq *(*find_lock_rq)(struct task_struct *p, struct rq *rq); 2246 #endif 2247 2248 void (*task_tick)(struct rq *rq, struct task_struct *p, int queued); 2249 void (*task_fork)(struct task_struct *p); 2250 void (*task_dead)(struct task_struct *p); 2251 2252 /* 2253 * The switched_from() call is allowed to drop rq->lock, therefore we 2254 * cannot assume the switched_from/switched_to pair is serialized by 2255 * rq->lock. They are however serialized by p->pi_lock. 2256 */ 2257 void (*switched_from)(struct rq *this_rq, struct task_struct *task); 2258 void (*switched_to) (struct rq *this_rq, struct task_struct *task); 2259 void (*prio_changed) (struct rq *this_rq, struct task_struct *task, 2260 int oldprio); 2261 2262 unsigned int (*get_rr_interval)(struct rq *rq, 2263 struct task_struct *task); 2264 2265 void (*update_curr)(struct rq *rq); 2266 2267 #ifdef CONFIG_FAIR_GROUP_SCHED 2268 void (*task_change_group)(struct task_struct *p); 2269 #endif 2270 2271 #ifdef CONFIG_SCHED_CORE 2272 int (*task_is_throttled)(struct task_struct *p, int cpu); 2273 #endif 2274 }; 2275 2276 static inline void put_prev_task(struct rq *rq, struct task_struct *prev) 2277 { 2278 WARN_ON_ONCE(rq->curr != prev); 2279 prev->sched_class->put_prev_task(rq, prev); 2280 } 2281 2282 static inline void set_next_task(struct rq *rq, struct task_struct *next) 2283 { 2284 next->sched_class->set_next_task(rq, next, false); 2285 } 2286 2287 2288 /* 2289 * Helper to define a sched_class instance; each one is placed in a separate 2290 * section which is ordered by the linker script: 2291 * 2292 * include/asm-generic/vmlinux.lds.h 2293 * 2294 * *CAREFUL* they are laid out in *REVERSE* order!!! 2295 * 2296 * Also enforce alignment on the instance, not the type, to guarantee layout. 2297 */ 2298 #define DEFINE_SCHED_CLASS(name) \ 2299 const struct sched_class name##_sched_class \ 2300 __aligned(__alignof__(struct sched_class)) \ 2301 __section("__" #name "_sched_class") 2302 2303 /* Defined in include/asm-generic/vmlinux.lds.h */ 2304 extern struct sched_class __sched_class_highest[]; 2305 extern struct sched_class __sched_class_lowest[]; 2306 2307 #define for_class_range(class, _from, _to) \ 2308 for (class = (_from); class < (_to); class++) 2309 2310 #define for_each_class(class) \ 2311 for_class_range(class, __sched_class_highest, __sched_class_lowest) 2312 2313 #define sched_class_above(_a, _b) ((_a) < (_b)) 2314 2315 extern const struct sched_class stop_sched_class; 2316 extern const struct sched_class dl_sched_class; 2317 extern const struct sched_class rt_sched_class; 2318 extern const struct sched_class fair_sched_class; 2319 extern const struct sched_class idle_sched_class; 2320 2321 static inline bool sched_stop_runnable(struct rq *rq) 2322 { 2323 return rq->stop && task_on_rq_queued(rq->stop); 2324 } 2325 2326 static inline bool sched_dl_runnable(struct rq *rq) 2327 { 2328 return rq->dl.dl_nr_running > 0; 2329 } 2330 2331 static inline bool sched_rt_runnable(struct rq *rq) 2332 { 2333 return rq->rt.rt_queued > 0; 2334 } 2335 2336 static inline bool sched_fair_runnable(struct rq *rq) 2337 { 2338 return rq->cfs.nr_running > 0; 2339 } 2340 2341 extern struct task_struct *pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf); 2342 extern struct task_struct *pick_next_task_idle(struct rq *rq); 2343 2344 #define SCA_CHECK 0x01 2345 #define SCA_MIGRATE_DISABLE 0x02 2346 #define SCA_MIGRATE_ENABLE 0x04 2347 #define SCA_USER 0x08 2348 2349 #ifdef CONFIG_SMP 2350 2351 extern void update_group_capacity(struct sched_domain *sd, int cpu); 2352 2353 extern void trigger_load_balance(struct rq *rq); 2354 2355 extern void set_cpus_allowed_common(struct task_struct *p, struct affinity_context *ctx); 2356 2357 static inline struct task_struct *get_push_task(struct rq *rq) 2358 { 2359 struct task_struct *p = rq->curr; 2360 2361 lockdep_assert_rq_held(rq); 2362 2363 if (rq->push_busy) 2364 return NULL; 2365 2366 if (p->nr_cpus_allowed == 1) 2367 return NULL; 2368 2369 if (p->migration_disabled) 2370 return NULL; 2371 2372 rq->push_busy = true; 2373 return get_task_struct(p); 2374 } 2375 2376 extern int push_cpu_stop(void *arg); 2377 2378 #endif 2379 2380 #ifdef CONFIG_CPU_IDLE 2381 static inline void idle_set_state(struct rq *rq, 2382 struct cpuidle_state *idle_state) 2383 { 2384 rq->idle_state = idle_state; 2385 } 2386 2387 static inline struct cpuidle_state *idle_get_state(struct rq *rq) 2388 { 2389 SCHED_WARN_ON(!rcu_read_lock_held()); 2390 2391 return rq->idle_state; 2392 } 2393 #else 2394 static inline void idle_set_state(struct rq *rq, 2395 struct cpuidle_state *idle_state) 2396 { 2397 } 2398 2399 static inline struct cpuidle_state *idle_get_state(struct rq *rq) 2400 { 2401 return NULL; 2402 } 2403 #endif 2404 2405 extern void schedule_idle(void); 2406 2407 extern void sysrq_sched_debug_show(void); 2408 extern void sched_init_granularity(void); 2409 extern void update_max_interval(void); 2410 2411 extern void init_sched_dl_class(void); 2412 extern void init_sched_rt_class(void); 2413 extern void init_sched_fair_class(void); 2414 2415 extern void reweight_task(struct task_struct *p, int prio); 2416 2417 extern void resched_curr(struct rq *rq); 2418 extern void resched_cpu(int cpu); 2419 2420 extern struct rt_bandwidth def_rt_bandwidth; 2421 extern void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime); 2422 extern bool sched_rt_bandwidth_account(struct rt_rq *rt_rq); 2423 2424 extern void init_dl_task_timer(struct sched_dl_entity *dl_se); 2425 extern void init_dl_inactive_task_timer(struct sched_dl_entity *dl_se); 2426 2427 #define BW_SHIFT 20 2428 #define BW_UNIT (1 << BW_SHIFT) 2429 #define RATIO_SHIFT 8 2430 #define MAX_BW_BITS (64 - BW_SHIFT) 2431 #define MAX_BW ((1ULL << MAX_BW_BITS) - 1) 2432 unsigned long to_ratio(u64 period, u64 runtime); 2433 2434 extern void init_entity_runnable_average(struct sched_entity *se); 2435 extern void post_init_entity_util_avg(struct task_struct *p); 2436 2437 #ifdef CONFIG_NO_HZ_FULL 2438 extern bool sched_can_stop_tick(struct rq *rq); 2439 extern int __init sched_tick_offload_init(void); 2440 2441 /* 2442 * Tick may be needed by tasks in the runqueue depending on their policy and 2443 * requirements. If tick is needed, lets send the target an IPI to kick it out of 2444 * nohz mode if necessary. 2445 */ 2446 static inline void sched_update_tick_dependency(struct rq *rq) 2447 { 2448 int cpu = cpu_of(rq); 2449 2450 if (!tick_nohz_full_cpu(cpu)) 2451 return; 2452 2453 if (sched_can_stop_tick(rq)) 2454 tick_nohz_dep_clear_cpu(cpu, TICK_DEP_BIT_SCHED); 2455 else 2456 tick_nohz_dep_set_cpu(cpu, TICK_DEP_BIT_SCHED); 2457 } 2458 #else 2459 static inline int sched_tick_offload_init(void) { return 0; } 2460 static inline void sched_update_tick_dependency(struct rq *rq) { } 2461 #endif 2462 2463 static inline void add_nr_running(struct rq *rq, unsigned count) 2464 { 2465 unsigned prev_nr = rq->nr_running; 2466 2467 rq->nr_running = prev_nr + count; 2468 if (trace_sched_update_nr_running_tp_enabled()) { 2469 call_trace_sched_update_nr_running(rq, count); 2470 } 2471 2472 #ifdef CONFIG_SMP 2473 if (prev_nr < 2 && rq->nr_running >= 2) { 2474 if (!READ_ONCE(rq->rd->overload)) 2475 WRITE_ONCE(rq->rd->overload, 1); 2476 } 2477 #endif 2478 2479 sched_update_tick_dependency(rq); 2480 } 2481 2482 static inline void sub_nr_running(struct rq *rq, unsigned count) 2483 { 2484 rq->nr_running -= count; 2485 if (trace_sched_update_nr_running_tp_enabled()) { 2486 call_trace_sched_update_nr_running(rq, -count); 2487 } 2488 2489 /* Check if we still need preemption */ 2490 sched_update_tick_dependency(rq); 2491 } 2492 2493 extern void activate_task(struct rq *rq, struct task_struct *p, int flags); 2494 extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags); 2495 2496 extern void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags); 2497 2498 #ifdef CONFIG_PREEMPT_RT 2499 #define SCHED_NR_MIGRATE_BREAK 8 2500 #else 2501 #define SCHED_NR_MIGRATE_BREAK 32 2502 #endif 2503 2504 extern const_debug unsigned int sysctl_sched_nr_migrate; 2505 extern const_debug unsigned int sysctl_sched_migration_cost; 2506 2507 extern unsigned int sysctl_sched_base_slice; 2508 2509 #ifdef CONFIG_SCHED_DEBUG 2510 extern int sysctl_resched_latency_warn_ms; 2511 extern int sysctl_resched_latency_warn_once; 2512 2513 extern unsigned int sysctl_sched_tunable_scaling; 2514 2515 extern unsigned int sysctl_numa_balancing_scan_delay; 2516 extern unsigned int sysctl_numa_balancing_scan_period_min; 2517 extern unsigned int sysctl_numa_balancing_scan_period_max; 2518 extern unsigned int sysctl_numa_balancing_scan_size; 2519 extern unsigned int sysctl_numa_balancing_hot_threshold; 2520 #endif 2521 2522 #ifdef CONFIG_SCHED_HRTICK 2523 2524 /* 2525 * Use hrtick when: 2526 * - enabled by features 2527 * - hrtimer is actually high res 2528 */ 2529 static inline int hrtick_enabled(struct rq *rq) 2530 { 2531 if (!cpu_active(cpu_of(rq))) 2532 return 0; 2533 return hrtimer_is_hres_active(&rq->hrtick_timer); 2534 } 2535 2536 static inline int hrtick_enabled_fair(struct rq *rq) 2537 { 2538 if (!sched_feat(HRTICK)) 2539 return 0; 2540 return hrtick_enabled(rq); 2541 } 2542 2543 static inline int hrtick_enabled_dl(struct rq *rq) 2544 { 2545 if (!sched_feat(HRTICK_DL)) 2546 return 0; 2547 return hrtick_enabled(rq); 2548 } 2549 2550 void hrtick_start(struct rq *rq, u64 delay); 2551 2552 #else 2553 2554 static inline int hrtick_enabled_fair(struct rq *rq) 2555 { 2556 return 0; 2557 } 2558 2559 static inline int hrtick_enabled_dl(struct rq *rq) 2560 { 2561 return 0; 2562 } 2563 2564 static inline int hrtick_enabled(struct rq *rq) 2565 { 2566 return 0; 2567 } 2568 2569 #endif /* CONFIG_SCHED_HRTICK */ 2570 2571 #ifndef arch_scale_freq_tick 2572 static __always_inline 2573 void arch_scale_freq_tick(void) 2574 { 2575 } 2576 #endif 2577 2578 #ifndef arch_scale_freq_capacity 2579 /** 2580 * arch_scale_freq_capacity - get the frequency scale factor of a given CPU. 2581 * @cpu: the CPU in question. 2582 * 2583 * Return: the frequency scale factor normalized against SCHED_CAPACITY_SCALE, i.e. 2584 * 2585 * f_curr 2586 * ------ * SCHED_CAPACITY_SCALE 2587 * f_max 2588 */ 2589 static __always_inline 2590 unsigned long arch_scale_freq_capacity(int cpu) 2591 { 2592 return SCHED_CAPACITY_SCALE; 2593 } 2594 #endif 2595 2596 #ifdef CONFIG_SCHED_DEBUG 2597 /* 2598 * In double_lock_balance()/double_rq_lock(), we use raw_spin_rq_lock() to 2599 * acquire rq lock instead of rq_lock(). So at the end of these two functions 2600 * we need to call double_rq_clock_clear_update() to clear RQCF_UPDATED of 2601 * rq->clock_update_flags to avoid the WARN_DOUBLE_CLOCK warning. 2602 */ 2603 static inline void double_rq_clock_clear_update(struct rq *rq1, struct rq *rq2) 2604 { 2605 rq1->clock_update_flags &= (RQCF_REQ_SKIP|RQCF_ACT_SKIP); 2606 /* rq1 == rq2 for !CONFIG_SMP, so just clear RQCF_UPDATED once. */ 2607 #ifdef CONFIG_SMP 2608 rq2->clock_update_flags &= (RQCF_REQ_SKIP|RQCF_ACT_SKIP); 2609 #endif 2610 } 2611 #else 2612 static inline void double_rq_clock_clear_update(struct rq *rq1, struct rq *rq2) {} 2613 #endif 2614 2615 #ifdef CONFIG_SMP 2616 2617 static inline bool rq_order_less(struct rq *rq1, struct rq *rq2) 2618 { 2619 #ifdef CONFIG_SCHED_CORE 2620 /* 2621 * In order to not have {0,2},{1,3} turn into into an AB-BA, 2622 * order by core-id first and cpu-id second. 2623 * 2624 * Notably: 2625 * 2626 * double_rq_lock(0,3); will take core-0, core-1 lock 2627 * double_rq_lock(1,2); will take core-1, core-0 lock 2628 * 2629 * when only cpu-id is considered. 2630 */ 2631 if (rq1->core->cpu < rq2->core->cpu) 2632 return true; 2633 if (rq1->core->cpu > rq2->core->cpu) 2634 return false; 2635 2636 /* 2637 * __sched_core_flip() relies on SMT having cpu-id lock order. 2638 */ 2639 #endif 2640 return rq1->cpu < rq2->cpu; 2641 } 2642 2643 extern void double_rq_lock(struct rq *rq1, struct rq *rq2); 2644 2645 #ifdef CONFIG_PREEMPTION 2646 2647 /* 2648 * fair double_lock_balance: Safely acquires both rq->locks in a fair 2649 * way at the expense of forcing extra atomic operations in all 2650 * invocations. This assures that the double_lock is acquired using the 2651 * same underlying policy as the spinlock_t on this architecture, which 2652 * reduces latency compared to the unfair variant below. However, it 2653 * also adds more overhead and therefore may reduce throughput. 2654 */ 2655 static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest) 2656 __releases(this_rq->lock) 2657 __acquires(busiest->lock) 2658 __acquires(this_rq->lock) 2659 { 2660 raw_spin_rq_unlock(this_rq); 2661 double_rq_lock(this_rq, busiest); 2662 2663 return 1; 2664 } 2665 2666 #else 2667 /* 2668 * Unfair double_lock_balance: Optimizes throughput at the expense of 2669 * latency by eliminating extra atomic operations when the locks are 2670 * already in proper order on entry. This favors lower CPU-ids and will 2671 * grant the double lock to lower CPUs over higher ids under contention, 2672 * regardless of entry order into the function. 2673 */ 2674 static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest) 2675 __releases(this_rq->lock) 2676 __acquires(busiest->lock) 2677 __acquires(this_rq->lock) 2678 { 2679 if (__rq_lockp(this_rq) == __rq_lockp(busiest) || 2680 likely(raw_spin_rq_trylock(busiest))) { 2681 double_rq_clock_clear_update(this_rq, busiest); 2682 return 0; 2683 } 2684 2685 if (rq_order_less(this_rq, busiest)) { 2686 raw_spin_rq_lock_nested(busiest, SINGLE_DEPTH_NESTING); 2687 double_rq_clock_clear_update(this_rq, busiest); 2688 return 0; 2689 } 2690 2691 raw_spin_rq_unlock(this_rq); 2692 double_rq_lock(this_rq, busiest); 2693 2694 return 1; 2695 } 2696 2697 #endif /* CONFIG_PREEMPTION */ 2698 2699 /* 2700 * double_lock_balance - lock the busiest runqueue, this_rq is locked already. 2701 */ 2702 static inline int double_lock_balance(struct rq *this_rq, struct rq *busiest) 2703 { 2704 lockdep_assert_irqs_disabled(); 2705 2706 return _double_lock_balance(this_rq, busiest); 2707 } 2708 2709 static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest) 2710 __releases(busiest->lock) 2711 { 2712 if (__rq_lockp(this_rq) != __rq_lockp(busiest)) 2713 raw_spin_rq_unlock(busiest); 2714 lock_set_subclass(&__rq_lockp(this_rq)->dep_map, 0, _RET_IP_); 2715 } 2716 2717 static inline void double_lock(spinlock_t *l1, spinlock_t *l2) 2718 { 2719 if (l1 > l2) 2720 swap(l1, l2); 2721 2722 spin_lock(l1); 2723 spin_lock_nested(l2, SINGLE_DEPTH_NESTING); 2724 } 2725 2726 static inline void double_lock_irq(spinlock_t *l1, spinlock_t *l2) 2727 { 2728 if (l1 > l2) 2729 swap(l1, l2); 2730 2731 spin_lock_irq(l1); 2732 spin_lock_nested(l2, SINGLE_DEPTH_NESTING); 2733 } 2734 2735 static inline void double_raw_lock(raw_spinlock_t *l1, raw_spinlock_t *l2) 2736 { 2737 if (l1 > l2) 2738 swap(l1, l2); 2739 2740 raw_spin_lock(l1); 2741 raw_spin_lock_nested(l2, SINGLE_DEPTH_NESTING); 2742 } 2743 2744 /* 2745 * double_rq_unlock - safely unlock two runqueues 2746 * 2747 * Note this does not restore interrupts like task_rq_unlock, 2748 * you need to do so manually after calling. 2749 */ 2750 static inline void double_rq_unlock(struct rq *rq1, struct rq *rq2) 2751 __releases(rq1->lock) 2752 __releases(rq2->lock) 2753 { 2754 if (__rq_lockp(rq1) != __rq_lockp(rq2)) 2755 raw_spin_rq_unlock(rq2); 2756 else 2757 __release(rq2->lock); 2758 raw_spin_rq_unlock(rq1); 2759 } 2760 2761 extern void set_rq_online (struct rq *rq); 2762 extern void set_rq_offline(struct rq *rq); 2763 extern bool sched_smp_initialized; 2764 2765 #else /* CONFIG_SMP */ 2766 2767 /* 2768 * double_rq_lock - safely lock two runqueues 2769 * 2770 * Note this does not disable interrupts like task_rq_lock, 2771 * you need to do so manually before calling. 2772 */ 2773 static inline void double_rq_lock(struct rq *rq1, struct rq *rq2) 2774 __acquires(rq1->lock) 2775 __acquires(rq2->lock) 2776 { 2777 WARN_ON_ONCE(!irqs_disabled()); 2778 WARN_ON_ONCE(rq1 != rq2); 2779 raw_spin_rq_lock(rq1); 2780 __acquire(rq2->lock); /* Fake it out ;) */ 2781 double_rq_clock_clear_update(rq1, rq2); 2782 } 2783 2784 /* 2785 * double_rq_unlock - safely unlock two runqueues 2786 * 2787 * Note this does not restore interrupts like task_rq_unlock, 2788 * you need to do so manually after calling. 2789 */ 2790 static inline void double_rq_unlock(struct rq *rq1, struct rq *rq2) 2791 __releases(rq1->lock) 2792 __releases(rq2->lock) 2793 { 2794 WARN_ON_ONCE(rq1 != rq2); 2795 raw_spin_rq_unlock(rq1); 2796 __release(rq2->lock); 2797 } 2798 2799 #endif 2800 2801 extern struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq); 2802 extern struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq); 2803 2804 #ifdef CONFIG_SCHED_DEBUG 2805 extern bool sched_debug_verbose; 2806 2807 extern void print_cfs_stats(struct seq_file *m, int cpu); 2808 extern void print_rt_stats(struct seq_file *m, int cpu); 2809 extern void print_dl_stats(struct seq_file *m, int cpu); 2810 extern void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq); 2811 extern void print_rt_rq(struct seq_file *m, int cpu, struct rt_rq *rt_rq); 2812 extern void print_dl_rq(struct seq_file *m, int cpu, struct dl_rq *dl_rq); 2813 2814 extern void resched_latency_warn(int cpu, u64 latency); 2815 #ifdef CONFIG_NUMA_BALANCING 2816 extern void 2817 show_numa_stats(struct task_struct *p, struct seq_file *m); 2818 extern void 2819 print_numa_stats(struct seq_file *m, int node, unsigned long tsf, 2820 unsigned long tpf, unsigned long gsf, unsigned long gpf); 2821 #endif /* CONFIG_NUMA_BALANCING */ 2822 #else 2823 static inline void resched_latency_warn(int cpu, u64 latency) {} 2824 #endif /* CONFIG_SCHED_DEBUG */ 2825 2826 extern void init_cfs_rq(struct cfs_rq *cfs_rq); 2827 extern void init_rt_rq(struct rt_rq *rt_rq); 2828 extern void init_dl_rq(struct dl_rq *dl_rq); 2829 2830 extern void cfs_bandwidth_usage_inc(void); 2831 extern void cfs_bandwidth_usage_dec(void); 2832 2833 #ifdef CONFIG_NO_HZ_COMMON 2834 #define NOHZ_BALANCE_KICK_BIT 0 2835 #define NOHZ_STATS_KICK_BIT 1 2836 #define NOHZ_NEWILB_KICK_BIT 2 2837 #define NOHZ_NEXT_KICK_BIT 3 2838 2839 /* Run rebalance_domains() */ 2840 #define NOHZ_BALANCE_KICK BIT(NOHZ_BALANCE_KICK_BIT) 2841 /* Update blocked load */ 2842 #define NOHZ_STATS_KICK BIT(NOHZ_STATS_KICK_BIT) 2843 /* Update blocked load when entering idle */ 2844 #define NOHZ_NEWILB_KICK BIT(NOHZ_NEWILB_KICK_BIT) 2845 /* Update nohz.next_balance */ 2846 #define NOHZ_NEXT_KICK BIT(NOHZ_NEXT_KICK_BIT) 2847 2848 #define NOHZ_KICK_MASK (NOHZ_BALANCE_KICK | NOHZ_STATS_KICK | NOHZ_NEXT_KICK) 2849 2850 #define nohz_flags(cpu) (&cpu_rq(cpu)->nohz_flags) 2851 2852 extern void nohz_balance_exit_idle(struct rq *rq); 2853 #else 2854 static inline void nohz_balance_exit_idle(struct rq *rq) { } 2855 #endif 2856 2857 #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON) 2858 extern void nohz_run_idle_balance(int cpu); 2859 #else 2860 static inline void nohz_run_idle_balance(int cpu) { } 2861 #endif 2862 2863 #ifdef CONFIG_IRQ_TIME_ACCOUNTING 2864 struct irqtime { 2865 u64 total; 2866 u64 tick_delta; 2867 u64 irq_start_time; 2868 struct u64_stats_sync sync; 2869 }; 2870 2871 DECLARE_PER_CPU(struct irqtime, cpu_irqtime); 2872 2873 /* 2874 * Returns the irqtime minus the softirq time computed by ksoftirqd. 2875 * Otherwise ksoftirqd's sum_exec_runtime is subtracted its own runtime 2876 * and never move forward. 2877 */ 2878 static inline u64 irq_time_read(int cpu) 2879 { 2880 struct irqtime *irqtime = &per_cpu(cpu_irqtime, cpu); 2881 unsigned int seq; 2882 u64 total; 2883 2884 do { 2885 seq = __u64_stats_fetch_begin(&irqtime->sync); 2886 total = irqtime->total; 2887 } while (__u64_stats_fetch_retry(&irqtime->sync, seq)); 2888 2889 return total; 2890 } 2891 #endif /* CONFIG_IRQ_TIME_ACCOUNTING */ 2892 2893 #ifdef CONFIG_CPU_FREQ 2894 DECLARE_PER_CPU(struct update_util_data __rcu *, cpufreq_update_util_data); 2895 2896 /** 2897 * cpufreq_update_util - Take a note about CPU utilization changes. 2898 * @rq: Runqueue to carry out the update for. 2899 * @flags: Update reason flags. 2900 * 2901 * This function is called by the scheduler on the CPU whose utilization is 2902 * being updated. 2903 * 2904 * It can only be called from RCU-sched read-side critical sections. 2905 * 2906 * The way cpufreq is currently arranged requires it to evaluate the CPU 2907 * performance state (frequency/voltage) on a regular basis to prevent it from 2908 * being stuck in a completely inadequate performance level for too long. 2909 * That is not guaranteed to happen if the updates are only triggered from CFS 2910 * and DL, though, because they may not be coming in if only RT tasks are 2911 * active all the time (or there are RT tasks only). 2912 * 2913 * As a workaround for that issue, this function is called periodically by the 2914 * RT sched class to trigger extra cpufreq updates to prevent it from stalling, 2915 * but that really is a band-aid. Going forward it should be replaced with 2916 * solutions targeted more specifically at RT tasks. 2917 */ 2918 static inline void cpufreq_update_util(struct rq *rq, unsigned int flags) 2919 { 2920 struct update_util_data *data; 2921 2922 data = rcu_dereference_sched(*per_cpu_ptr(&cpufreq_update_util_data, 2923 cpu_of(rq))); 2924 if (data) 2925 data->func(data, rq_clock(rq), flags); 2926 } 2927 #else 2928 static inline void cpufreq_update_util(struct rq *rq, unsigned int flags) {} 2929 #endif /* CONFIG_CPU_FREQ */ 2930 2931 #ifdef arch_scale_freq_capacity 2932 # ifndef arch_scale_freq_invariant 2933 # define arch_scale_freq_invariant() true 2934 # endif 2935 #else 2936 # define arch_scale_freq_invariant() false 2937 #endif 2938 2939 #ifdef CONFIG_SMP 2940 static inline unsigned long capacity_orig_of(int cpu) 2941 { 2942 return cpu_rq(cpu)->cpu_capacity_orig; 2943 } 2944 2945 /** 2946 * enum cpu_util_type - CPU utilization type 2947 * @FREQUENCY_UTIL: Utilization used to select frequency 2948 * @ENERGY_UTIL: Utilization used during energy calculation 2949 * 2950 * The utilization signals of all scheduling classes (CFS/RT/DL) and IRQ time 2951 * need to be aggregated differently depending on the usage made of them. This 2952 * enum is used within effective_cpu_util() to differentiate the types of 2953 * utilization expected by the callers, and adjust the aggregation accordingly. 2954 */ 2955 enum cpu_util_type { 2956 FREQUENCY_UTIL, 2957 ENERGY_UTIL, 2958 }; 2959 2960 unsigned long effective_cpu_util(int cpu, unsigned long util_cfs, 2961 enum cpu_util_type type, 2962 struct task_struct *p); 2963 2964 /* 2965 * Verify the fitness of task @p to run on @cpu taking into account the 2966 * CPU original capacity and the runtime/deadline ratio of the task. 2967 * 2968 * The function will return true if the original capacity of @cpu is 2969 * greater than or equal to task's deadline density right shifted by 2970 * (BW_SHIFT - SCHED_CAPACITY_SHIFT) and false otherwise. 2971 */ 2972 static inline bool dl_task_fits_capacity(struct task_struct *p, int cpu) 2973 { 2974 unsigned long cap = arch_scale_cpu_capacity(cpu); 2975 2976 return cap >= p->dl.dl_density >> (BW_SHIFT - SCHED_CAPACITY_SHIFT); 2977 } 2978 2979 static inline unsigned long cpu_bw_dl(struct rq *rq) 2980 { 2981 return (rq->dl.running_bw * SCHED_CAPACITY_SCALE) >> BW_SHIFT; 2982 } 2983 2984 static inline unsigned long cpu_util_dl(struct rq *rq) 2985 { 2986 return READ_ONCE(rq->avg_dl.util_avg); 2987 } 2988 2989 2990 extern unsigned long cpu_util_cfs(int cpu); 2991 extern unsigned long cpu_util_cfs_boost(int cpu); 2992 2993 static inline unsigned long cpu_util_rt(struct rq *rq) 2994 { 2995 return READ_ONCE(rq->avg_rt.util_avg); 2996 } 2997 #endif 2998 2999 #ifdef CONFIG_UCLAMP_TASK 3000 unsigned long uclamp_eff_value(struct task_struct *p, enum uclamp_id clamp_id); 3001 3002 static inline unsigned long uclamp_rq_get(struct rq *rq, 3003 enum uclamp_id clamp_id) 3004 { 3005 return READ_ONCE(rq->uclamp[clamp_id].value); 3006 } 3007 3008 static inline void uclamp_rq_set(struct rq *rq, enum uclamp_id clamp_id, 3009 unsigned int value) 3010 { 3011 WRITE_ONCE(rq->uclamp[clamp_id].value, value); 3012 } 3013 3014 static inline bool uclamp_rq_is_idle(struct rq *rq) 3015 { 3016 return rq->uclamp_flags & UCLAMP_FLAG_IDLE; 3017 } 3018 3019 /** 3020 * uclamp_rq_util_with - clamp @util with @rq and @p effective uclamp values. 3021 * @rq: The rq to clamp against. Must not be NULL. 3022 * @util: The util value to clamp. 3023 * @p: The task to clamp against. Can be NULL if you want to clamp 3024 * against @rq only. 3025 * 3026 * Clamps the passed @util to the max(@rq, @p) effective uclamp values. 3027 * 3028 * If sched_uclamp_used static key is disabled, then just return the util 3029 * without any clamping since uclamp aggregation at the rq level in the fast 3030 * path is disabled, rendering this operation a NOP. 3031 * 3032 * Use uclamp_eff_value() if you don't care about uclamp values at rq level. It 3033 * will return the correct effective uclamp value of the task even if the 3034 * static key is disabled. 3035 */ 3036 static __always_inline 3037 unsigned long uclamp_rq_util_with(struct rq *rq, unsigned long util, 3038 struct task_struct *p) 3039 { 3040 unsigned long min_util = 0; 3041 unsigned long max_util = 0; 3042 3043 if (!static_branch_likely(&sched_uclamp_used)) 3044 return util; 3045 3046 if (p) { 3047 min_util = uclamp_eff_value(p, UCLAMP_MIN); 3048 max_util = uclamp_eff_value(p, UCLAMP_MAX); 3049 3050 /* 3051 * Ignore last runnable task's max clamp, as this task will 3052 * reset it. Similarly, no need to read the rq's min clamp. 3053 */ 3054 if (uclamp_rq_is_idle(rq)) 3055 goto out; 3056 } 3057 3058 min_util = max_t(unsigned long, min_util, uclamp_rq_get(rq, UCLAMP_MIN)); 3059 max_util = max_t(unsigned long, max_util, uclamp_rq_get(rq, UCLAMP_MAX)); 3060 out: 3061 /* 3062 * Since CPU's {min,max}_util clamps are MAX aggregated considering 3063 * RUNNABLE tasks with _different_ clamps, we can end up with an 3064 * inversion. Fix it now when the clamps are applied. 3065 */ 3066 if (unlikely(min_util >= max_util)) 3067 return min_util; 3068 3069 return clamp(util, min_util, max_util); 3070 } 3071 3072 /* Is the rq being capped/throttled by uclamp_max? */ 3073 static inline bool uclamp_rq_is_capped(struct rq *rq) 3074 { 3075 unsigned long rq_util; 3076 unsigned long max_util; 3077 3078 if (!static_branch_likely(&sched_uclamp_used)) 3079 return false; 3080 3081 rq_util = cpu_util_cfs(cpu_of(rq)) + cpu_util_rt(rq); 3082 max_util = READ_ONCE(rq->uclamp[UCLAMP_MAX].value); 3083 3084 return max_util != SCHED_CAPACITY_SCALE && rq_util >= max_util; 3085 } 3086 3087 /* 3088 * When uclamp is compiled in, the aggregation at rq level is 'turned off' 3089 * by default in the fast path and only gets turned on once userspace performs 3090 * an operation that requires it. 3091 * 3092 * Returns true if userspace opted-in to use uclamp and aggregation at rq level 3093 * hence is active. 3094 */ 3095 static inline bool uclamp_is_used(void) 3096 { 3097 return static_branch_likely(&sched_uclamp_used); 3098 } 3099 #else /* CONFIG_UCLAMP_TASK */ 3100 static inline unsigned long uclamp_eff_value(struct task_struct *p, 3101 enum uclamp_id clamp_id) 3102 { 3103 if (clamp_id == UCLAMP_MIN) 3104 return 0; 3105 3106 return SCHED_CAPACITY_SCALE; 3107 } 3108 3109 static inline 3110 unsigned long uclamp_rq_util_with(struct rq *rq, unsigned long util, 3111 struct task_struct *p) 3112 { 3113 return util; 3114 } 3115 3116 static inline bool uclamp_rq_is_capped(struct rq *rq) { return false; } 3117 3118 static inline bool uclamp_is_used(void) 3119 { 3120 return false; 3121 } 3122 3123 static inline unsigned long uclamp_rq_get(struct rq *rq, 3124 enum uclamp_id clamp_id) 3125 { 3126 if (clamp_id == UCLAMP_MIN) 3127 return 0; 3128 3129 return SCHED_CAPACITY_SCALE; 3130 } 3131 3132 static inline void uclamp_rq_set(struct rq *rq, enum uclamp_id clamp_id, 3133 unsigned int value) 3134 { 3135 } 3136 3137 static inline bool uclamp_rq_is_idle(struct rq *rq) 3138 { 3139 return false; 3140 } 3141 #endif /* CONFIG_UCLAMP_TASK */ 3142 3143 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ 3144 static inline unsigned long cpu_util_irq(struct rq *rq) 3145 { 3146 return rq->avg_irq.util_avg; 3147 } 3148 3149 static inline 3150 unsigned long scale_irq_capacity(unsigned long util, unsigned long irq, unsigned long max) 3151 { 3152 util *= (max - irq); 3153 util /= max; 3154 3155 return util; 3156 3157 } 3158 #else 3159 static inline unsigned long cpu_util_irq(struct rq *rq) 3160 { 3161 return 0; 3162 } 3163 3164 static inline 3165 unsigned long scale_irq_capacity(unsigned long util, unsigned long irq, unsigned long max) 3166 { 3167 return util; 3168 } 3169 #endif 3170 3171 #if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) 3172 3173 #define perf_domain_span(pd) (to_cpumask(((pd)->em_pd->cpus))) 3174 3175 DECLARE_STATIC_KEY_FALSE(sched_energy_present); 3176 3177 static inline bool sched_energy_enabled(void) 3178 { 3179 return static_branch_unlikely(&sched_energy_present); 3180 } 3181 3182 #else /* ! (CONFIG_ENERGY_MODEL && CONFIG_CPU_FREQ_GOV_SCHEDUTIL) */ 3183 3184 #define perf_domain_span(pd) NULL 3185 static inline bool sched_energy_enabled(void) { return false; } 3186 3187 #endif /* CONFIG_ENERGY_MODEL && CONFIG_CPU_FREQ_GOV_SCHEDUTIL */ 3188 3189 #ifdef CONFIG_MEMBARRIER 3190 /* 3191 * The scheduler provides memory barriers required by membarrier between: 3192 * - prior user-space memory accesses and store to rq->membarrier_state, 3193 * - store to rq->membarrier_state and following user-space memory accesses. 3194 * In the same way it provides those guarantees around store to rq->curr. 3195 */ 3196 static inline void membarrier_switch_mm(struct rq *rq, 3197 struct mm_struct *prev_mm, 3198 struct mm_struct *next_mm) 3199 { 3200 int membarrier_state; 3201 3202 if (prev_mm == next_mm) 3203 return; 3204 3205 membarrier_state = atomic_read(&next_mm->membarrier_state); 3206 if (READ_ONCE(rq->membarrier_state) == membarrier_state) 3207 return; 3208 3209 WRITE_ONCE(rq->membarrier_state, membarrier_state); 3210 } 3211 #else 3212 static inline void membarrier_switch_mm(struct rq *rq, 3213 struct mm_struct *prev_mm, 3214 struct mm_struct *next_mm) 3215 { 3216 } 3217 #endif 3218 3219 #ifdef CONFIG_SMP 3220 static inline bool is_per_cpu_kthread(struct task_struct *p) 3221 { 3222 if (!(p->flags & PF_KTHREAD)) 3223 return false; 3224 3225 if (p->nr_cpus_allowed != 1) 3226 return false; 3227 3228 return true; 3229 } 3230 #endif 3231 3232 extern void swake_up_all_locked(struct swait_queue_head *q); 3233 extern void __prepare_to_swait(struct swait_queue_head *q, struct swait_queue *wait); 3234 3235 #ifdef CONFIG_PREEMPT_DYNAMIC 3236 extern int preempt_dynamic_mode; 3237 extern int sched_dynamic_mode(const char *str); 3238 extern void sched_dynamic_update(int mode); 3239 #endif 3240 3241 static inline void update_current_exec_runtime(struct task_struct *curr, 3242 u64 now, u64 delta_exec) 3243 { 3244 curr->se.sum_exec_runtime += delta_exec; 3245 account_group_exec_runtime(curr, delta_exec); 3246 3247 curr->se.exec_start = now; 3248 cgroup_account_cputime(curr, delta_exec); 3249 } 3250 3251 #ifdef CONFIG_SCHED_MM_CID 3252 3253 #define SCHED_MM_CID_PERIOD_NS (100ULL * 1000000) /* 100ms */ 3254 #define MM_CID_SCAN_DELAY 100 /* 100ms */ 3255 3256 extern raw_spinlock_t cid_lock; 3257 extern int use_cid_lock; 3258 3259 extern void sched_mm_cid_migrate_from(struct task_struct *t); 3260 extern void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t); 3261 extern void task_tick_mm_cid(struct rq *rq, struct task_struct *curr); 3262 extern void init_sched_mm_cid(struct task_struct *t); 3263 3264 static inline void __mm_cid_put(struct mm_struct *mm, int cid) 3265 { 3266 if (cid < 0) 3267 return; 3268 cpumask_clear_cpu(cid, mm_cidmask(mm)); 3269 } 3270 3271 /* 3272 * The per-mm/cpu cid can have the MM_CID_LAZY_PUT flag set or transition to 3273 * the MM_CID_UNSET state without holding the rq lock, but the rq lock needs to 3274 * be held to transition to other states. 3275 * 3276 * State transitions synchronized with cmpxchg or try_cmpxchg need to be 3277 * consistent across cpus, which prevents use of this_cpu_cmpxchg. 3278 */ 3279 static inline void mm_cid_put_lazy(struct task_struct *t) 3280 { 3281 struct mm_struct *mm = t->mm; 3282 struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid; 3283 int cid; 3284 3285 lockdep_assert_irqs_disabled(); 3286 cid = __this_cpu_read(pcpu_cid->cid); 3287 if (!mm_cid_is_lazy_put(cid) || 3288 !try_cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, &cid, MM_CID_UNSET)) 3289 return; 3290 __mm_cid_put(mm, mm_cid_clear_lazy_put(cid)); 3291 } 3292 3293 static inline int mm_cid_pcpu_unset(struct mm_struct *mm) 3294 { 3295 struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid; 3296 int cid, res; 3297 3298 lockdep_assert_irqs_disabled(); 3299 cid = __this_cpu_read(pcpu_cid->cid); 3300 for (;;) { 3301 if (mm_cid_is_unset(cid)) 3302 return MM_CID_UNSET; 3303 /* 3304 * Attempt transition from valid or lazy-put to unset. 3305 */ 3306 res = cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, cid, MM_CID_UNSET); 3307 if (res == cid) 3308 break; 3309 cid = res; 3310 } 3311 return cid; 3312 } 3313 3314 static inline void mm_cid_put(struct mm_struct *mm) 3315 { 3316 int cid; 3317 3318 lockdep_assert_irqs_disabled(); 3319 cid = mm_cid_pcpu_unset(mm); 3320 if (cid == MM_CID_UNSET) 3321 return; 3322 __mm_cid_put(mm, mm_cid_clear_lazy_put(cid)); 3323 } 3324 3325 static inline int __mm_cid_try_get(struct mm_struct *mm) 3326 { 3327 struct cpumask *cpumask; 3328 int cid; 3329 3330 cpumask = mm_cidmask(mm); 3331 /* 3332 * Retry finding first zero bit if the mask is temporarily 3333 * filled. This only happens during concurrent remote-clear 3334 * which owns a cid without holding a rq lock. 3335 */ 3336 for (;;) { 3337 cid = cpumask_first_zero(cpumask); 3338 if (cid < nr_cpu_ids) 3339 break; 3340 cpu_relax(); 3341 } 3342 if (cpumask_test_and_set_cpu(cid, cpumask)) 3343 return -1; 3344 return cid; 3345 } 3346 3347 /* 3348 * Save a snapshot of the current runqueue time of this cpu 3349 * with the per-cpu cid value, allowing to estimate how recently it was used. 3350 */ 3351 static inline void mm_cid_snapshot_time(struct rq *rq, struct mm_struct *mm) 3352 { 3353 struct mm_cid *pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu_of(rq)); 3354 3355 lockdep_assert_rq_held(rq); 3356 WRITE_ONCE(pcpu_cid->time, rq->clock); 3357 } 3358 3359 static inline int __mm_cid_get(struct rq *rq, struct mm_struct *mm) 3360 { 3361 int cid; 3362 3363 /* 3364 * All allocations (even those using the cid_lock) are lock-free. If 3365 * use_cid_lock is set, hold the cid_lock to perform cid allocation to 3366 * guarantee forward progress. 3367 */ 3368 if (!READ_ONCE(use_cid_lock)) { 3369 cid = __mm_cid_try_get(mm); 3370 if (cid >= 0) 3371 goto end; 3372 raw_spin_lock(&cid_lock); 3373 } else { 3374 raw_spin_lock(&cid_lock); 3375 cid = __mm_cid_try_get(mm); 3376 if (cid >= 0) 3377 goto unlock; 3378 } 3379 3380 /* 3381 * cid concurrently allocated. Retry while forcing following 3382 * allocations to use the cid_lock to ensure forward progress. 3383 */ 3384 WRITE_ONCE(use_cid_lock, 1); 3385 /* 3386 * Set use_cid_lock before allocation. Only care about program order 3387 * because this is only required for forward progress. 3388 */ 3389 barrier(); 3390 /* 3391 * Retry until it succeeds. It is guaranteed to eventually succeed once 3392 * all newcoming allocations observe the use_cid_lock flag set. 3393 */ 3394 do { 3395 cid = __mm_cid_try_get(mm); 3396 cpu_relax(); 3397 } while (cid < 0); 3398 /* 3399 * Allocate before clearing use_cid_lock. Only care about 3400 * program order because this is for forward progress. 3401 */ 3402 barrier(); 3403 WRITE_ONCE(use_cid_lock, 0); 3404 unlock: 3405 raw_spin_unlock(&cid_lock); 3406 end: 3407 mm_cid_snapshot_time(rq, mm); 3408 return cid; 3409 } 3410 3411 static inline int mm_cid_get(struct rq *rq, struct mm_struct *mm) 3412 { 3413 struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid; 3414 struct cpumask *cpumask; 3415 int cid; 3416 3417 lockdep_assert_rq_held(rq); 3418 cpumask = mm_cidmask(mm); 3419 cid = __this_cpu_read(pcpu_cid->cid); 3420 if (mm_cid_is_valid(cid)) { 3421 mm_cid_snapshot_time(rq, mm); 3422 return cid; 3423 } 3424 if (mm_cid_is_lazy_put(cid)) { 3425 if (try_cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, &cid, MM_CID_UNSET)) 3426 __mm_cid_put(mm, mm_cid_clear_lazy_put(cid)); 3427 } 3428 cid = __mm_cid_get(rq, mm); 3429 __this_cpu_write(pcpu_cid->cid, cid); 3430 return cid; 3431 } 3432 3433 static inline void switch_mm_cid(struct rq *rq, 3434 struct task_struct *prev, 3435 struct task_struct *next) 3436 { 3437 /* 3438 * Provide a memory barrier between rq->curr store and load of 3439 * {prev,next}->mm->pcpu_cid[cpu] on rq->curr->mm transition. 3440 * 3441 * Should be adapted if context_switch() is modified. 3442 */ 3443 if (!next->mm) { // to kernel 3444 /* 3445 * user -> kernel transition does not guarantee a barrier, but 3446 * we can use the fact that it performs an atomic operation in 3447 * mmgrab(). 3448 */ 3449 if (prev->mm) // from user 3450 smp_mb__after_mmgrab(); 3451 /* 3452 * kernel -> kernel transition does not change rq->curr->mm 3453 * state. It stays NULL. 3454 */ 3455 } else { // to user 3456 /* 3457 * kernel -> user transition does not provide a barrier 3458 * between rq->curr store and load of {prev,next}->mm->pcpu_cid[cpu]. 3459 * Provide it here. 3460 */ 3461 if (!prev->mm) // from kernel 3462 smp_mb(); 3463 /* 3464 * user -> user transition guarantees a memory barrier through 3465 * switch_mm() when current->mm changes. If current->mm is 3466 * unchanged, no barrier is needed. 3467 */ 3468 } 3469 if (prev->mm_cid_active) { 3470 mm_cid_snapshot_time(rq, prev->mm); 3471 mm_cid_put_lazy(prev); 3472 prev->mm_cid = -1; 3473 } 3474 if (next->mm_cid_active) 3475 next->last_mm_cid = next->mm_cid = mm_cid_get(rq, next->mm); 3476 } 3477 3478 #else 3479 static inline void switch_mm_cid(struct rq *rq, struct task_struct *prev, struct task_struct *next) { } 3480 static inline void sched_mm_cid_migrate_from(struct task_struct *t) { } 3481 static inline void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t) { } 3482 static inline void task_tick_mm_cid(struct rq *rq, struct task_struct *curr) { } 3483 static inline void init_sched_mm_cid(struct task_struct *t) { } 3484 #endif 3485 3486 extern u64 avg_vruntime(struct cfs_rq *cfs_rq); 3487 extern int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se); 3488 3489 #endif /* _KERNEL_SCHED_SCHED_H */ 3490