1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * kernel/sched/core.c 4 * 5 * Core kernel scheduler code and related syscalls 6 * 7 * Copyright (C) 1991-2002 Linus Torvalds 8 */ 9 #include <linux/highmem.h> 10 #include <linux/hrtimer_api.h> 11 #include <linux/ktime_api.h> 12 #include <linux/sched/signal.h> 13 #include <linux/syscalls_api.h> 14 #include <linux/debug_locks.h> 15 #include <linux/prefetch.h> 16 #include <linux/capability.h> 17 #include <linux/pgtable_api.h> 18 #include <linux/wait_bit.h> 19 #include <linux/jiffies.h> 20 #include <linux/spinlock_api.h> 21 #include <linux/cpumask_api.h> 22 #include <linux/lockdep_api.h> 23 #include <linux/hardirq.h> 24 #include <linux/softirq.h> 25 #include <linux/refcount_api.h> 26 #include <linux/topology.h> 27 #include <linux/sched/clock.h> 28 #include <linux/sched/cond_resched.h> 29 #include <linux/sched/cputime.h> 30 #include <linux/sched/debug.h> 31 #include <linux/sched/hotplug.h> 32 #include <linux/sched/init.h> 33 #include <linux/sched/isolation.h> 34 #include <linux/sched/loadavg.h> 35 #include <linux/sched/mm.h> 36 #include <linux/sched/nohz.h> 37 #include <linux/sched/rseq_api.h> 38 #include <linux/sched/rt.h> 39 40 #include <linux/blkdev.h> 41 #include <linux/context_tracking.h> 42 #include <linux/cpuset.h> 43 #include <linux/delayacct.h> 44 #include <linux/init_task.h> 45 #include <linux/interrupt.h> 46 #include <linux/ioprio.h> 47 #include <linux/kallsyms.h> 48 #include <linux/kcov.h> 49 #include <linux/kprobes.h> 50 #include <linux/llist_api.h> 51 #include <linux/mmu_context.h> 52 #include <linux/mmzone.h> 53 #include <linux/mutex_api.h> 54 #include <linux/nmi.h> 55 #include <linux/nospec.h> 56 #include <linux/perf_event_api.h> 57 #include <linux/profile.h> 58 #include <linux/psi.h> 59 #include <linux/rcuwait_api.h> 60 #include <linux/sched/wake_q.h> 61 #include <linux/scs.h> 62 #include <linux/slab.h> 63 #include <linux/syscalls.h> 64 #include <linux/vtime.h> 65 #include <linux/wait_api.h> 66 #include <linux/workqueue_api.h> 67 68 #ifdef CONFIG_PREEMPT_DYNAMIC 69 # ifdef CONFIG_GENERIC_ENTRY 70 # include <linux/entry-common.h> 71 # endif 72 #endif 73 74 #include <uapi/linux/sched/types.h> 75 76 #include <asm/irq_regs.h> 77 #include <asm/switch_to.h> 78 #include <asm/tlb.h> 79 80 #define CREATE_TRACE_POINTS 81 #include <linux/sched/rseq_api.h> 82 #include <trace/events/sched.h> 83 #include <trace/events/ipi.h> 84 #undef CREATE_TRACE_POINTS 85 86 #include "sched.h" 87 #include "stats.h" 88 #include "autogroup.h" 89 90 #include "autogroup.h" 91 #include "pelt.h" 92 #include "smp.h" 93 #include "stats.h" 94 95 #include "../workqueue_internal.h" 96 #include "../../io_uring/io-wq.h" 97 #include "../smpboot.h" 98 99 EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpu); 100 EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpumask); 101 102 /* 103 * Export tracepoints that act as a bare tracehook (ie: have no trace event 104 * associated with them) to allow external modules to probe them. 105 */ 106 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_cfs_tp); 107 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_rt_tp); 108 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_dl_tp); 109 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_irq_tp); 110 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_se_tp); 111 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_thermal_tp); 112 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_cpu_capacity_tp); 113 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_overutilized_tp); 114 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_cfs_tp); 115 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_se_tp); 116 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_update_nr_running_tp); 117 118 DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); 119 120 #ifdef CONFIG_SCHED_DEBUG 121 /* 122 * Debugging: various feature bits 123 * 124 * If SCHED_DEBUG is disabled, each compilation unit has its own copy of 125 * sysctl_sched_features, defined in sched.h, to allow constants propagation 126 * at compile time and compiler optimization based on features default. 127 */ 128 #define SCHED_FEAT(name, enabled) \ 129 (1UL << __SCHED_FEAT_##name) * enabled | 130 const_debug unsigned int sysctl_sched_features = 131 #include "features.h" 132 0; 133 #undef SCHED_FEAT 134 135 /* 136 * Print a warning if need_resched is set for the given duration (if 137 * LATENCY_WARN is enabled). 138 * 139 * If sysctl_resched_latency_warn_once is set, only one warning will be shown 140 * per boot. 141 */ 142 __read_mostly int sysctl_resched_latency_warn_ms = 100; 143 __read_mostly int sysctl_resched_latency_warn_once = 1; 144 #endif /* CONFIG_SCHED_DEBUG */ 145 146 /* 147 * Number of tasks to iterate in a single balance run. 148 * Limited because this is done with IRQs disabled. 149 */ 150 const_debug unsigned int sysctl_sched_nr_migrate = SCHED_NR_MIGRATE_BREAK; 151 152 __read_mostly int scheduler_running; 153 154 #ifdef CONFIG_SCHED_CORE 155 156 DEFINE_STATIC_KEY_FALSE(__sched_core_enabled); 157 158 /* kernel prio, less is more */ 159 static inline int __task_prio(const struct task_struct *p) 160 { 161 if (p->sched_class == &stop_sched_class) /* trumps deadline */ 162 return -2; 163 164 if (rt_prio(p->prio)) /* includes deadline */ 165 return p->prio; /* [-1, 99] */ 166 167 if (p->sched_class == &idle_sched_class) 168 return MAX_RT_PRIO + NICE_WIDTH; /* 140 */ 169 170 return MAX_RT_PRIO + MAX_NICE; /* 120, squash fair */ 171 } 172 173 /* 174 * l(a,b) 175 * le(a,b) := !l(b,a) 176 * g(a,b) := l(b,a) 177 * ge(a,b) := !l(a,b) 178 */ 179 180 /* real prio, less is less */ 181 static inline bool prio_less(const struct task_struct *a, 182 const struct task_struct *b, bool in_fi) 183 { 184 185 int pa = __task_prio(a), pb = __task_prio(b); 186 187 if (-pa < -pb) 188 return true; 189 190 if (-pb < -pa) 191 return false; 192 193 if (pa == -1) /* dl_prio() doesn't work because of stop_class above */ 194 return !dl_time_before(a->dl.deadline, b->dl.deadline); 195 196 if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */ 197 return cfs_prio_less(a, b, in_fi); 198 199 return false; 200 } 201 202 static inline bool __sched_core_less(const struct task_struct *a, 203 const struct task_struct *b) 204 { 205 if (a->core_cookie < b->core_cookie) 206 return true; 207 208 if (a->core_cookie > b->core_cookie) 209 return false; 210 211 /* flip prio, so high prio is leftmost */ 212 if (prio_less(b, a, !!task_rq(a)->core->core_forceidle_count)) 213 return true; 214 215 return false; 216 } 217 218 #define __node_2_sc(node) rb_entry((node), struct task_struct, core_node) 219 220 static inline bool rb_sched_core_less(struct rb_node *a, const struct rb_node *b) 221 { 222 return __sched_core_less(__node_2_sc(a), __node_2_sc(b)); 223 } 224 225 static inline int rb_sched_core_cmp(const void *key, const struct rb_node *node) 226 { 227 const struct task_struct *p = __node_2_sc(node); 228 unsigned long cookie = (unsigned long)key; 229 230 if (cookie < p->core_cookie) 231 return -1; 232 233 if (cookie > p->core_cookie) 234 return 1; 235 236 return 0; 237 } 238 239 void sched_core_enqueue(struct rq *rq, struct task_struct *p) 240 { 241 rq->core->core_task_seq++; 242 243 if (!p->core_cookie) 244 return; 245 246 rb_add(&p->core_node, &rq->core_tree, rb_sched_core_less); 247 } 248 249 void sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) 250 { 251 rq->core->core_task_seq++; 252 253 if (sched_core_enqueued(p)) { 254 rb_erase(&p->core_node, &rq->core_tree); 255 RB_CLEAR_NODE(&p->core_node); 256 } 257 258 /* 259 * Migrating the last task off the cpu, with the cpu in forced idle 260 * state. Reschedule to create an accounting edge for forced idle, 261 * and re-examine whether the core is still in forced idle state. 262 */ 263 if (!(flags & DEQUEUE_SAVE) && rq->nr_running == 1 && 264 rq->core->core_forceidle_count && rq->curr == rq->idle) 265 resched_curr(rq); 266 } 267 268 static int sched_task_is_throttled(struct task_struct *p, int cpu) 269 { 270 if (p->sched_class->task_is_throttled) 271 return p->sched_class->task_is_throttled(p, cpu); 272 273 return 0; 274 } 275 276 static struct task_struct *sched_core_next(struct task_struct *p, unsigned long cookie) 277 { 278 struct rb_node *node = &p->core_node; 279 int cpu = task_cpu(p); 280 281 do { 282 node = rb_next(node); 283 if (!node) 284 return NULL; 285 286 p = __node_2_sc(node); 287 if (p->core_cookie != cookie) 288 return NULL; 289 290 } while (sched_task_is_throttled(p, cpu)); 291 292 return p; 293 } 294 295 /* 296 * Find left-most (aka, highest priority) and unthrottled task matching @cookie. 297 * If no suitable task is found, NULL will be returned. 298 */ 299 static struct task_struct *sched_core_find(struct rq *rq, unsigned long cookie) 300 { 301 struct task_struct *p; 302 struct rb_node *node; 303 304 node = rb_find_first((void *)cookie, &rq->core_tree, rb_sched_core_cmp); 305 if (!node) 306 return NULL; 307 308 p = __node_2_sc(node); 309 if (!sched_task_is_throttled(p, rq->cpu)) 310 return p; 311 312 return sched_core_next(p, cookie); 313 } 314 315 /* 316 * Magic required such that: 317 * 318 * raw_spin_rq_lock(rq); 319 * ... 320 * raw_spin_rq_unlock(rq); 321 * 322 * ends up locking and unlocking the _same_ lock, and all CPUs 323 * always agree on what rq has what lock. 324 * 325 * XXX entirely possible to selectively enable cores, don't bother for now. 326 */ 327 328 static DEFINE_MUTEX(sched_core_mutex); 329 static atomic_t sched_core_count; 330 static struct cpumask sched_core_mask; 331 332 static void sched_core_lock(int cpu, unsigned long *flags) 333 { 334 const struct cpumask *smt_mask = cpu_smt_mask(cpu); 335 int t, i = 0; 336 337 local_irq_save(*flags); 338 for_each_cpu(t, smt_mask) 339 raw_spin_lock_nested(&cpu_rq(t)->__lock, i++); 340 } 341 342 static void sched_core_unlock(int cpu, unsigned long *flags) 343 { 344 const struct cpumask *smt_mask = cpu_smt_mask(cpu); 345 int t; 346 347 for_each_cpu(t, smt_mask) 348 raw_spin_unlock(&cpu_rq(t)->__lock); 349 local_irq_restore(*flags); 350 } 351 352 static void __sched_core_flip(bool enabled) 353 { 354 unsigned long flags; 355 int cpu, t; 356 357 cpus_read_lock(); 358 359 /* 360 * Toggle the online cores, one by one. 361 */ 362 cpumask_copy(&sched_core_mask, cpu_online_mask); 363 for_each_cpu(cpu, &sched_core_mask) { 364 const struct cpumask *smt_mask = cpu_smt_mask(cpu); 365 366 sched_core_lock(cpu, &flags); 367 368 for_each_cpu(t, smt_mask) 369 cpu_rq(t)->core_enabled = enabled; 370 371 cpu_rq(cpu)->core->core_forceidle_start = 0; 372 373 sched_core_unlock(cpu, &flags); 374 375 cpumask_andnot(&sched_core_mask, &sched_core_mask, smt_mask); 376 } 377 378 /* 379 * Toggle the offline CPUs. 380 */ 381 for_each_cpu_andnot(cpu, cpu_possible_mask, cpu_online_mask) 382 cpu_rq(cpu)->core_enabled = enabled; 383 384 cpus_read_unlock(); 385 } 386 387 static void sched_core_assert_empty(void) 388 { 389 int cpu; 390 391 for_each_possible_cpu(cpu) 392 WARN_ON_ONCE(!RB_EMPTY_ROOT(&cpu_rq(cpu)->core_tree)); 393 } 394 395 static void __sched_core_enable(void) 396 { 397 static_branch_enable(&__sched_core_enabled); 398 /* 399 * Ensure all previous instances of raw_spin_rq_*lock() have finished 400 * and future ones will observe !sched_core_disabled(). 401 */ 402 synchronize_rcu(); 403 __sched_core_flip(true); 404 sched_core_assert_empty(); 405 } 406 407 static void __sched_core_disable(void) 408 { 409 sched_core_assert_empty(); 410 __sched_core_flip(false); 411 static_branch_disable(&__sched_core_enabled); 412 } 413 414 void sched_core_get(void) 415 { 416 if (atomic_inc_not_zero(&sched_core_count)) 417 return; 418 419 mutex_lock(&sched_core_mutex); 420 if (!atomic_read(&sched_core_count)) 421 __sched_core_enable(); 422 423 smp_mb__before_atomic(); 424 atomic_inc(&sched_core_count); 425 mutex_unlock(&sched_core_mutex); 426 } 427 428 static void __sched_core_put(struct work_struct *work) 429 { 430 if (atomic_dec_and_mutex_lock(&sched_core_count, &sched_core_mutex)) { 431 __sched_core_disable(); 432 mutex_unlock(&sched_core_mutex); 433 } 434 } 435 436 void sched_core_put(void) 437 { 438 static DECLARE_WORK(_work, __sched_core_put); 439 440 /* 441 * "There can be only one" 442 * 443 * Either this is the last one, or we don't actually need to do any 444 * 'work'. If it is the last *again*, we rely on 445 * WORK_STRUCT_PENDING_BIT. 446 */ 447 if (!atomic_add_unless(&sched_core_count, -1, 1)) 448 schedule_work(&_work); 449 } 450 451 #else /* !CONFIG_SCHED_CORE */ 452 453 static inline void sched_core_enqueue(struct rq *rq, struct task_struct *p) { } 454 static inline void 455 sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) { } 456 457 #endif /* CONFIG_SCHED_CORE */ 458 459 /* 460 * Serialization rules: 461 * 462 * Lock order: 463 * 464 * p->pi_lock 465 * rq->lock 466 * hrtimer_cpu_base->lock (hrtimer_start() for bandwidth controls) 467 * 468 * rq1->lock 469 * rq2->lock where: rq1 < rq2 470 * 471 * Regular state: 472 * 473 * Normal scheduling state is serialized by rq->lock. __schedule() takes the 474 * local CPU's rq->lock, it optionally removes the task from the runqueue and 475 * always looks at the local rq data structures to find the most eligible task 476 * to run next. 477 * 478 * Task enqueue is also under rq->lock, possibly taken from another CPU. 479 * Wakeups from another LLC domain might use an IPI to transfer the enqueue to 480 * the local CPU to avoid bouncing the runqueue state around [ see 481 * ttwu_queue_wakelist() ] 482 * 483 * Task wakeup, specifically wakeups that involve migration, are horribly 484 * complicated to avoid having to take two rq->locks. 485 * 486 * Special state: 487 * 488 * System-calls and anything external will use task_rq_lock() which acquires 489 * both p->pi_lock and rq->lock. As a consequence the state they change is 490 * stable while holding either lock: 491 * 492 * - sched_setaffinity()/ 493 * set_cpus_allowed_ptr(): p->cpus_ptr, p->nr_cpus_allowed 494 * - set_user_nice(): p->se.load, p->*prio 495 * - __sched_setscheduler(): p->sched_class, p->policy, p->*prio, 496 * p->se.load, p->rt_priority, 497 * p->dl.dl_{runtime, deadline, period, flags, bw, density} 498 * - sched_setnuma(): p->numa_preferred_nid 499 * - sched_move_task(): p->sched_task_group 500 * - uclamp_update_active() p->uclamp* 501 * 502 * p->state <- TASK_*: 503 * 504 * is changed locklessly using set_current_state(), __set_current_state() or 505 * set_special_state(), see their respective comments, or by 506 * try_to_wake_up(). This latter uses p->pi_lock to serialize against 507 * concurrent self. 508 * 509 * p->on_rq <- { 0, 1 = TASK_ON_RQ_QUEUED, 2 = TASK_ON_RQ_MIGRATING }: 510 * 511 * is set by activate_task() and cleared by deactivate_task(), under 512 * rq->lock. Non-zero indicates the task is runnable, the special 513 * ON_RQ_MIGRATING state is used for migration without holding both 514 * rq->locks. It indicates task_cpu() is not stable, see task_rq_lock(). 515 * 516 * p->on_cpu <- { 0, 1 }: 517 * 518 * is set by prepare_task() and cleared by finish_task() such that it will be 519 * set before p is scheduled-in and cleared after p is scheduled-out, both 520 * under rq->lock. Non-zero indicates the task is running on its CPU. 521 * 522 * [ The astute reader will observe that it is possible for two tasks on one 523 * CPU to have ->on_cpu = 1 at the same time. ] 524 * 525 * task_cpu(p): is changed by set_task_cpu(), the rules are: 526 * 527 * - Don't call set_task_cpu() on a blocked task: 528 * 529 * We don't care what CPU we're not running on, this simplifies hotplug, 530 * the CPU assignment of blocked tasks isn't required to be valid. 531 * 532 * - for try_to_wake_up(), called under p->pi_lock: 533 * 534 * This allows try_to_wake_up() to only take one rq->lock, see its comment. 535 * 536 * - for migration called under rq->lock: 537 * [ see task_on_rq_migrating() in task_rq_lock() ] 538 * 539 * o move_queued_task() 540 * o detach_task() 541 * 542 * - for migration called under double_rq_lock(): 543 * 544 * o __migrate_swap_task() 545 * o push_rt_task() / pull_rt_task() 546 * o push_dl_task() / pull_dl_task() 547 * o dl_task_offline_migration() 548 * 549 */ 550 551 void raw_spin_rq_lock_nested(struct rq *rq, int subclass) 552 { 553 raw_spinlock_t *lock; 554 555 /* Matches synchronize_rcu() in __sched_core_enable() */ 556 preempt_disable(); 557 if (sched_core_disabled()) { 558 raw_spin_lock_nested(&rq->__lock, subclass); 559 /* preempt_count *MUST* be > 1 */ 560 preempt_enable_no_resched(); 561 return; 562 } 563 564 for (;;) { 565 lock = __rq_lockp(rq); 566 raw_spin_lock_nested(lock, subclass); 567 if (likely(lock == __rq_lockp(rq))) { 568 /* preempt_count *MUST* be > 1 */ 569 preempt_enable_no_resched(); 570 return; 571 } 572 raw_spin_unlock(lock); 573 } 574 } 575 576 bool raw_spin_rq_trylock(struct rq *rq) 577 { 578 raw_spinlock_t *lock; 579 bool ret; 580 581 /* Matches synchronize_rcu() in __sched_core_enable() */ 582 preempt_disable(); 583 if (sched_core_disabled()) { 584 ret = raw_spin_trylock(&rq->__lock); 585 preempt_enable(); 586 return ret; 587 } 588 589 for (;;) { 590 lock = __rq_lockp(rq); 591 ret = raw_spin_trylock(lock); 592 if (!ret || (likely(lock == __rq_lockp(rq)))) { 593 preempt_enable(); 594 return ret; 595 } 596 raw_spin_unlock(lock); 597 } 598 } 599 600 void raw_spin_rq_unlock(struct rq *rq) 601 { 602 raw_spin_unlock(rq_lockp(rq)); 603 } 604 605 #ifdef CONFIG_SMP 606 /* 607 * double_rq_lock - safely lock two runqueues 608 */ 609 void double_rq_lock(struct rq *rq1, struct rq *rq2) 610 { 611 lockdep_assert_irqs_disabled(); 612 613 if (rq_order_less(rq2, rq1)) 614 swap(rq1, rq2); 615 616 raw_spin_rq_lock(rq1); 617 if (__rq_lockp(rq1) != __rq_lockp(rq2)) 618 raw_spin_rq_lock_nested(rq2, SINGLE_DEPTH_NESTING); 619 620 double_rq_clock_clear_update(rq1, rq2); 621 } 622 #endif 623 624 /* 625 * __task_rq_lock - lock the rq @p resides on. 626 */ 627 struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf) 628 __acquires(rq->lock) 629 { 630 struct rq *rq; 631 632 lockdep_assert_held(&p->pi_lock); 633 634 for (;;) { 635 rq = task_rq(p); 636 raw_spin_rq_lock(rq); 637 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) { 638 rq_pin_lock(rq, rf); 639 return rq; 640 } 641 raw_spin_rq_unlock(rq); 642 643 while (unlikely(task_on_rq_migrating(p))) 644 cpu_relax(); 645 } 646 } 647 648 /* 649 * task_rq_lock - lock p->pi_lock and lock the rq @p resides on. 650 */ 651 struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf) 652 __acquires(p->pi_lock) 653 __acquires(rq->lock) 654 { 655 struct rq *rq; 656 657 for (;;) { 658 raw_spin_lock_irqsave(&p->pi_lock, rf->flags); 659 rq = task_rq(p); 660 raw_spin_rq_lock(rq); 661 /* 662 * move_queued_task() task_rq_lock() 663 * 664 * ACQUIRE (rq->lock) 665 * [S] ->on_rq = MIGRATING [L] rq = task_rq() 666 * WMB (__set_task_cpu()) ACQUIRE (rq->lock); 667 * [S] ->cpu = new_cpu [L] task_rq() 668 * [L] ->on_rq 669 * RELEASE (rq->lock) 670 * 671 * If we observe the old CPU in task_rq_lock(), the acquire of 672 * the old rq->lock will fully serialize against the stores. 673 * 674 * If we observe the new CPU in task_rq_lock(), the address 675 * dependency headed by '[L] rq = task_rq()' and the acquire 676 * will pair with the WMB to ensure we then also see migrating. 677 */ 678 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) { 679 rq_pin_lock(rq, rf); 680 return rq; 681 } 682 raw_spin_rq_unlock(rq); 683 raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags); 684 685 while (unlikely(task_on_rq_migrating(p))) 686 cpu_relax(); 687 } 688 } 689 690 /* 691 * RQ-clock updating methods: 692 */ 693 694 static void update_rq_clock_task(struct rq *rq, s64 delta) 695 { 696 /* 697 * In theory, the compile should just see 0 here, and optimize out the call 698 * to sched_rt_avg_update. But I don't trust it... 699 */ 700 s64 __maybe_unused steal = 0, irq_delta = 0; 701 702 #ifdef CONFIG_IRQ_TIME_ACCOUNTING 703 irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time; 704 705 /* 706 * Since irq_time is only updated on {soft,}irq_exit, we might run into 707 * this case when a previous update_rq_clock() happened inside a 708 * {soft,}irq region. 709 * 710 * When this happens, we stop ->clock_task and only update the 711 * prev_irq_time stamp to account for the part that fit, so that a next 712 * update will consume the rest. This ensures ->clock_task is 713 * monotonic. 714 * 715 * It does however cause some slight miss-attribution of {soft,}irq 716 * time, a more accurate solution would be to update the irq_time using 717 * the current rq->clock timestamp, except that would require using 718 * atomic ops. 719 */ 720 if (irq_delta > delta) 721 irq_delta = delta; 722 723 rq->prev_irq_time += irq_delta; 724 delta -= irq_delta; 725 psi_account_irqtime(rq->curr, irq_delta); 726 delayacct_irq(rq->curr, irq_delta); 727 #endif 728 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING 729 if (static_key_false((¶virt_steal_rq_enabled))) { 730 steal = paravirt_steal_clock(cpu_of(rq)); 731 steal -= rq->prev_steal_time_rq; 732 733 if (unlikely(steal > delta)) 734 steal = delta; 735 736 rq->prev_steal_time_rq += steal; 737 delta -= steal; 738 } 739 #endif 740 741 rq->clock_task += delta; 742 743 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ 744 if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY)) 745 update_irq_load_avg(rq, irq_delta + steal); 746 #endif 747 update_rq_clock_pelt(rq, delta); 748 } 749 750 void update_rq_clock(struct rq *rq) 751 { 752 s64 delta; 753 754 lockdep_assert_rq_held(rq); 755 756 if (rq->clock_update_flags & RQCF_ACT_SKIP) 757 return; 758 759 #ifdef CONFIG_SCHED_DEBUG 760 if (sched_feat(WARN_DOUBLE_CLOCK)) 761 SCHED_WARN_ON(rq->clock_update_flags & RQCF_UPDATED); 762 rq->clock_update_flags |= RQCF_UPDATED; 763 #endif 764 765 delta = sched_clock_cpu(cpu_of(rq)) - rq->clock; 766 if (delta < 0) 767 return; 768 rq->clock += delta; 769 update_rq_clock_task(rq, delta); 770 } 771 772 #ifdef CONFIG_SCHED_HRTICK 773 /* 774 * Use HR-timers to deliver accurate preemption points. 775 */ 776 777 static void hrtick_clear(struct rq *rq) 778 { 779 if (hrtimer_active(&rq->hrtick_timer)) 780 hrtimer_cancel(&rq->hrtick_timer); 781 } 782 783 /* 784 * High-resolution timer tick. 785 * Runs from hardirq context with interrupts disabled. 786 */ 787 static enum hrtimer_restart hrtick(struct hrtimer *timer) 788 { 789 struct rq *rq = container_of(timer, struct rq, hrtick_timer); 790 struct rq_flags rf; 791 792 WARN_ON_ONCE(cpu_of(rq) != smp_processor_id()); 793 794 rq_lock(rq, &rf); 795 update_rq_clock(rq); 796 rq->curr->sched_class->task_tick(rq, rq->curr, 1); 797 rq_unlock(rq, &rf); 798 799 return HRTIMER_NORESTART; 800 } 801 802 #ifdef CONFIG_SMP 803 804 static void __hrtick_restart(struct rq *rq) 805 { 806 struct hrtimer *timer = &rq->hrtick_timer; 807 ktime_t time = rq->hrtick_time; 808 809 hrtimer_start(timer, time, HRTIMER_MODE_ABS_PINNED_HARD); 810 } 811 812 /* 813 * called from hardirq (IPI) context 814 */ 815 static void __hrtick_start(void *arg) 816 { 817 struct rq *rq = arg; 818 struct rq_flags rf; 819 820 rq_lock(rq, &rf); 821 __hrtick_restart(rq); 822 rq_unlock(rq, &rf); 823 } 824 825 /* 826 * Called to set the hrtick timer state. 827 * 828 * called with rq->lock held and irqs disabled 829 */ 830 void hrtick_start(struct rq *rq, u64 delay) 831 { 832 struct hrtimer *timer = &rq->hrtick_timer; 833 s64 delta; 834 835 /* 836 * Don't schedule slices shorter than 10000ns, that just 837 * doesn't make sense and can cause timer DoS. 838 */ 839 delta = max_t(s64, delay, 10000LL); 840 rq->hrtick_time = ktime_add_ns(timer->base->get_time(), delta); 841 842 if (rq == this_rq()) 843 __hrtick_restart(rq); 844 else 845 smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd); 846 } 847 848 #else 849 /* 850 * Called to set the hrtick timer state. 851 * 852 * called with rq->lock held and irqs disabled 853 */ 854 void hrtick_start(struct rq *rq, u64 delay) 855 { 856 /* 857 * Don't schedule slices shorter than 10000ns, that just 858 * doesn't make sense. Rely on vruntime for fairness. 859 */ 860 delay = max_t(u64, delay, 10000LL); 861 hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay), 862 HRTIMER_MODE_REL_PINNED_HARD); 863 } 864 865 #endif /* CONFIG_SMP */ 866 867 static void hrtick_rq_init(struct rq *rq) 868 { 869 #ifdef CONFIG_SMP 870 INIT_CSD(&rq->hrtick_csd, __hrtick_start, rq); 871 #endif 872 hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); 873 rq->hrtick_timer.function = hrtick; 874 } 875 #else /* CONFIG_SCHED_HRTICK */ 876 static inline void hrtick_clear(struct rq *rq) 877 { 878 } 879 880 static inline void hrtick_rq_init(struct rq *rq) 881 { 882 } 883 #endif /* CONFIG_SCHED_HRTICK */ 884 885 /* 886 * cmpxchg based fetch_or, macro so it works for different integer types 887 */ 888 #define fetch_or(ptr, mask) \ 889 ({ \ 890 typeof(ptr) _ptr = (ptr); \ 891 typeof(mask) _mask = (mask); \ 892 typeof(*_ptr) _val = *_ptr; \ 893 \ 894 do { \ 895 } while (!try_cmpxchg(_ptr, &_val, _val | _mask)); \ 896 _val; \ 897 }) 898 899 #if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG) 900 /* 901 * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG, 902 * this avoids any races wrt polling state changes and thereby avoids 903 * spurious IPIs. 904 */ 905 static inline bool set_nr_and_not_polling(struct task_struct *p) 906 { 907 struct thread_info *ti = task_thread_info(p); 908 return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG); 909 } 910 911 /* 912 * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set. 913 * 914 * If this returns true, then the idle task promises to call 915 * sched_ttwu_pending() and reschedule soon. 916 */ 917 static bool set_nr_if_polling(struct task_struct *p) 918 { 919 struct thread_info *ti = task_thread_info(p); 920 typeof(ti->flags) val = READ_ONCE(ti->flags); 921 922 for (;;) { 923 if (!(val & _TIF_POLLING_NRFLAG)) 924 return false; 925 if (val & _TIF_NEED_RESCHED) 926 return true; 927 if (try_cmpxchg(&ti->flags, &val, val | _TIF_NEED_RESCHED)) 928 break; 929 } 930 return true; 931 } 932 933 #else 934 static inline bool set_nr_and_not_polling(struct task_struct *p) 935 { 936 set_tsk_need_resched(p); 937 return true; 938 } 939 940 #ifdef CONFIG_SMP 941 static inline bool set_nr_if_polling(struct task_struct *p) 942 { 943 return false; 944 } 945 #endif 946 #endif 947 948 static bool __wake_q_add(struct wake_q_head *head, struct task_struct *task) 949 { 950 struct wake_q_node *node = &task->wake_q; 951 952 /* 953 * Atomically grab the task, if ->wake_q is !nil already it means 954 * it's already queued (either by us or someone else) and will get the 955 * wakeup due to that. 956 * 957 * In order to ensure that a pending wakeup will observe our pending 958 * state, even in the failed case, an explicit smp_mb() must be used. 959 */ 960 smp_mb__before_atomic(); 961 if (unlikely(cmpxchg_relaxed(&node->next, NULL, WAKE_Q_TAIL))) 962 return false; 963 964 /* 965 * The head is context local, there can be no concurrency. 966 */ 967 *head->lastp = node; 968 head->lastp = &node->next; 969 return true; 970 } 971 972 /** 973 * wake_q_add() - queue a wakeup for 'later' waking. 974 * @head: the wake_q_head to add @task to 975 * @task: the task to queue for 'later' wakeup 976 * 977 * Queue a task for later wakeup, most likely by the wake_up_q() call in the 978 * same context, _HOWEVER_ this is not guaranteed, the wakeup can come 979 * instantly. 980 * 981 * This function must be used as-if it were wake_up_process(); IOW the task 982 * must be ready to be woken at this location. 983 */ 984 void wake_q_add(struct wake_q_head *head, struct task_struct *task) 985 { 986 if (__wake_q_add(head, task)) 987 get_task_struct(task); 988 } 989 990 /** 991 * wake_q_add_safe() - safely queue a wakeup for 'later' waking. 992 * @head: the wake_q_head to add @task to 993 * @task: the task to queue for 'later' wakeup 994 * 995 * Queue a task for later wakeup, most likely by the wake_up_q() call in the 996 * same context, _HOWEVER_ this is not guaranteed, the wakeup can come 997 * instantly. 998 * 999 * This function must be used as-if it were wake_up_process(); IOW the task 1000 * must be ready to be woken at this location. 1001 * 1002 * This function is essentially a task-safe equivalent to wake_q_add(). Callers 1003 * that already hold reference to @task can call the 'safe' version and trust 1004 * wake_q to do the right thing depending whether or not the @task is already 1005 * queued for wakeup. 1006 */ 1007 void wake_q_add_safe(struct wake_q_head *head, struct task_struct *task) 1008 { 1009 if (!__wake_q_add(head, task)) 1010 put_task_struct(task); 1011 } 1012 1013 void wake_up_q(struct wake_q_head *head) 1014 { 1015 struct wake_q_node *node = head->first; 1016 1017 while (node != WAKE_Q_TAIL) { 1018 struct task_struct *task; 1019 1020 task = container_of(node, struct task_struct, wake_q); 1021 /* Task can safely be re-inserted now: */ 1022 node = node->next; 1023 task->wake_q.next = NULL; 1024 1025 /* 1026 * wake_up_process() executes a full barrier, which pairs with 1027 * the queueing in wake_q_add() so as not to miss wakeups. 1028 */ 1029 wake_up_process(task); 1030 put_task_struct(task); 1031 } 1032 } 1033 1034 /* 1035 * resched_curr - mark rq's current task 'to be rescheduled now'. 1036 * 1037 * On UP this means the setting of the need_resched flag, on SMP it 1038 * might also involve a cross-CPU call to trigger the scheduler on 1039 * the target CPU. 1040 */ 1041 void resched_curr(struct rq *rq) 1042 { 1043 struct task_struct *curr = rq->curr; 1044 int cpu; 1045 1046 lockdep_assert_rq_held(rq); 1047 1048 if (test_tsk_need_resched(curr)) 1049 return; 1050 1051 cpu = cpu_of(rq); 1052 1053 if (cpu == smp_processor_id()) { 1054 set_tsk_need_resched(curr); 1055 set_preempt_need_resched(); 1056 return; 1057 } 1058 1059 if (set_nr_and_not_polling(curr)) 1060 smp_send_reschedule(cpu); 1061 else 1062 trace_sched_wake_idle_without_ipi(cpu); 1063 } 1064 1065 void resched_cpu(int cpu) 1066 { 1067 struct rq *rq = cpu_rq(cpu); 1068 unsigned long flags; 1069 1070 raw_spin_rq_lock_irqsave(rq, flags); 1071 if (cpu_online(cpu) || cpu == smp_processor_id()) 1072 resched_curr(rq); 1073 raw_spin_rq_unlock_irqrestore(rq, flags); 1074 } 1075 1076 #ifdef CONFIG_SMP 1077 #ifdef CONFIG_NO_HZ_COMMON 1078 /* 1079 * In the semi idle case, use the nearest busy CPU for migrating timers 1080 * from an idle CPU. This is good for power-savings. 1081 * 1082 * We don't do similar optimization for completely idle system, as 1083 * selecting an idle CPU will add more delays to the timers than intended 1084 * (as that CPU's timer base may not be uptodate wrt jiffies etc). 1085 */ 1086 int get_nohz_timer_target(void) 1087 { 1088 int i, cpu = smp_processor_id(), default_cpu = -1; 1089 struct sched_domain *sd; 1090 const struct cpumask *hk_mask; 1091 1092 if (housekeeping_cpu(cpu, HK_TYPE_TIMER)) { 1093 if (!idle_cpu(cpu)) 1094 return cpu; 1095 default_cpu = cpu; 1096 } 1097 1098 hk_mask = housekeeping_cpumask(HK_TYPE_TIMER); 1099 1100 guard(rcu)(); 1101 1102 for_each_domain(cpu, sd) { 1103 for_each_cpu_and(i, sched_domain_span(sd), hk_mask) { 1104 if (cpu == i) 1105 continue; 1106 1107 if (!idle_cpu(i)) 1108 return i; 1109 } 1110 } 1111 1112 if (default_cpu == -1) 1113 default_cpu = housekeeping_any_cpu(HK_TYPE_TIMER); 1114 1115 return default_cpu; 1116 } 1117 1118 /* 1119 * When add_timer_on() enqueues a timer into the timer wheel of an 1120 * idle CPU then this timer might expire before the next timer event 1121 * which is scheduled to wake up that CPU. In case of a completely 1122 * idle system the next event might even be infinite time into the 1123 * future. wake_up_idle_cpu() ensures that the CPU is woken up and 1124 * leaves the inner idle loop so the newly added timer is taken into 1125 * account when the CPU goes back to idle and evaluates the timer 1126 * wheel for the next timer event. 1127 */ 1128 static void wake_up_idle_cpu(int cpu) 1129 { 1130 struct rq *rq = cpu_rq(cpu); 1131 1132 if (cpu == smp_processor_id()) 1133 return; 1134 1135 if (set_nr_and_not_polling(rq->idle)) 1136 smp_send_reschedule(cpu); 1137 else 1138 trace_sched_wake_idle_without_ipi(cpu); 1139 } 1140 1141 static bool wake_up_full_nohz_cpu(int cpu) 1142 { 1143 /* 1144 * We just need the target to call irq_exit() and re-evaluate 1145 * the next tick. The nohz full kick at least implies that. 1146 * If needed we can still optimize that later with an 1147 * empty IRQ. 1148 */ 1149 if (cpu_is_offline(cpu)) 1150 return true; /* Don't try to wake offline CPUs. */ 1151 if (tick_nohz_full_cpu(cpu)) { 1152 if (cpu != smp_processor_id() || 1153 tick_nohz_tick_stopped()) 1154 tick_nohz_full_kick_cpu(cpu); 1155 return true; 1156 } 1157 1158 return false; 1159 } 1160 1161 /* 1162 * Wake up the specified CPU. If the CPU is going offline, it is the 1163 * caller's responsibility to deal with the lost wakeup, for example, 1164 * by hooking into the CPU_DEAD notifier like timers and hrtimers do. 1165 */ 1166 void wake_up_nohz_cpu(int cpu) 1167 { 1168 if (!wake_up_full_nohz_cpu(cpu)) 1169 wake_up_idle_cpu(cpu); 1170 } 1171 1172 static void nohz_csd_func(void *info) 1173 { 1174 struct rq *rq = info; 1175 int cpu = cpu_of(rq); 1176 unsigned int flags; 1177 1178 /* 1179 * Release the rq::nohz_csd. 1180 */ 1181 flags = atomic_fetch_andnot(NOHZ_KICK_MASK | NOHZ_NEWILB_KICK, nohz_flags(cpu)); 1182 WARN_ON(!(flags & NOHZ_KICK_MASK)); 1183 1184 rq->idle_balance = idle_cpu(cpu); 1185 if (rq->idle_balance && !need_resched()) { 1186 rq->nohz_idle_balance = flags; 1187 raise_softirq_irqoff(SCHED_SOFTIRQ); 1188 } 1189 } 1190 1191 #endif /* CONFIG_NO_HZ_COMMON */ 1192 1193 #ifdef CONFIG_NO_HZ_FULL 1194 static inline bool __need_bw_check(struct rq *rq, struct task_struct *p) 1195 { 1196 if (rq->nr_running != 1) 1197 return false; 1198 1199 if (p->sched_class != &fair_sched_class) 1200 return false; 1201 1202 if (!task_on_rq_queued(p)) 1203 return false; 1204 1205 return true; 1206 } 1207 1208 bool sched_can_stop_tick(struct rq *rq) 1209 { 1210 int fifo_nr_running; 1211 1212 /* Deadline tasks, even if single, need the tick */ 1213 if (rq->dl.dl_nr_running) 1214 return false; 1215 1216 /* 1217 * If there are more than one RR tasks, we need the tick to affect the 1218 * actual RR behaviour. 1219 */ 1220 if (rq->rt.rr_nr_running) { 1221 if (rq->rt.rr_nr_running == 1) 1222 return true; 1223 else 1224 return false; 1225 } 1226 1227 /* 1228 * If there's no RR tasks, but FIFO tasks, we can skip the tick, no 1229 * forced preemption between FIFO tasks. 1230 */ 1231 fifo_nr_running = rq->rt.rt_nr_running - rq->rt.rr_nr_running; 1232 if (fifo_nr_running) 1233 return true; 1234 1235 /* 1236 * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left; 1237 * if there's more than one we need the tick for involuntary 1238 * preemption. 1239 */ 1240 if (rq->nr_running > 1) 1241 return false; 1242 1243 /* 1244 * If there is one task and it has CFS runtime bandwidth constraints 1245 * and it's on the cpu now we don't want to stop the tick. 1246 * This check prevents clearing the bit if a newly enqueued task here is 1247 * dequeued by migrating while the constrained task continues to run. 1248 * E.g. going from 2->1 without going through pick_next_task(). 1249 */ 1250 if (sched_feat(HZ_BW) && __need_bw_check(rq, rq->curr)) { 1251 if (cfs_task_bw_constrained(rq->curr)) 1252 return false; 1253 } 1254 1255 return true; 1256 } 1257 #endif /* CONFIG_NO_HZ_FULL */ 1258 #endif /* CONFIG_SMP */ 1259 1260 #if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \ 1261 (defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH))) 1262 /* 1263 * Iterate task_group tree rooted at *from, calling @down when first entering a 1264 * node and @up when leaving it for the final time. 1265 * 1266 * Caller must hold rcu_lock or sufficient equivalent. 1267 */ 1268 int walk_tg_tree_from(struct task_group *from, 1269 tg_visitor down, tg_visitor up, void *data) 1270 { 1271 struct task_group *parent, *child; 1272 int ret; 1273 1274 parent = from; 1275 1276 down: 1277 ret = (*down)(parent, data); 1278 if (ret) 1279 goto out; 1280 list_for_each_entry_rcu(child, &parent->children, siblings) { 1281 parent = child; 1282 goto down; 1283 1284 up: 1285 continue; 1286 } 1287 ret = (*up)(parent, data); 1288 if (ret || parent == from) 1289 goto out; 1290 1291 child = parent; 1292 parent = parent->parent; 1293 if (parent) 1294 goto up; 1295 out: 1296 return ret; 1297 } 1298 1299 int tg_nop(struct task_group *tg, void *data) 1300 { 1301 return 0; 1302 } 1303 #endif 1304 1305 static void set_load_weight(struct task_struct *p, bool update_load) 1306 { 1307 int prio = p->static_prio - MAX_RT_PRIO; 1308 struct load_weight *load = &p->se.load; 1309 1310 /* 1311 * SCHED_IDLE tasks get minimal weight: 1312 */ 1313 if (task_has_idle_policy(p)) { 1314 load->weight = scale_load(WEIGHT_IDLEPRIO); 1315 load->inv_weight = WMULT_IDLEPRIO; 1316 return; 1317 } 1318 1319 /* 1320 * SCHED_OTHER tasks have to update their load when changing their 1321 * weight 1322 */ 1323 if (update_load && p->sched_class == &fair_sched_class) { 1324 reweight_task(p, prio); 1325 } else { 1326 load->weight = scale_load(sched_prio_to_weight[prio]); 1327 load->inv_weight = sched_prio_to_wmult[prio]; 1328 } 1329 } 1330 1331 #ifdef CONFIG_UCLAMP_TASK 1332 /* 1333 * Serializes updates of utilization clamp values 1334 * 1335 * The (slow-path) user-space triggers utilization clamp value updates which 1336 * can require updates on (fast-path) scheduler's data structures used to 1337 * support enqueue/dequeue operations. 1338 * While the per-CPU rq lock protects fast-path update operations, user-space 1339 * requests are serialized using a mutex to reduce the risk of conflicting 1340 * updates or API abuses. 1341 */ 1342 static DEFINE_MUTEX(uclamp_mutex); 1343 1344 /* Max allowed minimum utilization */ 1345 static unsigned int __maybe_unused sysctl_sched_uclamp_util_min = SCHED_CAPACITY_SCALE; 1346 1347 /* Max allowed maximum utilization */ 1348 static unsigned int __maybe_unused sysctl_sched_uclamp_util_max = SCHED_CAPACITY_SCALE; 1349 1350 /* 1351 * By default RT tasks run at the maximum performance point/capacity of the 1352 * system. Uclamp enforces this by always setting UCLAMP_MIN of RT tasks to 1353 * SCHED_CAPACITY_SCALE. 1354 * 1355 * This knob allows admins to change the default behavior when uclamp is being 1356 * used. In battery powered devices, particularly, running at the maximum 1357 * capacity and frequency will increase energy consumption and shorten the 1358 * battery life. 1359 * 1360 * This knob only affects RT tasks that their uclamp_se->user_defined == false. 1361 * 1362 * This knob will not override the system default sched_util_clamp_min defined 1363 * above. 1364 */ 1365 static unsigned int sysctl_sched_uclamp_util_min_rt_default = SCHED_CAPACITY_SCALE; 1366 1367 /* All clamps are required to be less or equal than these values */ 1368 static struct uclamp_se uclamp_default[UCLAMP_CNT]; 1369 1370 /* 1371 * This static key is used to reduce the uclamp overhead in the fast path. It 1372 * primarily disables the call to uclamp_rq_{inc, dec}() in 1373 * enqueue/dequeue_task(). 1374 * 1375 * This allows users to continue to enable uclamp in their kernel config with 1376 * minimum uclamp overhead in the fast path. 1377 * 1378 * As soon as userspace modifies any of the uclamp knobs, the static key is 1379 * enabled, since we have an actual users that make use of uclamp 1380 * functionality. 1381 * 1382 * The knobs that would enable this static key are: 1383 * 1384 * * A task modifying its uclamp value with sched_setattr(). 1385 * * An admin modifying the sysctl_sched_uclamp_{min, max} via procfs. 1386 * * An admin modifying the cgroup cpu.uclamp.{min, max} 1387 */ 1388 DEFINE_STATIC_KEY_FALSE(sched_uclamp_used); 1389 1390 /* Integer rounded range for each bucket */ 1391 #define UCLAMP_BUCKET_DELTA DIV_ROUND_CLOSEST(SCHED_CAPACITY_SCALE, UCLAMP_BUCKETS) 1392 1393 #define for_each_clamp_id(clamp_id) \ 1394 for ((clamp_id) = 0; (clamp_id) < UCLAMP_CNT; (clamp_id)++) 1395 1396 static inline unsigned int uclamp_bucket_id(unsigned int clamp_value) 1397 { 1398 return min_t(unsigned int, clamp_value / UCLAMP_BUCKET_DELTA, UCLAMP_BUCKETS - 1); 1399 } 1400 1401 static inline unsigned int uclamp_none(enum uclamp_id clamp_id) 1402 { 1403 if (clamp_id == UCLAMP_MIN) 1404 return 0; 1405 return SCHED_CAPACITY_SCALE; 1406 } 1407 1408 static inline void uclamp_se_set(struct uclamp_se *uc_se, 1409 unsigned int value, bool user_defined) 1410 { 1411 uc_se->value = value; 1412 uc_se->bucket_id = uclamp_bucket_id(value); 1413 uc_se->user_defined = user_defined; 1414 } 1415 1416 static inline unsigned int 1417 uclamp_idle_value(struct rq *rq, enum uclamp_id clamp_id, 1418 unsigned int clamp_value) 1419 { 1420 /* 1421 * Avoid blocked utilization pushing up the frequency when we go 1422 * idle (which drops the max-clamp) by retaining the last known 1423 * max-clamp. 1424 */ 1425 if (clamp_id == UCLAMP_MAX) { 1426 rq->uclamp_flags |= UCLAMP_FLAG_IDLE; 1427 return clamp_value; 1428 } 1429 1430 return uclamp_none(UCLAMP_MIN); 1431 } 1432 1433 static inline void uclamp_idle_reset(struct rq *rq, enum uclamp_id clamp_id, 1434 unsigned int clamp_value) 1435 { 1436 /* Reset max-clamp retention only on idle exit */ 1437 if (!(rq->uclamp_flags & UCLAMP_FLAG_IDLE)) 1438 return; 1439 1440 uclamp_rq_set(rq, clamp_id, clamp_value); 1441 } 1442 1443 static inline 1444 unsigned int uclamp_rq_max_value(struct rq *rq, enum uclamp_id clamp_id, 1445 unsigned int clamp_value) 1446 { 1447 struct uclamp_bucket *bucket = rq->uclamp[clamp_id].bucket; 1448 int bucket_id = UCLAMP_BUCKETS - 1; 1449 1450 /* 1451 * Since both min and max clamps are max aggregated, find the 1452 * top most bucket with tasks in. 1453 */ 1454 for ( ; bucket_id >= 0; bucket_id--) { 1455 if (!bucket[bucket_id].tasks) 1456 continue; 1457 return bucket[bucket_id].value; 1458 } 1459 1460 /* No tasks -- default clamp values */ 1461 return uclamp_idle_value(rq, clamp_id, clamp_value); 1462 } 1463 1464 static void __uclamp_update_util_min_rt_default(struct task_struct *p) 1465 { 1466 unsigned int default_util_min; 1467 struct uclamp_se *uc_se; 1468 1469 lockdep_assert_held(&p->pi_lock); 1470 1471 uc_se = &p->uclamp_req[UCLAMP_MIN]; 1472 1473 /* Only sync if user didn't override the default */ 1474 if (uc_se->user_defined) 1475 return; 1476 1477 default_util_min = sysctl_sched_uclamp_util_min_rt_default; 1478 uclamp_se_set(uc_se, default_util_min, false); 1479 } 1480 1481 static void uclamp_update_util_min_rt_default(struct task_struct *p) 1482 { 1483 struct rq_flags rf; 1484 struct rq *rq; 1485 1486 if (!rt_task(p)) 1487 return; 1488 1489 /* Protect updates to p->uclamp_* */ 1490 rq = task_rq_lock(p, &rf); 1491 __uclamp_update_util_min_rt_default(p); 1492 task_rq_unlock(rq, p, &rf); 1493 } 1494 1495 static inline struct uclamp_se 1496 uclamp_tg_restrict(struct task_struct *p, enum uclamp_id clamp_id) 1497 { 1498 /* Copy by value as we could modify it */ 1499 struct uclamp_se uc_req = p->uclamp_req[clamp_id]; 1500 #ifdef CONFIG_UCLAMP_TASK_GROUP 1501 unsigned int tg_min, tg_max, value; 1502 1503 /* 1504 * Tasks in autogroups or root task group will be 1505 * restricted by system defaults. 1506 */ 1507 if (task_group_is_autogroup(task_group(p))) 1508 return uc_req; 1509 if (task_group(p) == &root_task_group) 1510 return uc_req; 1511 1512 tg_min = task_group(p)->uclamp[UCLAMP_MIN].value; 1513 tg_max = task_group(p)->uclamp[UCLAMP_MAX].value; 1514 value = uc_req.value; 1515 value = clamp(value, tg_min, tg_max); 1516 uclamp_se_set(&uc_req, value, false); 1517 #endif 1518 1519 return uc_req; 1520 } 1521 1522 /* 1523 * The effective clamp bucket index of a task depends on, by increasing 1524 * priority: 1525 * - the task specific clamp value, when explicitly requested from userspace 1526 * - the task group effective clamp value, for tasks not either in the root 1527 * group or in an autogroup 1528 * - the system default clamp value, defined by the sysadmin 1529 */ 1530 static inline struct uclamp_se 1531 uclamp_eff_get(struct task_struct *p, enum uclamp_id clamp_id) 1532 { 1533 struct uclamp_se uc_req = uclamp_tg_restrict(p, clamp_id); 1534 struct uclamp_se uc_max = uclamp_default[clamp_id]; 1535 1536 /* System default restrictions always apply */ 1537 if (unlikely(uc_req.value > uc_max.value)) 1538 return uc_max; 1539 1540 return uc_req; 1541 } 1542 1543 unsigned long uclamp_eff_value(struct task_struct *p, enum uclamp_id clamp_id) 1544 { 1545 struct uclamp_se uc_eff; 1546 1547 /* Task currently refcounted: use back-annotated (effective) value */ 1548 if (p->uclamp[clamp_id].active) 1549 return (unsigned long)p->uclamp[clamp_id].value; 1550 1551 uc_eff = uclamp_eff_get(p, clamp_id); 1552 1553 return (unsigned long)uc_eff.value; 1554 } 1555 1556 /* 1557 * When a task is enqueued on a rq, the clamp bucket currently defined by the 1558 * task's uclamp::bucket_id is refcounted on that rq. This also immediately 1559 * updates the rq's clamp value if required. 1560 * 1561 * Tasks can have a task-specific value requested from user-space, track 1562 * within each bucket the maximum value for tasks refcounted in it. 1563 * This "local max aggregation" allows to track the exact "requested" value 1564 * for each bucket when all its RUNNABLE tasks require the same clamp. 1565 */ 1566 static inline void uclamp_rq_inc_id(struct rq *rq, struct task_struct *p, 1567 enum uclamp_id clamp_id) 1568 { 1569 struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id]; 1570 struct uclamp_se *uc_se = &p->uclamp[clamp_id]; 1571 struct uclamp_bucket *bucket; 1572 1573 lockdep_assert_rq_held(rq); 1574 1575 /* Update task effective clamp */ 1576 p->uclamp[clamp_id] = uclamp_eff_get(p, clamp_id); 1577 1578 bucket = &uc_rq->bucket[uc_se->bucket_id]; 1579 bucket->tasks++; 1580 uc_se->active = true; 1581 1582 uclamp_idle_reset(rq, clamp_id, uc_se->value); 1583 1584 /* 1585 * Local max aggregation: rq buckets always track the max 1586 * "requested" clamp value of its RUNNABLE tasks. 1587 */ 1588 if (bucket->tasks == 1 || uc_se->value > bucket->value) 1589 bucket->value = uc_se->value; 1590 1591 if (uc_se->value > uclamp_rq_get(rq, clamp_id)) 1592 uclamp_rq_set(rq, clamp_id, uc_se->value); 1593 } 1594 1595 /* 1596 * When a task is dequeued from a rq, the clamp bucket refcounted by the task 1597 * is released. If this is the last task reference counting the rq's max 1598 * active clamp value, then the rq's clamp value is updated. 1599 * 1600 * Both refcounted tasks and rq's cached clamp values are expected to be 1601 * always valid. If it's detected they are not, as defensive programming, 1602 * enforce the expected state and warn. 1603 */ 1604 static inline void uclamp_rq_dec_id(struct rq *rq, struct task_struct *p, 1605 enum uclamp_id clamp_id) 1606 { 1607 struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id]; 1608 struct uclamp_se *uc_se = &p->uclamp[clamp_id]; 1609 struct uclamp_bucket *bucket; 1610 unsigned int bkt_clamp; 1611 unsigned int rq_clamp; 1612 1613 lockdep_assert_rq_held(rq); 1614 1615 /* 1616 * If sched_uclamp_used was enabled after task @p was enqueued, 1617 * we could end up with unbalanced call to uclamp_rq_dec_id(). 1618 * 1619 * In this case the uc_se->active flag should be false since no uclamp 1620 * accounting was performed at enqueue time and we can just return 1621 * here. 1622 * 1623 * Need to be careful of the following enqueue/dequeue ordering 1624 * problem too 1625 * 1626 * enqueue(taskA) 1627 * // sched_uclamp_used gets enabled 1628 * enqueue(taskB) 1629 * dequeue(taskA) 1630 * // Must not decrement bucket->tasks here 1631 * dequeue(taskB) 1632 * 1633 * where we could end up with stale data in uc_se and 1634 * bucket[uc_se->bucket_id]. 1635 * 1636 * The following check here eliminates the possibility of such race. 1637 */ 1638 if (unlikely(!uc_se->active)) 1639 return; 1640 1641 bucket = &uc_rq->bucket[uc_se->bucket_id]; 1642 1643 SCHED_WARN_ON(!bucket->tasks); 1644 if (likely(bucket->tasks)) 1645 bucket->tasks--; 1646 1647 uc_se->active = false; 1648 1649 /* 1650 * Keep "local max aggregation" simple and accept to (possibly) 1651 * overboost some RUNNABLE tasks in the same bucket. 1652 * The rq clamp bucket value is reset to its base value whenever 1653 * there are no more RUNNABLE tasks refcounting it. 1654 */ 1655 if (likely(bucket->tasks)) 1656 return; 1657 1658 rq_clamp = uclamp_rq_get(rq, clamp_id); 1659 /* 1660 * Defensive programming: this should never happen. If it happens, 1661 * e.g. due to future modification, warn and fixup the expected value. 1662 */ 1663 SCHED_WARN_ON(bucket->value > rq_clamp); 1664 if (bucket->value >= rq_clamp) { 1665 bkt_clamp = uclamp_rq_max_value(rq, clamp_id, uc_se->value); 1666 uclamp_rq_set(rq, clamp_id, bkt_clamp); 1667 } 1668 } 1669 1670 static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p) 1671 { 1672 enum uclamp_id clamp_id; 1673 1674 /* 1675 * Avoid any overhead until uclamp is actually used by the userspace. 1676 * 1677 * The condition is constructed such that a NOP is generated when 1678 * sched_uclamp_used is disabled. 1679 */ 1680 if (!static_branch_unlikely(&sched_uclamp_used)) 1681 return; 1682 1683 if (unlikely(!p->sched_class->uclamp_enabled)) 1684 return; 1685 1686 for_each_clamp_id(clamp_id) 1687 uclamp_rq_inc_id(rq, p, clamp_id); 1688 1689 /* Reset clamp idle holding when there is one RUNNABLE task */ 1690 if (rq->uclamp_flags & UCLAMP_FLAG_IDLE) 1691 rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE; 1692 } 1693 1694 static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) 1695 { 1696 enum uclamp_id clamp_id; 1697 1698 /* 1699 * Avoid any overhead until uclamp is actually used by the userspace. 1700 * 1701 * The condition is constructed such that a NOP is generated when 1702 * sched_uclamp_used is disabled. 1703 */ 1704 if (!static_branch_unlikely(&sched_uclamp_used)) 1705 return; 1706 1707 if (unlikely(!p->sched_class->uclamp_enabled)) 1708 return; 1709 1710 for_each_clamp_id(clamp_id) 1711 uclamp_rq_dec_id(rq, p, clamp_id); 1712 } 1713 1714 static inline void uclamp_rq_reinc_id(struct rq *rq, struct task_struct *p, 1715 enum uclamp_id clamp_id) 1716 { 1717 if (!p->uclamp[clamp_id].active) 1718 return; 1719 1720 uclamp_rq_dec_id(rq, p, clamp_id); 1721 uclamp_rq_inc_id(rq, p, clamp_id); 1722 1723 /* 1724 * Make sure to clear the idle flag if we've transiently reached 0 1725 * active tasks on rq. 1726 */ 1727 if (clamp_id == UCLAMP_MAX && (rq->uclamp_flags & UCLAMP_FLAG_IDLE)) 1728 rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE; 1729 } 1730 1731 static inline void 1732 uclamp_update_active(struct task_struct *p) 1733 { 1734 enum uclamp_id clamp_id; 1735 struct rq_flags rf; 1736 struct rq *rq; 1737 1738 /* 1739 * Lock the task and the rq where the task is (or was) queued. 1740 * 1741 * We might lock the (previous) rq of a !RUNNABLE task, but that's the 1742 * price to pay to safely serialize util_{min,max} updates with 1743 * enqueues, dequeues and migration operations. 1744 * This is the same locking schema used by __set_cpus_allowed_ptr(). 1745 */ 1746 rq = task_rq_lock(p, &rf); 1747 1748 /* 1749 * Setting the clamp bucket is serialized by task_rq_lock(). 1750 * If the task is not yet RUNNABLE and its task_struct is not 1751 * affecting a valid clamp bucket, the next time it's enqueued, 1752 * it will already see the updated clamp bucket value. 1753 */ 1754 for_each_clamp_id(clamp_id) 1755 uclamp_rq_reinc_id(rq, p, clamp_id); 1756 1757 task_rq_unlock(rq, p, &rf); 1758 } 1759 1760 #ifdef CONFIG_UCLAMP_TASK_GROUP 1761 static inline void 1762 uclamp_update_active_tasks(struct cgroup_subsys_state *css) 1763 { 1764 struct css_task_iter it; 1765 struct task_struct *p; 1766 1767 css_task_iter_start(css, 0, &it); 1768 while ((p = css_task_iter_next(&it))) 1769 uclamp_update_active(p); 1770 css_task_iter_end(&it); 1771 } 1772 1773 static void cpu_util_update_eff(struct cgroup_subsys_state *css); 1774 #endif 1775 1776 #ifdef CONFIG_SYSCTL 1777 #ifdef CONFIG_UCLAMP_TASK 1778 #ifdef CONFIG_UCLAMP_TASK_GROUP 1779 static void uclamp_update_root_tg(void) 1780 { 1781 struct task_group *tg = &root_task_group; 1782 1783 uclamp_se_set(&tg->uclamp_req[UCLAMP_MIN], 1784 sysctl_sched_uclamp_util_min, false); 1785 uclamp_se_set(&tg->uclamp_req[UCLAMP_MAX], 1786 sysctl_sched_uclamp_util_max, false); 1787 1788 rcu_read_lock(); 1789 cpu_util_update_eff(&root_task_group.css); 1790 rcu_read_unlock(); 1791 } 1792 #else 1793 static void uclamp_update_root_tg(void) { } 1794 #endif 1795 1796 static void uclamp_sync_util_min_rt_default(void) 1797 { 1798 struct task_struct *g, *p; 1799 1800 /* 1801 * copy_process() sysctl_uclamp 1802 * uclamp_min_rt = X; 1803 * write_lock(&tasklist_lock) read_lock(&tasklist_lock) 1804 * // link thread smp_mb__after_spinlock() 1805 * write_unlock(&tasklist_lock) read_unlock(&tasklist_lock); 1806 * sched_post_fork() for_each_process_thread() 1807 * __uclamp_sync_rt() __uclamp_sync_rt() 1808 * 1809 * Ensures that either sched_post_fork() will observe the new 1810 * uclamp_min_rt or for_each_process_thread() will observe the new 1811 * task. 1812 */ 1813 read_lock(&tasklist_lock); 1814 smp_mb__after_spinlock(); 1815 read_unlock(&tasklist_lock); 1816 1817 rcu_read_lock(); 1818 for_each_process_thread(g, p) 1819 uclamp_update_util_min_rt_default(p); 1820 rcu_read_unlock(); 1821 } 1822 1823 static int sysctl_sched_uclamp_handler(struct ctl_table *table, int write, 1824 void *buffer, size_t *lenp, loff_t *ppos) 1825 { 1826 bool update_root_tg = false; 1827 int old_min, old_max, old_min_rt; 1828 int result; 1829 1830 guard(mutex)(&uclamp_mutex); 1831 1832 old_min = sysctl_sched_uclamp_util_min; 1833 old_max = sysctl_sched_uclamp_util_max; 1834 old_min_rt = sysctl_sched_uclamp_util_min_rt_default; 1835 1836 result = proc_dointvec(table, write, buffer, lenp, ppos); 1837 if (result) 1838 goto undo; 1839 if (!write) 1840 return 0; 1841 1842 if (sysctl_sched_uclamp_util_min > sysctl_sched_uclamp_util_max || 1843 sysctl_sched_uclamp_util_max > SCHED_CAPACITY_SCALE || 1844 sysctl_sched_uclamp_util_min_rt_default > SCHED_CAPACITY_SCALE) { 1845 1846 result = -EINVAL; 1847 goto undo; 1848 } 1849 1850 if (old_min != sysctl_sched_uclamp_util_min) { 1851 uclamp_se_set(&uclamp_default[UCLAMP_MIN], 1852 sysctl_sched_uclamp_util_min, false); 1853 update_root_tg = true; 1854 } 1855 if (old_max != sysctl_sched_uclamp_util_max) { 1856 uclamp_se_set(&uclamp_default[UCLAMP_MAX], 1857 sysctl_sched_uclamp_util_max, false); 1858 update_root_tg = true; 1859 } 1860 1861 if (update_root_tg) { 1862 static_branch_enable(&sched_uclamp_used); 1863 uclamp_update_root_tg(); 1864 } 1865 1866 if (old_min_rt != sysctl_sched_uclamp_util_min_rt_default) { 1867 static_branch_enable(&sched_uclamp_used); 1868 uclamp_sync_util_min_rt_default(); 1869 } 1870 1871 /* 1872 * We update all RUNNABLE tasks only when task groups are in use. 1873 * Otherwise, keep it simple and do just a lazy update at each next 1874 * task enqueue time. 1875 */ 1876 return 0; 1877 1878 undo: 1879 sysctl_sched_uclamp_util_min = old_min; 1880 sysctl_sched_uclamp_util_max = old_max; 1881 sysctl_sched_uclamp_util_min_rt_default = old_min_rt; 1882 return result; 1883 } 1884 #endif 1885 #endif 1886 1887 static int uclamp_validate(struct task_struct *p, 1888 const struct sched_attr *attr) 1889 { 1890 int util_min = p->uclamp_req[UCLAMP_MIN].value; 1891 int util_max = p->uclamp_req[UCLAMP_MAX].value; 1892 1893 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MIN) { 1894 util_min = attr->sched_util_min; 1895 1896 if (util_min + 1 > SCHED_CAPACITY_SCALE + 1) 1897 return -EINVAL; 1898 } 1899 1900 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MAX) { 1901 util_max = attr->sched_util_max; 1902 1903 if (util_max + 1 > SCHED_CAPACITY_SCALE + 1) 1904 return -EINVAL; 1905 } 1906 1907 if (util_min != -1 && util_max != -1 && util_min > util_max) 1908 return -EINVAL; 1909 1910 /* 1911 * We have valid uclamp attributes; make sure uclamp is enabled. 1912 * 1913 * We need to do that here, because enabling static branches is a 1914 * blocking operation which obviously cannot be done while holding 1915 * scheduler locks. 1916 */ 1917 static_branch_enable(&sched_uclamp_used); 1918 1919 return 0; 1920 } 1921 1922 static bool uclamp_reset(const struct sched_attr *attr, 1923 enum uclamp_id clamp_id, 1924 struct uclamp_se *uc_se) 1925 { 1926 /* Reset on sched class change for a non user-defined clamp value. */ 1927 if (likely(!(attr->sched_flags & SCHED_FLAG_UTIL_CLAMP)) && 1928 !uc_se->user_defined) 1929 return true; 1930 1931 /* Reset on sched_util_{min,max} == -1. */ 1932 if (clamp_id == UCLAMP_MIN && 1933 attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MIN && 1934 attr->sched_util_min == -1) { 1935 return true; 1936 } 1937 1938 if (clamp_id == UCLAMP_MAX && 1939 attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MAX && 1940 attr->sched_util_max == -1) { 1941 return true; 1942 } 1943 1944 return false; 1945 } 1946 1947 static void __setscheduler_uclamp(struct task_struct *p, 1948 const struct sched_attr *attr) 1949 { 1950 enum uclamp_id clamp_id; 1951 1952 for_each_clamp_id(clamp_id) { 1953 struct uclamp_se *uc_se = &p->uclamp_req[clamp_id]; 1954 unsigned int value; 1955 1956 if (!uclamp_reset(attr, clamp_id, uc_se)) 1957 continue; 1958 1959 /* 1960 * RT by default have a 100% boost value that could be modified 1961 * at runtime. 1962 */ 1963 if (unlikely(rt_task(p) && clamp_id == UCLAMP_MIN)) 1964 value = sysctl_sched_uclamp_util_min_rt_default; 1965 else 1966 value = uclamp_none(clamp_id); 1967 1968 uclamp_se_set(uc_se, value, false); 1969 1970 } 1971 1972 if (likely(!(attr->sched_flags & SCHED_FLAG_UTIL_CLAMP))) 1973 return; 1974 1975 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MIN && 1976 attr->sched_util_min != -1) { 1977 uclamp_se_set(&p->uclamp_req[UCLAMP_MIN], 1978 attr->sched_util_min, true); 1979 } 1980 1981 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MAX && 1982 attr->sched_util_max != -1) { 1983 uclamp_se_set(&p->uclamp_req[UCLAMP_MAX], 1984 attr->sched_util_max, true); 1985 } 1986 } 1987 1988 static void uclamp_fork(struct task_struct *p) 1989 { 1990 enum uclamp_id clamp_id; 1991 1992 /* 1993 * We don't need to hold task_rq_lock() when updating p->uclamp_* here 1994 * as the task is still at its early fork stages. 1995 */ 1996 for_each_clamp_id(clamp_id) 1997 p->uclamp[clamp_id].active = false; 1998 1999 if (likely(!p->sched_reset_on_fork)) 2000 return; 2001 2002 for_each_clamp_id(clamp_id) { 2003 uclamp_se_set(&p->uclamp_req[clamp_id], 2004 uclamp_none(clamp_id), false); 2005 } 2006 } 2007 2008 static void uclamp_post_fork(struct task_struct *p) 2009 { 2010 uclamp_update_util_min_rt_default(p); 2011 } 2012 2013 static void __init init_uclamp_rq(struct rq *rq) 2014 { 2015 enum uclamp_id clamp_id; 2016 struct uclamp_rq *uc_rq = rq->uclamp; 2017 2018 for_each_clamp_id(clamp_id) { 2019 uc_rq[clamp_id] = (struct uclamp_rq) { 2020 .value = uclamp_none(clamp_id) 2021 }; 2022 } 2023 2024 rq->uclamp_flags = UCLAMP_FLAG_IDLE; 2025 } 2026 2027 static void __init init_uclamp(void) 2028 { 2029 struct uclamp_se uc_max = {}; 2030 enum uclamp_id clamp_id; 2031 int cpu; 2032 2033 for_each_possible_cpu(cpu) 2034 init_uclamp_rq(cpu_rq(cpu)); 2035 2036 for_each_clamp_id(clamp_id) { 2037 uclamp_se_set(&init_task.uclamp_req[clamp_id], 2038 uclamp_none(clamp_id), false); 2039 } 2040 2041 /* System defaults allow max clamp values for both indexes */ 2042 uclamp_se_set(&uc_max, uclamp_none(UCLAMP_MAX), false); 2043 for_each_clamp_id(clamp_id) { 2044 uclamp_default[clamp_id] = uc_max; 2045 #ifdef CONFIG_UCLAMP_TASK_GROUP 2046 root_task_group.uclamp_req[clamp_id] = uc_max; 2047 root_task_group.uclamp[clamp_id] = uc_max; 2048 #endif 2049 } 2050 } 2051 2052 #else /* CONFIG_UCLAMP_TASK */ 2053 static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p) { } 2054 static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) { } 2055 static inline int uclamp_validate(struct task_struct *p, 2056 const struct sched_attr *attr) 2057 { 2058 return -EOPNOTSUPP; 2059 } 2060 static void __setscheduler_uclamp(struct task_struct *p, 2061 const struct sched_attr *attr) { } 2062 static inline void uclamp_fork(struct task_struct *p) { } 2063 static inline void uclamp_post_fork(struct task_struct *p) { } 2064 static inline void init_uclamp(void) { } 2065 #endif /* CONFIG_UCLAMP_TASK */ 2066 2067 bool sched_task_on_rq(struct task_struct *p) 2068 { 2069 return task_on_rq_queued(p); 2070 } 2071 2072 unsigned long get_wchan(struct task_struct *p) 2073 { 2074 unsigned long ip = 0; 2075 unsigned int state; 2076 2077 if (!p || p == current) 2078 return 0; 2079 2080 /* Only get wchan if task is blocked and we can keep it that way. */ 2081 raw_spin_lock_irq(&p->pi_lock); 2082 state = READ_ONCE(p->__state); 2083 smp_rmb(); /* see try_to_wake_up() */ 2084 if (state != TASK_RUNNING && state != TASK_WAKING && !p->on_rq) 2085 ip = __get_wchan(p); 2086 raw_spin_unlock_irq(&p->pi_lock); 2087 2088 return ip; 2089 } 2090 2091 static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags) 2092 { 2093 if (!(flags & ENQUEUE_NOCLOCK)) 2094 update_rq_clock(rq); 2095 2096 if (!(flags & ENQUEUE_RESTORE)) { 2097 sched_info_enqueue(rq, p); 2098 psi_enqueue(p, (flags & ENQUEUE_WAKEUP) && !(flags & ENQUEUE_MIGRATED)); 2099 } 2100 2101 uclamp_rq_inc(rq, p); 2102 p->sched_class->enqueue_task(rq, p, flags); 2103 2104 if (sched_core_enabled(rq)) 2105 sched_core_enqueue(rq, p); 2106 } 2107 2108 static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags) 2109 { 2110 if (sched_core_enabled(rq)) 2111 sched_core_dequeue(rq, p, flags); 2112 2113 if (!(flags & DEQUEUE_NOCLOCK)) 2114 update_rq_clock(rq); 2115 2116 if (!(flags & DEQUEUE_SAVE)) { 2117 sched_info_dequeue(rq, p); 2118 psi_dequeue(p, flags & DEQUEUE_SLEEP); 2119 } 2120 2121 uclamp_rq_dec(rq, p); 2122 p->sched_class->dequeue_task(rq, p, flags); 2123 } 2124 2125 void activate_task(struct rq *rq, struct task_struct *p, int flags) 2126 { 2127 if (task_on_rq_migrating(p)) 2128 flags |= ENQUEUE_MIGRATED; 2129 if (flags & ENQUEUE_MIGRATED) 2130 sched_mm_cid_migrate_to(rq, p); 2131 2132 enqueue_task(rq, p, flags); 2133 2134 p->on_rq = TASK_ON_RQ_QUEUED; 2135 } 2136 2137 void deactivate_task(struct rq *rq, struct task_struct *p, int flags) 2138 { 2139 p->on_rq = (flags & DEQUEUE_SLEEP) ? 0 : TASK_ON_RQ_MIGRATING; 2140 2141 dequeue_task(rq, p, flags); 2142 } 2143 2144 static inline int __normal_prio(int policy, int rt_prio, int nice) 2145 { 2146 int prio; 2147 2148 if (dl_policy(policy)) 2149 prio = MAX_DL_PRIO - 1; 2150 else if (rt_policy(policy)) 2151 prio = MAX_RT_PRIO - 1 - rt_prio; 2152 else 2153 prio = NICE_TO_PRIO(nice); 2154 2155 return prio; 2156 } 2157 2158 /* 2159 * Calculate the expected normal priority: i.e. priority 2160 * without taking RT-inheritance into account. Might be 2161 * boosted by interactivity modifiers. Changes upon fork, 2162 * setprio syscalls, and whenever the interactivity 2163 * estimator recalculates. 2164 */ 2165 static inline int normal_prio(struct task_struct *p) 2166 { 2167 return __normal_prio(p->policy, p->rt_priority, PRIO_TO_NICE(p->static_prio)); 2168 } 2169 2170 /* 2171 * Calculate the current priority, i.e. the priority 2172 * taken into account by the scheduler. This value might 2173 * be boosted by RT tasks, or might be boosted by 2174 * interactivity modifiers. Will be RT if the task got 2175 * RT-boosted. If not then it returns p->normal_prio. 2176 */ 2177 static int effective_prio(struct task_struct *p) 2178 { 2179 p->normal_prio = normal_prio(p); 2180 /* 2181 * If we are RT tasks or we were boosted to RT priority, 2182 * keep the priority unchanged. Otherwise, update priority 2183 * to the normal priority: 2184 */ 2185 if (!rt_prio(p->prio)) 2186 return p->normal_prio; 2187 return p->prio; 2188 } 2189 2190 /** 2191 * task_curr - is this task currently executing on a CPU? 2192 * @p: the task in question. 2193 * 2194 * Return: 1 if the task is currently executing. 0 otherwise. 2195 */ 2196 inline int task_curr(const struct task_struct *p) 2197 { 2198 return cpu_curr(task_cpu(p)) == p; 2199 } 2200 2201 /* 2202 * switched_from, switched_to and prio_changed must _NOT_ drop rq->lock, 2203 * use the balance_callback list if you want balancing. 2204 * 2205 * this means any call to check_class_changed() must be followed by a call to 2206 * balance_callback(). 2207 */ 2208 static inline void check_class_changed(struct rq *rq, struct task_struct *p, 2209 const struct sched_class *prev_class, 2210 int oldprio) 2211 { 2212 if (prev_class != p->sched_class) { 2213 if (prev_class->switched_from) 2214 prev_class->switched_from(rq, p); 2215 2216 p->sched_class->switched_to(rq, p); 2217 } else if (oldprio != p->prio || dl_task(p)) 2218 p->sched_class->prio_changed(rq, p, oldprio); 2219 } 2220 2221 void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags) 2222 { 2223 if (p->sched_class == rq->curr->sched_class) 2224 rq->curr->sched_class->check_preempt_curr(rq, p, flags); 2225 else if (sched_class_above(p->sched_class, rq->curr->sched_class)) 2226 resched_curr(rq); 2227 2228 /* 2229 * A queue event has occurred, and we're going to schedule. In 2230 * this case, we can save a useless back to back clock update. 2231 */ 2232 if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr)) 2233 rq_clock_skip_update(rq); 2234 } 2235 2236 static __always_inline 2237 int __task_state_match(struct task_struct *p, unsigned int state) 2238 { 2239 if (READ_ONCE(p->__state) & state) 2240 return 1; 2241 2242 #ifdef CONFIG_PREEMPT_RT 2243 if (READ_ONCE(p->saved_state) & state) 2244 return -1; 2245 #endif 2246 return 0; 2247 } 2248 2249 static __always_inline 2250 int task_state_match(struct task_struct *p, unsigned int state) 2251 { 2252 #ifdef CONFIG_PREEMPT_RT 2253 int match; 2254 2255 /* 2256 * Serialize against current_save_and_set_rtlock_wait_state() and 2257 * current_restore_rtlock_saved_state(). 2258 */ 2259 raw_spin_lock_irq(&p->pi_lock); 2260 match = __task_state_match(p, state); 2261 raw_spin_unlock_irq(&p->pi_lock); 2262 2263 return match; 2264 #else 2265 return __task_state_match(p, state); 2266 #endif 2267 } 2268 2269 /* 2270 * wait_task_inactive - wait for a thread to unschedule. 2271 * 2272 * Wait for the thread to block in any of the states set in @match_state. 2273 * If it changes, i.e. @p might have woken up, then return zero. When we 2274 * succeed in waiting for @p to be off its CPU, we return a positive number 2275 * (its total switch count). If a second call a short while later returns the 2276 * same number, the caller can be sure that @p has remained unscheduled the 2277 * whole time. 2278 * 2279 * The caller must ensure that the task *will* unschedule sometime soon, 2280 * else this function might spin for a *long* time. This function can't 2281 * be called with interrupts off, or it may introduce deadlock with 2282 * smp_call_function() if an IPI is sent by the same process we are 2283 * waiting to become inactive. 2284 */ 2285 unsigned long wait_task_inactive(struct task_struct *p, unsigned int match_state) 2286 { 2287 int running, queued, match; 2288 struct rq_flags rf; 2289 unsigned long ncsw; 2290 struct rq *rq; 2291 2292 for (;;) { 2293 /* 2294 * We do the initial early heuristics without holding 2295 * any task-queue locks at all. We'll only try to get 2296 * the runqueue lock when things look like they will 2297 * work out! 2298 */ 2299 rq = task_rq(p); 2300 2301 /* 2302 * If the task is actively running on another CPU 2303 * still, just relax and busy-wait without holding 2304 * any locks. 2305 * 2306 * NOTE! Since we don't hold any locks, it's not 2307 * even sure that "rq" stays as the right runqueue! 2308 * But we don't care, since "task_on_cpu()" will 2309 * return false if the runqueue has changed and p 2310 * is actually now running somewhere else! 2311 */ 2312 while (task_on_cpu(rq, p)) { 2313 if (!task_state_match(p, match_state)) 2314 return 0; 2315 cpu_relax(); 2316 } 2317 2318 /* 2319 * Ok, time to look more closely! We need the rq 2320 * lock now, to be *sure*. If we're wrong, we'll 2321 * just go back and repeat. 2322 */ 2323 rq = task_rq_lock(p, &rf); 2324 trace_sched_wait_task(p); 2325 running = task_on_cpu(rq, p); 2326 queued = task_on_rq_queued(p); 2327 ncsw = 0; 2328 if ((match = __task_state_match(p, match_state))) { 2329 /* 2330 * When matching on p->saved_state, consider this task 2331 * still queued so it will wait. 2332 */ 2333 if (match < 0) 2334 queued = 1; 2335 ncsw = p->nvcsw | LONG_MIN; /* sets MSB */ 2336 } 2337 task_rq_unlock(rq, p, &rf); 2338 2339 /* 2340 * If it changed from the expected state, bail out now. 2341 */ 2342 if (unlikely(!ncsw)) 2343 break; 2344 2345 /* 2346 * Was it really running after all now that we 2347 * checked with the proper locks actually held? 2348 * 2349 * Oops. Go back and try again.. 2350 */ 2351 if (unlikely(running)) { 2352 cpu_relax(); 2353 continue; 2354 } 2355 2356 /* 2357 * It's not enough that it's not actively running, 2358 * it must be off the runqueue _entirely_, and not 2359 * preempted! 2360 * 2361 * So if it was still runnable (but just not actively 2362 * running right now), it's preempted, and we should 2363 * yield - it could be a while. 2364 */ 2365 if (unlikely(queued)) { 2366 ktime_t to = NSEC_PER_SEC / HZ; 2367 2368 set_current_state(TASK_UNINTERRUPTIBLE); 2369 schedule_hrtimeout(&to, HRTIMER_MODE_REL_HARD); 2370 continue; 2371 } 2372 2373 /* 2374 * Ahh, all good. It wasn't running, and it wasn't 2375 * runnable, which means that it will never become 2376 * running in the future either. We're all done! 2377 */ 2378 break; 2379 } 2380 2381 return ncsw; 2382 } 2383 2384 #ifdef CONFIG_SMP 2385 2386 static void 2387 __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx); 2388 2389 static int __set_cpus_allowed_ptr(struct task_struct *p, 2390 struct affinity_context *ctx); 2391 2392 static void migrate_disable_switch(struct rq *rq, struct task_struct *p) 2393 { 2394 struct affinity_context ac = { 2395 .new_mask = cpumask_of(rq->cpu), 2396 .flags = SCA_MIGRATE_DISABLE, 2397 }; 2398 2399 if (likely(!p->migration_disabled)) 2400 return; 2401 2402 if (p->cpus_ptr != &p->cpus_mask) 2403 return; 2404 2405 /* 2406 * Violates locking rules! see comment in __do_set_cpus_allowed(). 2407 */ 2408 __do_set_cpus_allowed(p, &ac); 2409 } 2410 2411 void migrate_disable(void) 2412 { 2413 struct task_struct *p = current; 2414 2415 if (p->migration_disabled) { 2416 p->migration_disabled++; 2417 return; 2418 } 2419 2420 preempt_disable(); 2421 this_rq()->nr_pinned++; 2422 p->migration_disabled = 1; 2423 preempt_enable(); 2424 } 2425 EXPORT_SYMBOL_GPL(migrate_disable); 2426 2427 void migrate_enable(void) 2428 { 2429 struct task_struct *p = current; 2430 struct affinity_context ac = { 2431 .new_mask = &p->cpus_mask, 2432 .flags = SCA_MIGRATE_ENABLE, 2433 }; 2434 2435 if (p->migration_disabled > 1) { 2436 p->migration_disabled--; 2437 return; 2438 } 2439 2440 if (WARN_ON_ONCE(!p->migration_disabled)) 2441 return; 2442 2443 /* 2444 * Ensure stop_task runs either before or after this, and that 2445 * __set_cpus_allowed_ptr(SCA_MIGRATE_ENABLE) doesn't schedule(). 2446 */ 2447 preempt_disable(); 2448 if (p->cpus_ptr != &p->cpus_mask) 2449 __set_cpus_allowed_ptr(p, &ac); 2450 /* 2451 * Mustn't clear migration_disabled() until cpus_ptr points back at the 2452 * regular cpus_mask, otherwise things that race (eg. 2453 * select_fallback_rq) get confused. 2454 */ 2455 barrier(); 2456 p->migration_disabled = 0; 2457 this_rq()->nr_pinned--; 2458 preempt_enable(); 2459 } 2460 EXPORT_SYMBOL_GPL(migrate_enable); 2461 2462 static inline bool rq_has_pinned_tasks(struct rq *rq) 2463 { 2464 return rq->nr_pinned; 2465 } 2466 2467 /* 2468 * Per-CPU kthreads are allowed to run on !active && online CPUs, see 2469 * __set_cpus_allowed_ptr() and select_fallback_rq(). 2470 */ 2471 static inline bool is_cpu_allowed(struct task_struct *p, int cpu) 2472 { 2473 /* When not in the task's cpumask, no point in looking further. */ 2474 if (!cpumask_test_cpu(cpu, p->cpus_ptr)) 2475 return false; 2476 2477 /* migrate_disabled() must be allowed to finish. */ 2478 if (is_migration_disabled(p)) 2479 return cpu_online(cpu); 2480 2481 /* Non kernel threads are not allowed during either online or offline. */ 2482 if (!(p->flags & PF_KTHREAD)) 2483 return cpu_active(cpu) && task_cpu_possible(cpu, p); 2484 2485 /* KTHREAD_IS_PER_CPU is always allowed. */ 2486 if (kthread_is_per_cpu(p)) 2487 return cpu_online(cpu); 2488 2489 /* Regular kernel threads don't get to stay during offline. */ 2490 if (cpu_dying(cpu)) 2491 return false; 2492 2493 /* But are allowed during online. */ 2494 return cpu_online(cpu); 2495 } 2496 2497 /* 2498 * This is how migration works: 2499 * 2500 * 1) we invoke migration_cpu_stop() on the target CPU using 2501 * stop_one_cpu(). 2502 * 2) stopper starts to run (implicitly forcing the migrated thread 2503 * off the CPU) 2504 * 3) it checks whether the migrated task is still in the wrong runqueue. 2505 * 4) if it's in the wrong runqueue then the migration thread removes 2506 * it and puts it into the right queue. 2507 * 5) stopper completes and stop_one_cpu() returns and the migration 2508 * is done. 2509 */ 2510 2511 /* 2512 * move_queued_task - move a queued task to new rq. 2513 * 2514 * Returns (locked) new rq. Old rq's lock is released. 2515 */ 2516 static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf, 2517 struct task_struct *p, int new_cpu) 2518 { 2519 lockdep_assert_rq_held(rq); 2520 2521 deactivate_task(rq, p, DEQUEUE_NOCLOCK); 2522 set_task_cpu(p, new_cpu); 2523 rq_unlock(rq, rf); 2524 2525 rq = cpu_rq(new_cpu); 2526 2527 rq_lock(rq, rf); 2528 WARN_ON_ONCE(task_cpu(p) != new_cpu); 2529 activate_task(rq, p, 0); 2530 check_preempt_curr(rq, p, 0); 2531 2532 return rq; 2533 } 2534 2535 struct migration_arg { 2536 struct task_struct *task; 2537 int dest_cpu; 2538 struct set_affinity_pending *pending; 2539 }; 2540 2541 /* 2542 * @refs: number of wait_for_completion() 2543 * @stop_pending: is @stop_work in use 2544 */ 2545 struct set_affinity_pending { 2546 refcount_t refs; 2547 unsigned int stop_pending; 2548 struct completion done; 2549 struct cpu_stop_work stop_work; 2550 struct migration_arg arg; 2551 }; 2552 2553 /* 2554 * Move (not current) task off this CPU, onto the destination CPU. We're doing 2555 * this because either it can't run here any more (set_cpus_allowed() 2556 * away from this CPU, or CPU going down), or because we're 2557 * attempting to rebalance this task on exec (sched_exec). 2558 * 2559 * So we race with normal scheduler movements, but that's OK, as long 2560 * as the task is no longer on this CPU. 2561 */ 2562 static struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf, 2563 struct task_struct *p, int dest_cpu) 2564 { 2565 /* Affinity changed (again). */ 2566 if (!is_cpu_allowed(p, dest_cpu)) 2567 return rq; 2568 2569 rq = move_queued_task(rq, rf, p, dest_cpu); 2570 2571 return rq; 2572 } 2573 2574 /* 2575 * migration_cpu_stop - this will be executed by a highprio stopper thread 2576 * and performs thread migration by bumping thread off CPU then 2577 * 'pushing' onto another runqueue. 2578 */ 2579 static int migration_cpu_stop(void *data) 2580 { 2581 struct migration_arg *arg = data; 2582 struct set_affinity_pending *pending = arg->pending; 2583 struct task_struct *p = arg->task; 2584 struct rq *rq = this_rq(); 2585 bool complete = false; 2586 struct rq_flags rf; 2587 2588 /* 2589 * The original target CPU might have gone down and we might 2590 * be on another CPU but it doesn't matter. 2591 */ 2592 local_irq_save(rf.flags); 2593 /* 2594 * We need to explicitly wake pending tasks before running 2595 * __migrate_task() such that we will not miss enforcing cpus_ptr 2596 * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test. 2597 */ 2598 flush_smp_call_function_queue(); 2599 2600 raw_spin_lock(&p->pi_lock); 2601 rq_lock(rq, &rf); 2602 2603 /* 2604 * If we were passed a pending, then ->stop_pending was set, thus 2605 * p->migration_pending must have remained stable. 2606 */ 2607 WARN_ON_ONCE(pending && pending != p->migration_pending); 2608 2609 /* 2610 * If task_rq(p) != rq, it cannot be migrated here, because we're 2611 * holding rq->lock, if p->on_rq == 0 it cannot get enqueued because 2612 * we're holding p->pi_lock. 2613 */ 2614 if (task_rq(p) == rq) { 2615 if (is_migration_disabled(p)) 2616 goto out; 2617 2618 if (pending) { 2619 p->migration_pending = NULL; 2620 complete = true; 2621 2622 if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask)) 2623 goto out; 2624 } 2625 2626 if (task_on_rq_queued(p)) { 2627 update_rq_clock(rq); 2628 rq = __migrate_task(rq, &rf, p, arg->dest_cpu); 2629 } else { 2630 p->wake_cpu = arg->dest_cpu; 2631 } 2632 2633 /* 2634 * XXX __migrate_task() can fail, at which point we might end 2635 * up running on a dodgy CPU, AFAICT this can only happen 2636 * during CPU hotplug, at which point we'll get pushed out 2637 * anyway, so it's probably not a big deal. 2638 */ 2639 2640 } else if (pending) { 2641 /* 2642 * This happens when we get migrated between migrate_enable()'s 2643 * preempt_enable() and scheduling the stopper task. At that 2644 * point we're a regular task again and not current anymore. 2645 * 2646 * A !PREEMPT kernel has a giant hole here, which makes it far 2647 * more likely. 2648 */ 2649 2650 /* 2651 * The task moved before the stopper got to run. We're holding 2652 * ->pi_lock, so the allowed mask is stable - if it got 2653 * somewhere allowed, we're done. 2654 */ 2655 if (cpumask_test_cpu(task_cpu(p), p->cpus_ptr)) { 2656 p->migration_pending = NULL; 2657 complete = true; 2658 goto out; 2659 } 2660 2661 /* 2662 * When migrate_enable() hits a rq mis-match we can't reliably 2663 * determine is_migration_disabled() and so have to chase after 2664 * it. 2665 */ 2666 WARN_ON_ONCE(!pending->stop_pending); 2667 task_rq_unlock(rq, p, &rf); 2668 stop_one_cpu_nowait(task_cpu(p), migration_cpu_stop, 2669 &pending->arg, &pending->stop_work); 2670 return 0; 2671 } 2672 out: 2673 if (pending) 2674 pending->stop_pending = false; 2675 task_rq_unlock(rq, p, &rf); 2676 2677 if (complete) 2678 complete_all(&pending->done); 2679 2680 return 0; 2681 } 2682 2683 int push_cpu_stop(void *arg) 2684 { 2685 struct rq *lowest_rq = NULL, *rq = this_rq(); 2686 struct task_struct *p = arg; 2687 2688 raw_spin_lock_irq(&p->pi_lock); 2689 raw_spin_rq_lock(rq); 2690 2691 if (task_rq(p) != rq) 2692 goto out_unlock; 2693 2694 if (is_migration_disabled(p)) { 2695 p->migration_flags |= MDF_PUSH; 2696 goto out_unlock; 2697 } 2698 2699 p->migration_flags &= ~MDF_PUSH; 2700 2701 if (p->sched_class->find_lock_rq) 2702 lowest_rq = p->sched_class->find_lock_rq(p, rq); 2703 2704 if (!lowest_rq) 2705 goto out_unlock; 2706 2707 // XXX validate p is still the highest prio task 2708 if (task_rq(p) == rq) { 2709 deactivate_task(rq, p, 0); 2710 set_task_cpu(p, lowest_rq->cpu); 2711 activate_task(lowest_rq, p, 0); 2712 resched_curr(lowest_rq); 2713 } 2714 2715 double_unlock_balance(rq, lowest_rq); 2716 2717 out_unlock: 2718 rq->push_busy = false; 2719 raw_spin_rq_unlock(rq); 2720 raw_spin_unlock_irq(&p->pi_lock); 2721 2722 put_task_struct(p); 2723 return 0; 2724 } 2725 2726 /* 2727 * sched_class::set_cpus_allowed must do the below, but is not required to 2728 * actually call this function. 2729 */ 2730 void set_cpus_allowed_common(struct task_struct *p, struct affinity_context *ctx) 2731 { 2732 if (ctx->flags & (SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) { 2733 p->cpus_ptr = ctx->new_mask; 2734 return; 2735 } 2736 2737 cpumask_copy(&p->cpus_mask, ctx->new_mask); 2738 p->nr_cpus_allowed = cpumask_weight(ctx->new_mask); 2739 2740 /* 2741 * Swap in a new user_cpus_ptr if SCA_USER flag set 2742 */ 2743 if (ctx->flags & SCA_USER) 2744 swap(p->user_cpus_ptr, ctx->user_mask); 2745 } 2746 2747 static void 2748 __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) 2749 { 2750 struct rq *rq = task_rq(p); 2751 bool queued, running; 2752 2753 /* 2754 * This here violates the locking rules for affinity, since we're only 2755 * supposed to change these variables while holding both rq->lock and 2756 * p->pi_lock. 2757 * 2758 * HOWEVER, it magically works, because ttwu() is the only code that 2759 * accesses these variables under p->pi_lock and only does so after 2760 * smp_cond_load_acquire(&p->on_cpu, !VAL), and we're in __schedule() 2761 * before finish_task(). 2762 * 2763 * XXX do further audits, this smells like something putrid. 2764 */ 2765 if (ctx->flags & SCA_MIGRATE_DISABLE) 2766 SCHED_WARN_ON(!p->on_cpu); 2767 else 2768 lockdep_assert_held(&p->pi_lock); 2769 2770 queued = task_on_rq_queued(p); 2771 running = task_current(rq, p); 2772 2773 if (queued) { 2774 /* 2775 * Because __kthread_bind() calls this on blocked tasks without 2776 * holding rq->lock. 2777 */ 2778 lockdep_assert_rq_held(rq); 2779 dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); 2780 } 2781 if (running) 2782 put_prev_task(rq, p); 2783 2784 p->sched_class->set_cpus_allowed(p, ctx); 2785 2786 if (queued) 2787 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); 2788 if (running) 2789 set_next_task(rq, p); 2790 } 2791 2792 /* 2793 * Used for kthread_bind() and select_fallback_rq(), in both cases the user 2794 * affinity (if any) should be destroyed too. 2795 */ 2796 void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) 2797 { 2798 struct affinity_context ac = { 2799 .new_mask = new_mask, 2800 .user_mask = NULL, 2801 .flags = SCA_USER, /* clear the user requested mask */ 2802 }; 2803 union cpumask_rcuhead { 2804 cpumask_t cpumask; 2805 struct rcu_head rcu; 2806 }; 2807 2808 __do_set_cpus_allowed(p, &ac); 2809 2810 /* 2811 * Because this is called with p->pi_lock held, it is not possible 2812 * to use kfree() here (when PREEMPT_RT=y), therefore punt to using 2813 * kfree_rcu(). 2814 */ 2815 kfree_rcu((union cpumask_rcuhead *)ac.user_mask, rcu); 2816 } 2817 2818 static cpumask_t *alloc_user_cpus_ptr(int node) 2819 { 2820 /* 2821 * See do_set_cpus_allowed() above for the rcu_head usage. 2822 */ 2823 int size = max_t(int, cpumask_size(), sizeof(struct rcu_head)); 2824 2825 return kmalloc_node(size, GFP_KERNEL, node); 2826 } 2827 2828 int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src, 2829 int node) 2830 { 2831 cpumask_t *user_mask; 2832 unsigned long flags; 2833 2834 /* 2835 * Always clear dst->user_cpus_ptr first as their user_cpus_ptr's 2836 * may differ by now due to racing. 2837 */ 2838 dst->user_cpus_ptr = NULL; 2839 2840 /* 2841 * This check is racy and losing the race is a valid situation. 2842 * It is not worth the extra overhead of taking the pi_lock on 2843 * every fork/clone. 2844 */ 2845 if (data_race(!src->user_cpus_ptr)) 2846 return 0; 2847 2848 user_mask = alloc_user_cpus_ptr(node); 2849 if (!user_mask) 2850 return -ENOMEM; 2851 2852 /* 2853 * Use pi_lock to protect content of user_cpus_ptr 2854 * 2855 * Though unlikely, user_cpus_ptr can be reset to NULL by a concurrent 2856 * do_set_cpus_allowed(). 2857 */ 2858 raw_spin_lock_irqsave(&src->pi_lock, flags); 2859 if (src->user_cpus_ptr) { 2860 swap(dst->user_cpus_ptr, user_mask); 2861 cpumask_copy(dst->user_cpus_ptr, src->user_cpus_ptr); 2862 } 2863 raw_spin_unlock_irqrestore(&src->pi_lock, flags); 2864 2865 if (unlikely(user_mask)) 2866 kfree(user_mask); 2867 2868 return 0; 2869 } 2870 2871 static inline struct cpumask *clear_user_cpus_ptr(struct task_struct *p) 2872 { 2873 struct cpumask *user_mask = NULL; 2874 2875 swap(p->user_cpus_ptr, user_mask); 2876 2877 return user_mask; 2878 } 2879 2880 void release_user_cpus_ptr(struct task_struct *p) 2881 { 2882 kfree(clear_user_cpus_ptr(p)); 2883 } 2884 2885 /* 2886 * This function is wildly self concurrent; here be dragons. 2887 * 2888 * 2889 * When given a valid mask, __set_cpus_allowed_ptr() must block until the 2890 * designated task is enqueued on an allowed CPU. If that task is currently 2891 * running, we have to kick it out using the CPU stopper. 2892 * 2893 * Migrate-Disable comes along and tramples all over our nice sandcastle. 2894 * Consider: 2895 * 2896 * Initial conditions: P0->cpus_mask = [0, 1] 2897 * 2898 * P0@CPU0 P1 2899 * 2900 * migrate_disable(); 2901 * <preempted> 2902 * set_cpus_allowed_ptr(P0, [1]); 2903 * 2904 * P1 *cannot* return from this set_cpus_allowed_ptr() call until P0 executes 2905 * its outermost migrate_enable() (i.e. it exits its Migrate-Disable region). 2906 * This means we need the following scheme: 2907 * 2908 * P0@CPU0 P1 2909 * 2910 * migrate_disable(); 2911 * <preempted> 2912 * set_cpus_allowed_ptr(P0, [1]); 2913 * <blocks> 2914 * <resumes> 2915 * migrate_enable(); 2916 * __set_cpus_allowed_ptr(); 2917 * <wakes local stopper> 2918 * `--> <woken on migration completion> 2919 * 2920 * Now the fun stuff: there may be several P1-like tasks, i.e. multiple 2921 * concurrent set_cpus_allowed_ptr(P0, [*]) calls. CPU affinity changes of any 2922 * task p are serialized by p->pi_lock, which we can leverage: the one that 2923 * should come into effect at the end of the Migrate-Disable region is the last 2924 * one. This means we only need to track a single cpumask (i.e. p->cpus_mask), 2925 * but we still need to properly signal those waiting tasks at the appropriate 2926 * moment. 2927 * 2928 * This is implemented using struct set_affinity_pending. The first 2929 * __set_cpus_allowed_ptr() caller within a given Migrate-Disable region will 2930 * setup an instance of that struct and install it on the targeted task_struct. 2931 * Any and all further callers will reuse that instance. Those then wait for 2932 * a completion signaled at the tail of the CPU stopper callback (1), triggered 2933 * on the end of the Migrate-Disable region (i.e. outermost migrate_enable()). 2934 * 2935 * 2936 * (1) In the cases covered above. There is one more where the completion is 2937 * signaled within affine_move_task() itself: when a subsequent affinity request 2938 * occurs after the stopper bailed out due to the targeted task still being 2939 * Migrate-Disable. Consider: 2940 * 2941 * Initial conditions: P0->cpus_mask = [0, 1] 2942 * 2943 * CPU0 P1 P2 2944 * <P0> 2945 * migrate_disable(); 2946 * <preempted> 2947 * set_cpus_allowed_ptr(P0, [1]); 2948 * <blocks> 2949 * <migration/0> 2950 * migration_cpu_stop() 2951 * is_migration_disabled() 2952 * <bails> 2953 * set_cpus_allowed_ptr(P0, [0, 1]); 2954 * <signal completion> 2955 * <awakes> 2956 * 2957 * Note that the above is safe vs a concurrent migrate_enable(), as any 2958 * pending affinity completion is preceded by an uninstallation of 2959 * p->migration_pending done with p->pi_lock held. 2960 */ 2961 static int affine_move_task(struct rq *rq, struct task_struct *p, struct rq_flags *rf, 2962 int dest_cpu, unsigned int flags) 2963 __releases(rq->lock) 2964 __releases(p->pi_lock) 2965 { 2966 struct set_affinity_pending my_pending = { }, *pending = NULL; 2967 bool stop_pending, complete = false; 2968 2969 /* Can the task run on the task's current CPU? If so, we're done */ 2970 if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask)) { 2971 struct task_struct *push_task = NULL; 2972 2973 if ((flags & SCA_MIGRATE_ENABLE) && 2974 (p->migration_flags & MDF_PUSH) && !rq->push_busy) { 2975 rq->push_busy = true; 2976 push_task = get_task_struct(p); 2977 } 2978 2979 /* 2980 * If there are pending waiters, but no pending stop_work, 2981 * then complete now. 2982 */ 2983 pending = p->migration_pending; 2984 if (pending && !pending->stop_pending) { 2985 p->migration_pending = NULL; 2986 complete = true; 2987 } 2988 2989 task_rq_unlock(rq, p, rf); 2990 2991 if (push_task) { 2992 stop_one_cpu_nowait(rq->cpu, push_cpu_stop, 2993 p, &rq->push_work); 2994 } 2995 2996 if (complete) 2997 complete_all(&pending->done); 2998 2999 return 0; 3000 } 3001 3002 if (!(flags & SCA_MIGRATE_ENABLE)) { 3003 /* serialized by p->pi_lock */ 3004 if (!p->migration_pending) { 3005 /* Install the request */ 3006 refcount_set(&my_pending.refs, 1); 3007 init_completion(&my_pending.done); 3008 my_pending.arg = (struct migration_arg) { 3009 .task = p, 3010 .dest_cpu = dest_cpu, 3011 .pending = &my_pending, 3012 }; 3013 3014 p->migration_pending = &my_pending; 3015 } else { 3016 pending = p->migration_pending; 3017 refcount_inc(&pending->refs); 3018 /* 3019 * Affinity has changed, but we've already installed a 3020 * pending. migration_cpu_stop() *must* see this, else 3021 * we risk a completion of the pending despite having a 3022 * task on a disallowed CPU. 3023 * 3024 * Serialized by p->pi_lock, so this is safe. 3025 */ 3026 pending->arg.dest_cpu = dest_cpu; 3027 } 3028 } 3029 pending = p->migration_pending; 3030 /* 3031 * - !MIGRATE_ENABLE: 3032 * we'll have installed a pending if there wasn't one already. 3033 * 3034 * - MIGRATE_ENABLE: 3035 * we're here because the current CPU isn't matching anymore, 3036 * the only way that can happen is because of a concurrent 3037 * set_cpus_allowed_ptr() call, which should then still be 3038 * pending completion. 3039 * 3040 * Either way, we really should have a @pending here. 3041 */ 3042 if (WARN_ON_ONCE(!pending)) { 3043 task_rq_unlock(rq, p, rf); 3044 return -EINVAL; 3045 } 3046 3047 if (task_on_cpu(rq, p) || READ_ONCE(p->__state) == TASK_WAKING) { 3048 /* 3049 * MIGRATE_ENABLE gets here because 'p == current', but for 3050 * anything else we cannot do is_migration_disabled(), punt 3051 * and have the stopper function handle it all race-free. 3052 */ 3053 stop_pending = pending->stop_pending; 3054 if (!stop_pending) 3055 pending->stop_pending = true; 3056 3057 if (flags & SCA_MIGRATE_ENABLE) 3058 p->migration_flags &= ~MDF_PUSH; 3059 3060 task_rq_unlock(rq, p, rf); 3061 3062 if (!stop_pending) { 3063 stop_one_cpu_nowait(cpu_of(rq), migration_cpu_stop, 3064 &pending->arg, &pending->stop_work); 3065 } 3066 3067 if (flags & SCA_MIGRATE_ENABLE) 3068 return 0; 3069 } else { 3070 3071 if (!is_migration_disabled(p)) { 3072 if (task_on_rq_queued(p)) 3073 rq = move_queued_task(rq, rf, p, dest_cpu); 3074 3075 if (!pending->stop_pending) { 3076 p->migration_pending = NULL; 3077 complete = true; 3078 } 3079 } 3080 task_rq_unlock(rq, p, rf); 3081 3082 if (complete) 3083 complete_all(&pending->done); 3084 } 3085 3086 wait_for_completion(&pending->done); 3087 3088 if (refcount_dec_and_test(&pending->refs)) 3089 wake_up_var(&pending->refs); /* No UaF, just an address */ 3090 3091 /* 3092 * Block the original owner of &pending until all subsequent callers 3093 * have seen the completion and decremented the refcount 3094 */ 3095 wait_var_event(&my_pending.refs, !refcount_read(&my_pending.refs)); 3096 3097 /* ARGH */ 3098 WARN_ON_ONCE(my_pending.stop_pending); 3099 3100 return 0; 3101 } 3102 3103 /* 3104 * Called with both p->pi_lock and rq->lock held; drops both before returning. 3105 */ 3106 static int __set_cpus_allowed_ptr_locked(struct task_struct *p, 3107 struct affinity_context *ctx, 3108 struct rq *rq, 3109 struct rq_flags *rf) 3110 __releases(rq->lock) 3111 __releases(p->pi_lock) 3112 { 3113 const struct cpumask *cpu_allowed_mask = task_cpu_possible_mask(p); 3114 const struct cpumask *cpu_valid_mask = cpu_active_mask; 3115 bool kthread = p->flags & PF_KTHREAD; 3116 unsigned int dest_cpu; 3117 int ret = 0; 3118 3119 update_rq_clock(rq); 3120 3121 if (kthread || is_migration_disabled(p)) { 3122 /* 3123 * Kernel threads are allowed on online && !active CPUs, 3124 * however, during cpu-hot-unplug, even these might get pushed 3125 * away if not KTHREAD_IS_PER_CPU. 3126 * 3127 * Specifically, migration_disabled() tasks must not fail the 3128 * cpumask_any_and_distribute() pick below, esp. so on 3129 * SCA_MIGRATE_ENABLE, otherwise we'll not call 3130 * set_cpus_allowed_common() and actually reset p->cpus_ptr. 3131 */ 3132 cpu_valid_mask = cpu_online_mask; 3133 } 3134 3135 if (!kthread && !cpumask_subset(ctx->new_mask, cpu_allowed_mask)) { 3136 ret = -EINVAL; 3137 goto out; 3138 } 3139 3140 /* 3141 * Must re-check here, to close a race against __kthread_bind(), 3142 * sched_setaffinity() is not guaranteed to observe the flag. 3143 */ 3144 if ((ctx->flags & SCA_CHECK) && (p->flags & PF_NO_SETAFFINITY)) { 3145 ret = -EINVAL; 3146 goto out; 3147 } 3148 3149 if (!(ctx->flags & SCA_MIGRATE_ENABLE)) { 3150 if (cpumask_equal(&p->cpus_mask, ctx->new_mask)) { 3151 if (ctx->flags & SCA_USER) 3152 swap(p->user_cpus_ptr, ctx->user_mask); 3153 goto out; 3154 } 3155 3156 if (WARN_ON_ONCE(p == current && 3157 is_migration_disabled(p) && 3158 !cpumask_test_cpu(task_cpu(p), ctx->new_mask))) { 3159 ret = -EBUSY; 3160 goto out; 3161 } 3162 } 3163 3164 /* 3165 * Picking a ~random cpu helps in cases where we are changing affinity 3166 * for groups of tasks (ie. cpuset), so that load balancing is not 3167 * immediately required to distribute the tasks within their new mask. 3168 */ 3169 dest_cpu = cpumask_any_and_distribute(cpu_valid_mask, ctx->new_mask); 3170 if (dest_cpu >= nr_cpu_ids) { 3171 ret = -EINVAL; 3172 goto out; 3173 } 3174 3175 __do_set_cpus_allowed(p, ctx); 3176 3177 return affine_move_task(rq, p, rf, dest_cpu, ctx->flags); 3178 3179 out: 3180 task_rq_unlock(rq, p, rf); 3181 3182 return ret; 3183 } 3184 3185 /* 3186 * Change a given task's CPU affinity. Migrate the thread to a 3187 * proper CPU and schedule it away if the CPU it's executing on 3188 * is removed from the allowed bitmask. 3189 * 3190 * NOTE: the caller must have a valid reference to the task, the 3191 * task must not exit() & deallocate itself prematurely. The 3192 * call is not atomic; no spinlocks may be held. 3193 */ 3194 static int __set_cpus_allowed_ptr(struct task_struct *p, 3195 struct affinity_context *ctx) 3196 { 3197 struct rq_flags rf; 3198 struct rq *rq; 3199 3200 rq = task_rq_lock(p, &rf); 3201 /* 3202 * Masking should be skipped if SCA_USER or any of the SCA_MIGRATE_* 3203 * flags are set. 3204 */ 3205 if (p->user_cpus_ptr && 3206 !(ctx->flags & (SCA_USER | SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) && 3207 cpumask_and(rq->scratch_mask, ctx->new_mask, p->user_cpus_ptr)) 3208 ctx->new_mask = rq->scratch_mask; 3209 3210 return __set_cpus_allowed_ptr_locked(p, ctx, rq, &rf); 3211 } 3212 3213 int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) 3214 { 3215 struct affinity_context ac = { 3216 .new_mask = new_mask, 3217 .flags = 0, 3218 }; 3219 3220 return __set_cpus_allowed_ptr(p, &ac); 3221 } 3222 EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr); 3223 3224 /* 3225 * Change a given task's CPU affinity to the intersection of its current 3226 * affinity mask and @subset_mask, writing the resulting mask to @new_mask. 3227 * If user_cpus_ptr is defined, use it as the basis for restricting CPU 3228 * affinity or use cpu_online_mask instead. 3229 * 3230 * If the resulting mask is empty, leave the affinity unchanged and return 3231 * -EINVAL. 3232 */ 3233 static int restrict_cpus_allowed_ptr(struct task_struct *p, 3234 struct cpumask *new_mask, 3235 const struct cpumask *subset_mask) 3236 { 3237 struct affinity_context ac = { 3238 .new_mask = new_mask, 3239 .flags = 0, 3240 }; 3241 struct rq_flags rf; 3242 struct rq *rq; 3243 int err; 3244 3245 rq = task_rq_lock(p, &rf); 3246 3247 /* 3248 * Forcefully restricting the affinity of a deadline task is 3249 * likely to cause problems, so fail and noisily override the 3250 * mask entirely. 3251 */ 3252 if (task_has_dl_policy(p) && dl_bandwidth_enabled()) { 3253 err = -EPERM; 3254 goto err_unlock; 3255 } 3256 3257 if (!cpumask_and(new_mask, task_user_cpus(p), subset_mask)) { 3258 err = -EINVAL; 3259 goto err_unlock; 3260 } 3261 3262 return __set_cpus_allowed_ptr_locked(p, &ac, rq, &rf); 3263 3264 err_unlock: 3265 task_rq_unlock(rq, p, &rf); 3266 return err; 3267 } 3268 3269 /* 3270 * Restrict the CPU affinity of task @p so that it is a subset of 3271 * task_cpu_possible_mask() and point @p->user_cpus_ptr to a copy of the 3272 * old affinity mask. If the resulting mask is empty, we warn and walk 3273 * up the cpuset hierarchy until we find a suitable mask. 3274 */ 3275 void force_compatible_cpus_allowed_ptr(struct task_struct *p) 3276 { 3277 cpumask_var_t new_mask; 3278 const struct cpumask *override_mask = task_cpu_possible_mask(p); 3279 3280 alloc_cpumask_var(&new_mask, GFP_KERNEL); 3281 3282 /* 3283 * __migrate_task() can fail silently in the face of concurrent 3284 * offlining of the chosen destination CPU, so take the hotplug 3285 * lock to ensure that the migration succeeds. 3286 */ 3287 cpus_read_lock(); 3288 if (!cpumask_available(new_mask)) 3289 goto out_set_mask; 3290 3291 if (!restrict_cpus_allowed_ptr(p, new_mask, override_mask)) 3292 goto out_free_mask; 3293 3294 /* 3295 * We failed to find a valid subset of the affinity mask for the 3296 * task, so override it based on its cpuset hierarchy. 3297 */ 3298 cpuset_cpus_allowed(p, new_mask); 3299 override_mask = new_mask; 3300 3301 out_set_mask: 3302 if (printk_ratelimit()) { 3303 printk_deferred("Overriding affinity for process %d (%s) to CPUs %*pbl\n", 3304 task_pid_nr(p), p->comm, 3305 cpumask_pr_args(override_mask)); 3306 } 3307 3308 WARN_ON(set_cpus_allowed_ptr(p, override_mask)); 3309 out_free_mask: 3310 cpus_read_unlock(); 3311 free_cpumask_var(new_mask); 3312 } 3313 3314 static int 3315 __sched_setaffinity(struct task_struct *p, struct affinity_context *ctx); 3316 3317 /* 3318 * Restore the affinity of a task @p which was previously restricted by a 3319 * call to force_compatible_cpus_allowed_ptr(). 3320 * 3321 * It is the caller's responsibility to serialise this with any calls to 3322 * force_compatible_cpus_allowed_ptr(@p). 3323 */ 3324 void relax_compatible_cpus_allowed_ptr(struct task_struct *p) 3325 { 3326 struct affinity_context ac = { 3327 .new_mask = task_user_cpus(p), 3328 .flags = 0, 3329 }; 3330 int ret; 3331 3332 /* 3333 * Try to restore the old affinity mask with __sched_setaffinity(). 3334 * Cpuset masking will be done there too. 3335 */ 3336 ret = __sched_setaffinity(p, &ac); 3337 WARN_ON_ONCE(ret); 3338 } 3339 3340 void set_task_cpu(struct task_struct *p, unsigned int new_cpu) 3341 { 3342 #ifdef CONFIG_SCHED_DEBUG 3343 unsigned int state = READ_ONCE(p->__state); 3344 3345 /* 3346 * We should never call set_task_cpu() on a blocked task, 3347 * ttwu() will sort out the placement. 3348 */ 3349 WARN_ON_ONCE(state != TASK_RUNNING && state != TASK_WAKING && !p->on_rq); 3350 3351 /* 3352 * Migrating fair class task must have p->on_rq = TASK_ON_RQ_MIGRATING, 3353 * because schedstat_wait_{start,end} rebase migrating task's wait_start 3354 * time relying on p->on_rq. 3355 */ 3356 WARN_ON_ONCE(state == TASK_RUNNING && 3357 p->sched_class == &fair_sched_class && 3358 (p->on_rq && !task_on_rq_migrating(p))); 3359 3360 #ifdef CONFIG_LOCKDEP 3361 /* 3362 * The caller should hold either p->pi_lock or rq->lock, when changing 3363 * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks. 3364 * 3365 * sched_move_task() holds both and thus holding either pins the cgroup, 3366 * see task_group(). 3367 * 3368 * Furthermore, all task_rq users should acquire both locks, see 3369 * task_rq_lock(). 3370 */ 3371 WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) || 3372 lockdep_is_held(__rq_lockp(task_rq(p))))); 3373 #endif 3374 /* 3375 * Clearly, migrating tasks to offline CPUs is a fairly daft thing. 3376 */ 3377 WARN_ON_ONCE(!cpu_online(new_cpu)); 3378 3379 WARN_ON_ONCE(is_migration_disabled(p)); 3380 #endif 3381 3382 trace_sched_migrate_task(p, new_cpu); 3383 3384 if (task_cpu(p) != new_cpu) { 3385 if (p->sched_class->migrate_task_rq) 3386 p->sched_class->migrate_task_rq(p, new_cpu); 3387 p->se.nr_migrations++; 3388 rseq_migrate(p); 3389 sched_mm_cid_migrate_from(p); 3390 perf_event_task_migrate(p); 3391 } 3392 3393 __set_task_cpu(p, new_cpu); 3394 } 3395 3396 #ifdef CONFIG_NUMA_BALANCING 3397 static void __migrate_swap_task(struct task_struct *p, int cpu) 3398 { 3399 if (task_on_rq_queued(p)) { 3400 struct rq *src_rq, *dst_rq; 3401 struct rq_flags srf, drf; 3402 3403 src_rq = task_rq(p); 3404 dst_rq = cpu_rq(cpu); 3405 3406 rq_pin_lock(src_rq, &srf); 3407 rq_pin_lock(dst_rq, &drf); 3408 3409 deactivate_task(src_rq, p, 0); 3410 set_task_cpu(p, cpu); 3411 activate_task(dst_rq, p, 0); 3412 check_preempt_curr(dst_rq, p, 0); 3413 3414 rq_unpin_lock(dst_rq, &drf); 3415 rq_unpin_lock(src_rq, &srf); 3416 3417 } else { 3418 /* 3419 * Task isn't running anymore; make it appear like we migrated 3420 * it before it went to sleep. This means on wakeup we make the 3421 * previous CPU our target instead of where it really is. 3422 */ 3423 p->wake_cpu = cpu; 3424 } 3425 } 3426 3427 struct migration_swap_arg { 3428 struct task_struct *src_task, *dst_task; 3429 int src_cpu, dst_cpu; 3430 }; 3431 3432 static int migrate_swap_stop(void *data) 3433 { 3434 struct migration_swap_arg *arg = data; 3435 struct rq *src_rq, *dst_rq; 3436 3437 if (!cpu_active(arg->src_cpu) || !cpu_active(arg->dst_cpu)) 3438 return -EAGAIN; 3439 3440 src_rq = cpu_rq(arg->src_cpu); 3441 dst_rq = cpu_rq(arg->dst_cpu); 3442 3443 guard(double_raw_spinlock)(&arg->src_task->pi_lock, &arg->dst_task->pi_lock); 3444 guard(double_rq_lock)(src_rq, dst_rq); 3445 3446 if (task_cpu(arg->dst_task) != arg->dst_cpu) 3447 return -EAGAIN; 3448 3449 if (task_cpu(arg->src_task) != arg->src_cpu) 3450 return -EAGAIN; 3451 3452 if (!cpumask_test_cpu(arg->dst_cpu, arg->src_task->cpus_ptr)) 3453 return -EAGAIN; 3454 3455 if (!cpumask_test_cpu(arg->src_cpu, arg->dst_task->cpus_ptr)) 3456 return -EAGAIN; 3457 3458 __migrate_swap_task(arg->src_task, arg->dst_cpu); 3459 __migrate_swap_task(arg->dst_task, arg->src_cpu); 3460 3461 return 0; 3462 } 3463 3464 /* 3465 * Cross migrate two tasks 3466 */ 3467 int migrate_swap(struct task_struct *cur, struct task_struct *p, 3468 int target_cpu, int curr_cpu) 3469 { 3470 struct migration_swap_arg arg; 3471 int ret = -EINVAL; 3472 3473 arg = (struct migration_swap_arg){ 3474 .src_task = cur, 3475 .src_cpu = curr_cpu, 3476 .dst_task = p, 3477 .dst_cpu = target_cpu, 3478 }; 3479 3480 if (arg.src_cpu == arg.dst_cpu) 3481 goto out; 3482 3483 /* 3484 * These three tests are all lockless; this is OK since all of them 3485 * will be re-checked with proper locks held further down the line. 3486 */ 3487 if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu)) 3488 goto out; 3489 3490 if (!cpumask_test_cpu(arg.dst_cpu, arg.src_task->cpus_ptr)) 3491 goto out; 3492 3493 if (!cpumask_test_cpu(arg.src_cpu, arg.dst_task->cpus_ptr)) 3494 goto out; 3495 3496 trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu); 3497 ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg); 3498 3499 out: 3500 return ret; 3501 } 3502 #endif /* CONFIG_NUMA_BALANCING */ 3503 3504 /*** 3505 * kick_process - kick a running thread to enter/exit the kernel 3506 * @p: the to-be-kicked thread 3507 * 3508 * Cause a process which is running on another CPU to enter 3509 * kernel-mode, without any delay. (to get signals handled.) 3510 * 3511 * NOTE: this function doesn't have to take the runqueue lock, 3512 * because all it wants to ensure is that the remote task enters 3513 * the kernel. If the IPI races and the task has been migrated 3514 * to another CPU then no harm is done and the purpose has been 3515 * achieved as well. 3516 */ 3517 void kick_process(struct task_struct *p) 3518 { 3519 int cpu; 3520 3521 preempt_disable(); 3522 cpu = task_cpu(p); 3523 if ((cpu != smp_processor_id()) && task_curr(p)) 3524 smp_send_reschedule(cpu); 3525 preempt_enable(); 3526 } 3527 EXPORT_SYMBOL_GPL(kick_process); 3528 3529 /* 3530 * ->cpus_ptr is protected by both rq->lock and p->pi_lock 3531 * 3532 * A few notes on cpu_active vs cpu_online: 3533 * 3534 * - cpu_active must be a subset of cpu_online 3535 * 3536 * - on CPU-up we allow per-CPU kthreads on the online && !active CPU, 3537 * see __set_cpus_allowed_ptr(). At this point the newly online 3538 * CPU isn't yet part of the sched domains, and balancing will not 3539 * see it. 3540 * 3541 * - on CPU-down we clear cpu_active() to mask the sched domains and 3542 * avoid the load balancer to place new tasks on the to be removed 3543 * CPU. Existing tasks will remain running there and will be taken 3544 * off. 3545 * 3546 * This means that fallback selection must not select !active CPUs. 3547 * And can assume that any active CPU must be online. Conversely 3548 * select_task_rq() below may allow selection of !active CPUs in order 3549 * to satisfy the above rules. 3550 */ 3551 static int select_fallback_rq(int cpu, struct task_struct *p) 3552 { 3553 int nid = cpu_to_node(cpu); 3554 const struct cpumask *nodemask = NULL; 3555 enum { cpuset, possible, fail } state = cpuset; 3556 int dest_cpu; 3557 3558 /* 3559 * If the node that the CPU is on has been offlined, cpu_to_node() 3560 * will return -1. There is no CPU on the node, and we should 3561 * select the CPU on the other node. 3562 */ 3563 if (nid != -1) { 3564 nodemask = cpumask_of_node(nid); 3565 3566 /* Look for allowed, online CPU in same node. */ 3567 for_each_cpu(dest_cpu, nodemask) { 3568 if (is_cpu_allowed(p, dest_cpu)) 3569 return dest_cpu; 3570 } 3571 } 3572 3573 for (;;) { 3574 /* Any allowed, online CPU? */ 3575 for_each_cpu(dest_cpu, p->cpus_ptr) { 3576 if (!is_cpu_allowed(p, dest_cpu)) 3577 continue; 3578 3579 goto out; 3580 } 3581 3582 /* No more Mr. Nice Guy. */ 3583 switch (state) { 3584 case cpuset: 3585 if (cpuset_cpus_allowed_fallback(p)) { 3586 state = possible; 3587 break; 3588 } 3589 fallthrough; 3590 case possible: 3591 /* 3592 * XXX When called from select_task_rq() we only 3593 * hold p->pi_lock and again violate locking order. 3594 * 3595 * More yuck to audit. 3596 */ 3597 do_set_cpus_allowed(p, task_cpu_possible_mask(p)); 3598 state = fail; 3599 break; 3600 case fail: 3601 BUG(); 3602 break; 3603 } 3604 } 3605 3606 out: 3607 if (state != cpuset) { 3608 /* 3609 * Don't tell them about moving exiting tasks or 3610 * kernel threads (both mm NULL), since they never 3611 * leave kernel. 3612 */ 3613 if (p->mm && printk_ratelimit()) { 3614 printk_deferred("process %d (%s) no longer affine to cpu%d\n", 3615 task_pid_nr(p), p->comm, cpu); 3616 } 3617 } 3618 3619 return dest_cpu; 3620 } 3621 3622 /* 3623 * The caller (fork, wakeup) owns p->pi_lock, ->cpus_ptr is stable. 3624 */ 3625 static inline 3626 int select_task_rq(struct task_struct *p, int cpu, int wake_flags) 3627 { 3628 lockdep_assert_held(&p->pi_lock); 3629 3630 if (p->nr_cpus_allowed > 1 && !is_migration_disabled(p)) 3631 cpu = p->sched_class->select_task_rq(p, cpu, wake_flags); 3632 else 3633 cpu = cpumask_any(p->cpus_ptr); 3634 3635 /* 3636 * In order not to call set_task_cpu() on a blocking task we need 3637 * to rely on ttwu() to place the task on a valid ->cpus_ptr 3638 * CPU. 3639 * 3640 * Since this is common to all placement strategies, this lives here. 3641 * 3642 * [ this allows ->select_task() to simply return task_cpu(p) and 3643 * not worry about this generic constraint ] 3644 */ 3645 if (unlikely(!is_cpu_allowed(p, cpu))) 3646 cpu = select_fallback_rq(task_cpu(p), p); 3647 3648 return cpu; 3649 } 3650 3651 void sched_set_stop_task(int cpu, struct task_struct *stop) 3652 { 3653 static struct lock_class_key stop_pi_lock; 3654 struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 }; 3655 struct task_struct *old_stop = cpu_rq(cpu)->stop; 3656 3657 if (stop) { 3658 /* 3659 * Make it appear like a SCHED_FIFO task, its something 3660 * userspace knows about and won't get confused about. 3661 * 3662 * Also, it will make PI more or less work without too 3663 * much confusion -- but then, stop work should not 3664 * rely on PI working anyway. 3665 */ 3666 sched_setscheduler_nocheck(stop, SCHED_FIFO, ¶m); 3667 3668 stop->sched_class = &stop_sched_class; 3669 3670 /* 3671 * The PI code calls rt_mutex_setprio() with ->pi_lock held to 3672 * adjust the effective priority of a task. As a result, 3673 * rt_mutex_setprio() can trigger (RT) balancing operations, 3674 * which can then trigger wakeups of the stop thread to push 3675 * around the current task. 3676 * 3677 * The stop task itself will never be part of the PI-chain, it 3678 * never blocks, therefore that ->pi_lock recursion is safe. 3679 * Tell lockdep about this by placing the stop->pi_lock in its 3680 * own class. 3681 */ 3682 lockdep_set_class(&stop->pi_lock, &stop_pi_lock); 3683 } 3684 3685 cpu_rq(cpu)->stop = stop; 3686 3687 if (old_stop) { 3688 /* 3689 * Reset it back to a normal scheduling class so that 3690 * it can die in pieces. 3691 */ 3692 old_stop->sched_class = &rt_sched_class; 3693 } 3694 } 3695 3696 #else /* CONFIG_SMP */ 3697 3698 static inline int __set_cpus_allowed_ptr(struct task_struct *p, 3699 struct affinity_context *ctx) 3700 { 3701 return set_cpus_allowed_ptr(p, ctx->new_mask); 3702 } 3703 3704 static inline void migrate_disable_switch(struct rq *rq, struct task_struct *p) { } 3705 3706 static inline bool rq_has_pinned_tasks(struct rq *rq) 3707 { 3708 return false; 3709 } 3710 3711 static inline cpumask_t *alloc_user_cpus_ptr(int node) 3712 { 3713 return NULL; 3714 } 3715 3716 #endif /* !CONFIG_SMP */ 3717 3718 static void 3719 ttwu_stat(struct task_struct *p, int cpu, int wake_flags) 3720 { 3721 struct rq *rq; 3722 3723 if (!schedstat_enabled()) 3724 return; 3725 3726 rq = this_rq(); 3727 3728 #ifdef CONFIG_SMP 3729 if (cpu == rq->cpu) { 3730 __schedstat_inc(rq->ttwu_local); 3731 __schedstat_inc(p->stats.nr_wakeups_local); 3732 } else { 3733 struct sched_domain *sd; 3734 3735 __schedstat_inc(p->stats.nr_wakeups_remote); 3736 rcu_read_lock(); 3737 for_each_domain(rq->cpu, sd) { 3738 if (cpumask_test_cpu(cpu, sched_domain_span(sd))) { 3739 __schedstat_inc(sd->ttwu_wake_remote); 3740 break; 3741 } 3742 } 3743 rcu_read_unlock(); 3744 } 3745 3746 if (wake_flags & WF_MIGRATED) 3747 __schedstat_inc(p->stats.nr_wakeups_migrate); 3748 #endif /* CONFIG_SMP */ 3749 3750 __schedstat_inc(rq->ttwu_count); 3751 __schedstat_inc(p->stats.nr_wakeups); 3752 3753 if (wake_flags & WF_SYNC) 3754 __schedstat_inc(p->stats.nr_wakeups_sync); 3755 } 3756 3757 /* 3758 * Mark the task runnable. 3759 */ 3760 static inline void ttwu_do_wakeup(struct task_struct *p) 3761 { 3762 WRITE_ONCE(p->__state, TASK_RUNNING); 3763 trace_sched_wakeup(p); 3764 } 3765 3766 static void 3767 ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags, 3768 struct rq_flags *rf) 3769 { 3770 int en_flags = ENQUEUE_WAKEUP | ENQUEUE_NOCLOCK; 3771 3772 lockdep_assert_rq_held(rq); 3773 3774 if (p->sched_contributes_to_load) 3775 rq->nr_uninterruptible--; 3776 3777 #ifdef CONFIG_SMP 3778 if (wake_flags & WF_MIGRATED) 3779 en_flags |= ENQUEUE_MIGRATED; 3780 else 3781 #endif 3782 if (p->in_iowait) { 3783 delayacct_blkio_end(p); 3784 atomic_dec(&task_rq(p)->nr_iowait); 3785 } 3786 3787 activate_task(rq, p, en_flags); 3788 check_preempt_curr(rq, p, wake_flags); 3789 3790 ttwu_do_wakeup(p); 3791 3792 #ifdef CONFIG_SMP 3793 if (p->sched_class->task_woken) { 3794 /* 3795 * Our task @p is fully woken up and running; so it's safe to 3796 * drop the rq->lock, hereafter rq is only used for statistics. 3797 */ 3798 rq_unpin_lock(rq, rf); 3799 p->sched_class->task_woken(rq, p); 3800 rq_repin_lock(rq, rf); 3801 } 3802 3803 if (rq->idle_stamp) { 3804 u64 delta = rq_clock(rq) - rq->idle_stamp; 3805 u64 max = 2*rq->max_idle_balance_cost; 3806 3807 update_avg(&rq->avg_idle, delta); 3808 3809 if (rq->avg_idle > max) 3810 rq->avg_idle = max; 3811 3812 rq->wake_stamp = jiffies; 3813 rq->wake_avg_idle = rq->avg_idle / 2; 3814 3815 rq->idle_stamp = 0; 3816 } 3817 #endif 3818 } 3819 3820 /* 3821 * Consider @p being inside a wait loop: 3822 * 3823 * for (;;) { 3824 * set_current_state(TASK_UNINTERRUPTIBLE); 3825 * 3826 * if (CONDITION) 3827 * break; 3828 * 3829 * schedule(); 3830 * } 3831 * __set_current_state(TASK_RUNNING); 3832 * 3833 * between set_current_state() and schedule(). In this case @p is still 3834 * runnable, so all that needs doing is change p->state back to TASK_RUNNING in 3835 * an atomic manner. 3836 * 3837 * By taking task_rq(p)->lock we serialize against schedule(), if @p->on_rq 3838 * then schedule() must still happen and p->state can be changed to 3839 * TASK_RUNNING. Otherwise we lost the race, schedule() has happened, and we 3840 * need to do a full wakeup with enqueue. 3841 * 3842 * Returns: %true when the wakeup is done, 3843 * %false otherwise. 3844 */ 3845 static int ttwu_runnable(struct task_struct *p, int wake_flags) 3846 { 3847 struct rq_flags rf; 3848 struct rq *rq; 3849 int ret = 0; 3850 3851 rq = __task_rq_lock(p, &rf); 3852 if (task_on_rq_queued(p)) { 3853 if (!task_on_cpu(rq, p)) { 3854 /* 3855 * When on_rq && !on_cpu the task is preempted, see if 3856 * it should preempt the task that is current now. 3857 */ 3858 update_rq_clock(rq); 3859 check_preempt_curr(rq, p, wake_flags); 3860 } 3861 ttwu_do_wakeup(p); 3862 ret = 1; 3863 } 3864 __task_rq_unlock(rq, &rf); 3865 3866 return ret; 3867 } 3868 3869 #ifdef CONFIG_SMP 3870 void sched_ttwu_pending(void *arg) 3871 { 3872 struct llist_node *llist = arg; 3873 struct rq *rq = this_rq(); 3874 struct task_struct *p, *t; 3875 struct rq_flags rf; 3876 3877 if (!llist) 3878 return; 3879 3880 rq_lock_irqsave(rq, &rf); 3881 update_rq_clock(rq); 3882 3883 llist_for_each_entry_safe(p, t, llist, wake_entry.llist) { 3884 if (WARN_ON_ONCE(p->on_cpu)) 3885 smp_cond_load_acquire(&p->on_cpu, !VAL); 3886 3887 if (WARN_ON_ONCE(task_cpu(p) != cpu_of(rq))) 3888 set_task_cpu(p, cpu_of(rq)); 3889 3890 ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0, &rf); 3891 } 3892 3893 /* 3894 * Must be after enqueueing at least once task such that 3895 * idle_cpu() does not observe a false-negative -- if it does, 3896 * it is possible for select_idle_siblings() to stack a number 3897 * of tasks on this CPU during that window. 3898 * 3899 * It is ok to clear ttwu_pending when another task pending. 3900 * We will receive IPI after local irq enabled and then enqueue it. 3901 * Since now nr_running > 0, idle_cpu() will always get correct result. 3902 */ 3903 WRITE_ONCE(rq->ttwu_pending, 0); 3904 rq_unlock_irqrestore(rq, &rf); 3905 } 3906 3907 /* 3908 * Prepare the scene for sending an IPI for a remote smp_call 3909 * 3910 * Returns true if the caller can proceed with sending the IPI. 3911 * Returns false otherwise. 3912 */ 3913 bool call_function_single_prep_ipi(int cpu) 3914 { 3915 if (set_nr_if_polling(cpu_rq(cpu)->idle)) { 3916 trace_sched_wake_idle_without_ipi(cpu); 3917 return false; 3918 } 3919 3920 return true; 3921 } 3922 3923 /* 3924 * Queue a task on the target CPUs wake_list and wake the CPU via IPI if 3925 * necessary. The wakee CPU on receipt of the IPI will queue the task 3926 * via sched_ttwu_wakeup() for activation so the wakee incurs the cost 3927 * of the wakeup instead of the waker. 3928 */ 3929 static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) 3930 { 3931 struct rq *rq = cpu_rq(cpu); 3932 3933 p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED); 3934 3935 WRITE_ONCE(rq->ttwu_pending, 1); 3936 __smp_call_single_queue(cpu, &p->wake_entry.llist); 3937 } 3938 3939 void wake_up_if_idle(int cpu) 3940 { 3941 struct rq *rq = cpu_rq(cpu); 3942 struct rq_flags rf; 3943 3944 rcu_read_lock(); 3945 3946 if (!is_idle_task(rcu_dereference(rq->curr))) 3947 goto out; 3948 3949 rq_lock_irqsave(rq, &rf); 3950 if (is_idle_task(rq->curr)) 3951 resched_curr(rq); 3952 /* Else CPU is not idle, do nothing here: */ 3953 rq_unlock_irqrestore(rq, &rf); 3954 3955 out: 3956 rcu_read_unlock(); 3957 } 3958 3959 bool cpus_share_cache(int this_cpu, int that_cpu) 3960 { 3961 if (this_cpu == that_cpu) 3962 return true; 3963 3964 return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu); 3965 } 3966 3967 static inline bool ttwu_queue_cond(struct task_struct *p, int cpu) 3968 { 3969 /* 3970 * Do not complicate things with the async wake_list while the CPU is 3971 * in hotplug state. 3972 */ 3973 if (!cpu_active(cpu)) 3974 return false; 3975 3976 /* Ensure the task will still be allowed to run on the CPU. */ 3977 if (!cpumask_test_cpu(cpu, p->cpus_ptr)) 3978 return false; 3979 3980 /* 3981 * If the CPU does not share cache, then queue the task on the 3982 * remote rqs wakelist to avoid accessing remote data. 3983 */ 3984 if (!cpus_share_cache(smp_processor_id(), cpu)) 3985 return true; 3986 3987 if (cpu == smp_processor_id()) 3988 return false; 3989 3990 /* 3991 * If the wakee cpu is idle, or the task is descheduling and the 3992 * only running task on the CPU, then use the wakelist to offload 3993 * the task activation to the idle (or soon-to-be-idle) CPU as 3994 * the current CPU is likely busy. nr_running is checked to 3995 * avoid unnecessary task stacking. 3996 * 3997 * Note that we can only get here with (wakee) p->on_rq=0, 3998 * p->on_cpu can be whatever, we've done the dequeue, so 3999 * the wakee has been accounted out of ->nr_running. 4000 */ 4001 if (!cpu_rq(cpu)->nr_running) 4002 return true; 4003 4004 return false; 4005 } 4006 4007 static bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) 4008 { 4009 if (sched_feat(TTWU_QUEUE) && ttwu_queue_cond(p, cpu)) { 4010 sched_clock_cpu(cpu); /* Sync clocks across CPUs */ 4011 __ttwu_queue_wakelist(p, cpu, wake_flags); 4012 return true; 4013 } 4014 4015 return false; 4016 } 4017 4018 #else /* !CONFIG_SMP */ 4019 4020 static inline bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) 4021 { 4022 return false; 4023 } 4024 4025 #endif /* CONFIG_SMP */ 4026 4027 static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags) 4028 { 4029 struct rq *rq = cpu_rq(cpu); 4030 struct rq_flags rf; 4031 4032 if (ttwu_queue_wakelist(p, cpu, wake_flags)) 4033 return; 4034 4035 rq_lock(rq, &rf); 4036 update_rq_clock(rq); 4037 ttwu_do_activate(rq, p, wake_flags, &rf); 4038 rq_unlock(rq, &rf); 4039 } 4040 4041 /* 4042 * Invoked from try_to_wake_up() to check whether the task can be woken up. 4043 * 4044 * The caller holds p::pi_lock if p != current or has preemption 4045 * disabled when p == current. 4046 * 4047 * The rules of PREEMPT_RT saved_state: 4048 * 4049 * The related locking code always holds p::pi_lock when updating 4050 * p::saved_state, which means the code is fully serialized in both cases. 4051 * 4052 * The lock wait and lock wakeups happen via TASK_RTLOCK_WAIT. No other 4053 * bits set. This allows to distinguish all wakeup scenarios. 4054 */ 4055 static __always_inline 4056 bool ttwu_state_match(struct task_struct *p, unsigned int state, int *success) 4057 { 4058 int match; 4059 4060 if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)) { 4061 WARN_ON_ONCE((state & TASK_RTLOCK_WAIT) && 4062 state != TASK_RTLOCK_WAIT); 4063 } 4064 4065 *success = !!(match = __task_state_match(p, state)); 4066 4067 #ifdef CONFIG_PREEMPT_RT 4068 /* 4069 * Saved state preserves the task state across blocking on 4070 * an RT lock. If the state matches, set p::saved_state to 4071 * TASK_RUNNING, but do not wake the task because it waits 4072 * for a lock wakeup. Also indicate success because from 4073 * the regular waker's point of view this has succeeded. 4074 * 4075 * After acquiring the lock the task will restore p::__state 4076 * from p::saved_state which ensures that the regular 4077 * wakeup is not lost. The restore will also set 4078 * p::saved_state to TASK_RUNNING so any further tests will 4079 * not result in false positives vs. @success 4080 */ 4081 if (match < 0) 4082 p->saved_state = TASK_RUNNING; 4083 #endif 4084 return match > 0; 4085 } 4086 4087 /* 4088 * Notes on Program-Order guarantees on SMP systems. 4089 * 4090 * MIGRATION 4091 * 4092 * The basic program-order guarantee on SMP systems is that when a task [t] 4093 * migrates, all its activity on its old CPU [c0] happens-before any subsequent 4094 * execution on its new CPU [c1]. 4095 * 4096 * For migration (of runnable tasks) this is provided by the following means: 4097 * 4098 * A) UNLOCK of the rq(c0)->lock scheduling out task t 4099 * B) migration for t is required to synchronize *both* rq(c0)->lock and 4100 * rq(c1)->lock (if not at the same time, then in that order). 4101 * C) LOCK of the rq(c1)->lock scheduling in task 4102 * 4103 * Release/acquire chaining guarantees that B happens after A and C after B. 4104 * Note: the CPU doing B need not be c0 or c1 4105 * 4106 * Example: 4107 * 4108 * CPU0 CPU1 CPU2 4109 * 4110 * LOCK rq(0)->lock 4111 * sched-out X 4112 * sched-in Y 4113 * UNLOCK rq(0)->lock 4114 * 4115 * LOCK rq(0)->lock // orders against CPU0 4116 * dequeue X 4117 * UNLOCK rq(0)->lock 4118 * 4119 * LOCK rq(1)->lock 4120 * enqueue X 4121 * UNLOCK rq(1)->lock 4122 * 4123 * LOCK rq(1)->lock // orders against CPU2 4124 * sched-out Z 4125 * sched-in X 4126 * UNLOCK rq(1)->lock 4127 * 4128 * 4129 * BLOCKING -- aka. SLEEP + WAKEUP 4130 * 4131 * For blocking we (obviously) need to provide the same guarantee as for 4132 * migration. However the means are completely different as there is no lock 4133 * chain to provide order. Instead we do: 4134 * 4135 * 1) smp_store_release(X->on_cpu, 0) -- finish_task() 4136 * 2) smp_cond_load_acquire(!X->on_cpu) -- try_to_wake_up() 4137 * 4138 * Example: 4139 * 4140 * CPU0 (schedule) CPU1 (try_to_wake_up) CPU2 (schedule) 4141 * 4142 * LOCK rq(0)->lock LOCK X->pi_lock 4143 * dequeue X 4144 * sched-out X 4145 * smp_store_release(X->on_cpu, 0); 4146 * 4147 * smp_cond_load_acquire(&X->on_cpu, !VAL); 4148 * X->state = WAKING 4149 * set_task_cpu(X,2) 4150 * 4151 * LOCK rq(2)->lock 4152 * enqueue X 4153 * X->state = RUNNING 4154 * UNLOCK rq(2)->lock 4155 * 4156 * LOCK rq(2)->lock // orders against CPU1 4157 * sched-out Z 4158 * sched-in X 4159 * UNLOCK rq(2)->lock 4160 * 4161 * UNLOCK X->pi_lock 4162 * UNLOCK rq(0)->lock 4163 * 4164 * 4165 * However, for wakeups there is a second guarantee we must provide, namely we 4166 * must ensure that CONDITION=1 done by the caller can not be reordered with 4167 * accesses to the task state; see try_to_wake_up() and set_current_state(). 4168 */ 4169 4170 /** 4171 * try_to_wake_up - wake up a thread 4172 * @p: the thread to be awakened 4173 * @state: the mask of task states that can be woken 4174 * @wake_flags: wake modifier flags (WF_*) 4175 * 4176 * Conceptually does: 4177 * 4178 * If (@state & @p->state) @p->state = TASK_RUNNING. 4179 * 4180 * If the task was not queued/runnable, also place it back on a runqueue. 4181 * 4182 * This function is atomic against schedule() which would dequeue the task. 4183 * 4184 * It issues a full memory barrier before accessing @p->state, see the comment 4185 * with set_current_state(). 4186 * 4187 * Uses p->pi_lock to serialize against concurrent wake-ups. 4188 * 4189 * Relies on p->pi_lock stabilizing: 4190 * - p->sched_class 4191 * - p->cpus_ptr 4192 * - p->sched_task_group 4193 * in order to do migration, see its use of select_task_rq()/set_task_cpu(). 4194 * 4195 * Tries really hard to only take one task_rq(p)->lock for performance. 4196 * Takes rq->lock in: 4197 * - ttwu_runnable() -- old rq, unavoidable, see comment there; 4198 * - ttwu_queue() -- new rq, for enqueue of the task; 4199 * - psi_ttwu_dequeue() -- much sadness :-( accounting will kill us. 4200 * 4201 * As a consequence we race really badly with just about everything. See the 4202 * many memory barriers and their comments for details. 4203 * 4204 * Return: %true if @p->state changes (an actual wakeup was done), 4205 * %false otherwise. 4206 */ 4207 static int 4208 try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) 4209 { 4210 unsigned long flags; 4211 int cpu, success = 0; 4212 4213 preempt_disable(); 4214 if (p == current) { 4215 /* 4216 * We're waking current, this means 'p->on_rq' and 'task_cpu(p) 4217 * == smp_processor_id()'. Together this means we can special 4218 * case the whole 'p->on_rq && ttwu_runnable()' case below 4219 * without taking any locks. 4220 * 4221 * In particular: 4222 * - we rely on Program-Order guarantees for all the ordering, 4223 * - we're serialized against set_special_state() by virtue of 4224 * it disabling IRQs (this allows not taking ->pi_lock). 4225 */ 4226 if (!ttwu_state_match(p, state, &success)) 4227 goto out; 4228 4229 trace_sched_waking(p); 4230 ttwu_do_wakeup(p); 4231 goto out; 4232 } 4233 4234 /* 4235 * If we are going to wake up a thread waiting for CONDITION we 4236 * need to ensure that CONDITION=1 done by the caller can not be 4237 * reordered with p->state check below. This pairs with smp_store_mb() 4238 * in set_current_state() that the waiting thread does. 4239 */ 4240 raw_spin_lock_irqsave(&p->pi_lock, flags); 4241 smp_mb__after_spinlock(); 4242 if (!ttwu_state_match(p, state, &success)) 4243 goto unlock; 4244 4245 trace_sched_waking(p); 4246 4247 /* 4248 * Ensure we load p->on_rq _after_ p->state, otherwise it would 4249 * be possible to, falsely, observe p->on_rq == 0 and get stuck 4250 * in smp_cond_load_acquire() below. 4251 * 4252 * sched_ttwu_pending() try_to_wake_up() 4253 * STORE p->on_rq = 1 LOAD p->state 4254 * UNLOCK rq->lock 4255 * 4256 * __schedule() (switch to task 'p') 4257 * LOCK rq->lock smp_rmb(); 4258 * smp_mb__after_spinlock(); 4259 * UNLOCK rq->lock 4260 * 4261 * [task p] 4262 * STORE p->state = UNINTERRUPTIBLE LOAD p->on_rq 4263 * 4264 * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in 4265 * __schedule(). See the comment for smp_mb__after_spinlock(). 4266 * 4267 * A similar smb_rmb() lives in try_invoke_on_locked_down_task(). 4268 */ 4269 smp_rmb(); 4270 if (READ_ONCE(p->on_rq) && ttwu_runnable(p, wake_flags)) 4271 goto unlock; 4272 4273 #ifdef CONFIG_SMP 4274 /* 4275 * Ensure we load p->on_cpu _after_ p->on_rq, otherwise it would be 4276 * possible to, falsely, observe p->on_cpu == 0. 4277 * 4278 * One must be running (->on_cpu == 1) in order to remove oneself 4279 * from the runqueue. 4280 * 4281 * __schedule() (switch to task 'p') try_to_wake_up() 4282 * STORE p->on_cpu = 1 LOAD p->on_rq 4283 * UNLOCK rq->lock 4284 * 4285 * __schedule() (put 'p' to sleep) 4286 * LOCK rq->lock smp_rmb(); 4287 * smp_mb__after_spinlock(); 4288 * STORE p->on_rq = 0 LOAD p->on_cpu 4289 * 4290 * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in 4291 * __schedule(). See the comment for smp_mb__after_spinlock(). 4292 * 4293 * Form a control-dep-acquire with p->on_rq == 0 above, to ensure 4294 * schedule()'s deactivate_task() has 'happened' and p will no longer 4295 * care about it's own p->state. See the comment in __schedule(). 4296 */ 4297 smp_acquire__after_ctrl_dep(); 4298 4299 /* 4300 * We're doing the wakeup (@success == 1), they did a dequeue (p->on_rq 4301 * == 0), which means we need to do an enqueue, change p->state to 4302 * TASK_WAKING such that we can unlock p->pi_lock before doing the 4303 * enqueue, such as ttwu_queue_wakelist(). 4304 */ 4305 WRITE_ONCE(p->__state, TASK_WAKING); 4306 4307 /* 4308 * If the owning (remote) CPU is still in the middle of schedule() with 4309 * this task as prev, considering queueing p on the remote CPUs wake_list 4310 * which potentially sends an IPI instead of spinning on p->on_cpu to 4311 * let the waker make forward progress. This is safe because IRQs are 4312 * disabled and the IPI will deliver after on_cpu is cleared. 4313 * 4314 * Ensure we load task_cpu(p) after p->on_cpu: 4315 * 4316 * set_task_cpu(p, cpu); 4317 * STORE p->cpu = @cpu 4318 * __schedule() (switch to task 'p') 4319 * LOCK rq->lock 4320 * smp_mb__after_spin_lock() smp_cond_load_acquire(&p->on_cpu) 4321 * STORE p->on_cpu = 1 LOAD p->cpu 4322 * 4323 * to ensure we observe the correct CPU on which the task is currently 4324 * scheduling. 4325 */ 4326 if (smp_load_acquire(&p->on_cpu) && 4327 ttwu_queue_wakelist(p, task_cpu(p), wake_flags)) 4328 goto unlock; 4329 4330 /* 4331 * If the owning (remote) CPU is still in the middle of schedule() with 4332 * this task as prev, wait until it's done referencing the task. 4333 * 4334 * Pairs with the smp_store_release() in finish_task(). 4335 * 4336 * This ensures that tasks getting woken will be fully ordered against 4337 * their previous state and preserve Program Order. 4338 */ 4339 smp_cond_load_acquire(&p->on_cpu, !VAL); 4340 4341 cpu = select_task_rq(p, p->wake_cpu, wake_flags | WF_TTWU); 4342 if (task_cpu(p) != cpu) { 4343 if (p->in_iowait) { 4344 delayacct_blkio_end(p); 4345 atomic_dec(&task_rq(p)->nr_iowait); 4346 } 4347 4348 wake_flags |= WF_MIGRATED; 4349 psi_ttwu_dequeue(p); 4350 set_task_cpu(p, cpu); 4351 } 4352 #else 4353 cpu = task_cpu(p); 4354 #endif /* CONFIG_SMP */ 4355 4356 ttwu_queue(p, cpu, wake_flags); 4357 unlock: 4358 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 4359 out: 4360 if (success) 4361 ttwu_stat(p, task_cpu(p), wake_flags); 4362 preempt_enable(); 4363 4364 return success; 4365 } 4366 4367 static bool __task_needs_rq_lock(struct task_struct *p) 4368 { 4369 unsigned int state = READ_ONCE(p->__state); 4370 4371 /* 4372 * Since pi->lock blocks try_to_wake_up(), we don't need rq->lock when 4373 * the task is blocked. Make sure to check @state since ttwu() can drop 4374 * locks at the end, see ttwu_queue_wakelist(). 4375 */ 4376 if (state == TASK_RUNNING || state == TASK_WAKING) 4377 return true; 4378 4379 /* 4380 * Ensure we load p->on_rq after p->__state, otherwise it would be 4381 * possible to, falsely, observe p->on_rq == 0. 4382 * 4383 * See try_to_wake_up() for a longer comment. 4384 */ 4385 smp_rmb(); 4386 if (p->on_rq) 4387 return true; 4388 4389 #ifdef CONFIG_SMP 4390 /* 4391 * Ensure the task has finished __schedule() and will not be referenced 4392 * anymore. Again, see try_to_wake_up() for a longer comment. 4393 */ 4394 smp_rmb(); 4395 smp_cond_load_acquire(&p->on_cpu, !VAL); 4396 #endif 4397 4398 return false; 4399 } 4400 4401 /** 4402 * task_call_func - Invoke a function on task in fixed state 4403 * @p: Process for which the function is to be invoked, can be @current. 4404 * @func: Function to invoke. 4405 * @arg: Argument to function. 4406 * 4407 * Fix the task in it's current state by avoiding wakeups and or rq operations 4408 * and call @func(@arg) on it. This function can use ->on_rq and task_curr() 4409 * to work out what the state is, if required. Given that @func can be invoked 4410 * with a runqueue lock held, it had better be quite lightweight. 4411 * 4412 * Returns: 4413 * Whatever @func returns 4414 */ 4415 int task_call_func(struct task_struct *p, task_call_f func, void *arg) 4416 { 4417 struct rq *rq = NULL; 4418 struct rq_flags rf; 4419 int ret; 4420 4421 raw_spin_lock_irqsave(&p->pi_lock, rf.flags); 4422 4423 if (__task_needs_rq_lock(p)) 4424 rq = __task_rq_lock(p, &rf); 4425 4426 /* 4427 * At this point the task is pinned; either: 4428 * - blocked and we're holding off wakeups (pi->lock) 4429 * - woken, and we're holding off enqueue (rq->lock) 4430 * - queued, and we're holding off schedule (rq->lock) 4431 * - running, and we're holding off de-schedule (rq->lock) 4432 * 4433 * The called function (@func) can use: task_curr(), p->on_rq and 4434 * p->__state to differentiate between these states. 4435 */ 4436 ret = func(p, arg); 4437 4438 if (rq) 4439 rq_unlock(rq, &rf); 4440 4441 raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags); 4442 return ret; 4443 } 4444 4445 /** 4446 * cpu_curr_snapshot - Return a snapshot of the currently running task 4447 * @cpu: The CPU on which to snapshot the task. 4448 * 4449 * Returns the task_struct pointer of the task "currently" running on 4450 * the specified CPU. If the same task is running on that CPU throughout, 4451 * the return value will be a pointer to that task's task_struct structure. 4452 * If the CPU did any context switches even vaguely concurrently with the 4453 * execution of this function, the return value will be a pointer to the 4454 * task_struct structure of a randomly chosen task that was running on 4455 * that CPU somewhere around the time that this function was executing. 4456 * 4457 * If the specified CPU was offline, the return value is whatever it 4458 * is, perhaps a pointer to the task_struct structure of that CPU's idle 4459 * task, but there is no guarantee. Callers wishing a useful return 4460 * value must take some action to ensure that the specified CPU remains 4461 * online throughout. 4462 * 4463 * This function executes full memory barriers before and after fetching 4464 * the pointer, which permits the caller to confine this function's fetch 4465 * with respect to the caller's accesses to other shared variables. 4466 */ 4467 struct task_struct *cpu_curr_snapshot(int cpu) 4468 { 4469 struct task_struct *t; 4470 4471 smp_mb(); /* Pairing determined by caller's synchronization design. */ 4472 t = rcu_dereference(cpu_curr(cpu)); 4473 smp_mb(); /* Pairing determined by caller's synchronization design. */ 4474 return t; 4475 } 4476 4477 /** 4478 * wake_up_process - Wake up a specific process 4479 * @p: The process to be woken up. 4480 * 4481 * Attempt to wake up the nominated process and move it to the set of runnable 4482 * processes. 4483 * 4484 * Return: 1 if the process was woken up, 0 if it was already running. 4485 * 4486 * This function executes a full memory barrier before accessing the task state. 4487 */ 4488 int wake_up_process(struct task_struct *p) 4489 { 4490 return try_to_wake_up(p, TASK_NORMAL, 0); 4491 } 4492 EXPORT_SYMBOL(wake_up_process); 4493 4494 int wake_up_state(struct task_struct *p, unsigned int state) 4495 { 4496 return try_to_wake_up(p, state, 0); 4497 } 4498 4499 /* 4500 * Perform scheduler related setup for a newly forked process p. 4501 * p is forked by current. 4502 * 4503 * __sched_fork() is basic setup used by init_idle() too: 4504 */ 4505 static void __sched_fork(unsigned long clone_flags, struct task_struct *p) 4506 { 4507 p->on_rq = 0; 4508 4509 p->se.on_rq = 0; 4510 p->se.exec_start = 0; 4511 p->se.sum_exec_runtime = 0; 4512 p->se.prev_sum_exec_runtime = 0; 4513 p->se.nr_migrations = 0; 4514 p->se.vruntime = 0; 4515 p->se.vlag = 0; 4516 p->se.slice = sysctl_sched_base_slice; 4517 INIT_LIST_HEAD(&p->se.group_node); 4518 4519 #ifdef CONFIG_FAIR_GROUP_SCHED 4520 p->se.cfs_rq = NULL; 4521 #endif 4522 4523 #ifdef CONFIG_SCHEDSTATS 4524 /* Even if schedstat is disabled, there should not be garbage */ 4525 memset(&p->stats, 0, sizeof(p->stats)); 4526 #endif 4527 4528 RB_CLEAR_NODE(&p->dl.rb_node); 4529 init_dl_task_timer(&p->dl); 4530 init_dl_inactive_task_timer(&p->dl); 4531 __dl_clear_params(p); 4532 4533 INIT_LIST_HEAD(&p->rt.run_list); 4534 p->rt.timeout = 0; 4535 p->rt.time_slice = sched_rr_timeslice; 4536 p->rt.on_rq = 0; 4537 p->rt.on_list = 0; 4538 4539 #ifdef CONFIG_PREEMPT_NOTIFIERS 4540 INIT_HLIST_HEAD(&p->preempt_notifiers); 4541 #endif 4542 4543 #ifdef CONFIG_COMPACTION 4544 p->capture_control = NULL; 4545 #endif 4546 init_numa_balancing(clone_flags, p); 4547 #ifdef CONFIG_SMP 4548 p->wake_entry.u_flags = CSD_TYPE_TTWU; 4549 p->migration_pending = NULL; 4550 #endif 4551 init_sched_mm_cid(p); 4552 } 4553 4554 DEFINE_STATIC_KEY_FALSE(sched_numa_balancing); 4555 4556 #ifdef CONFIG_NUMA_BALANCING 4557 4558 int sysctl_numa_balancing_mode; 4559 4560 static void __set_numabalancing_state(bool enabled) 4561 { 4562 if (enabled) 4563 static_branch_enable(&sched_numa_balancing); 4564 else 4565 static_branch_disable(&sched_numa_balancing); 4566 } 4567 4568 void set_numabalancing_state(bool enabled) 4569 { 4570 if (enabled) 4571 sysctl_numa_balancing_mode = NUMA_BALANCING_NORMAL; 4572 else 4573 sysctl_numa_balancing_mode = NUMA_BALANCING_DISABLED; 4574 __set_numabalancing_state(enabled); 4575 } 4576 4577 #ifdef CONFIG_PROC_SYSCTL 4578 static void reset_memory_tiering(void) 4579 { 4580 struct pglist_data *pgdat; 4581 4582 for_each_online_pgdat(pgdat) { 4583 pgdat->nbp_threshold = 0; 4584 pgdat->nbp_th_nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE); 4585 pgdat->nbp_th_start = jiffies_to_msecs(jiffies); 4586 } 4587 } 4588 4589 static int sysctl_numa_balancing(struct ctl_table *table, int write, 4590 void *buffer, size_t *lenp, loff_t *ppos) 4591 { 4592 struct ctl_table t; 4593 int err; 4594 int state = sysctl_numa_balancing_mode; 4595 4596 if (write && !capable(CAP_SYS_ADMIN)) 4597 return -EPERM; 4598 4599 t = *table; 4600 t.data = &state; 4601 err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); 4602 if (err < 0) 4603 return err; 4604 if (write) { 4605 if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) && 4606 (state & NUMA_BALANCING_MEMORY_TIERING)) 4607 reset_memory_tiering(); 4608 sysctl_numa_balancing_mode = state; 4609 __set_numabalancing_state(state); 4610 } 4611 return err; 4612 } 4613 #endif 4614 #endif 4615 4616 #ifdef CONFIG_SCHEDSTATS 4617 4618 DEFINE_STATIC_KEY_FALSE(sched_schedstats); 4619 4620 static void set_schedstats(bool enabled) 4621 { 4622 if (enabled) 4623 static_branch_enable(&sched_schedstats); 4624 else 4625 static_branch_disable(&sched_schedstats); 4626 } 4627 4628 void force_schedstat_enabled(void) 4629 { 4630 if (!schedstat_enabled()) { 4631 pr_info("kernel profiling enabled schedstats, disable via kernel.sched_schedstats.\n"); 4632 static_branch_enable(&sched_schedstats); 4633 } 4634 } 4635 4636 static int __init setup_schedstats(char *str) 4637 { 4638 int ret = 0; 4639 if (!str) 4640 goto out; 4641 4642 if (!strcmp(str, "enable")) { 4643 set_schedstats(true); 4644 ret = 1; 4645 } else if (!strcmp(str, "disable")) { 4646 set_schedstats(false); 4647 ret = 1; 4648 } 4649 out: 4650 if (!ret) 4651 pr_warn("Unable to parse schedstats=\n"); 4652 4653 return ret; 4654 } 4655 __setup("schedstats=", setup_schedstats); 4656 4657 #ifdef CONFIG_PROC_SYSCTL 4658 static int sysctl_schedstats(struct ctl_table *table, int write, void *buffer, 4659 size_t *lenp, loff_t *ppos) 4660 { 4661 struct ctl_table t; 4662 int err; 4663 int state = static_branch_likely(&sched_schedstats); 4664 4665 if (write && !capable(CAP_SYS_ADMIN)) 4666 return -EPERM; 4667 4668 t = *table; 4669 t.data = &state; 4670 err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); 4671 if (err < 0) 4672 return err; 4673 if (write) 4674 set_schedstats(state); 4675 return err; 4676 } 4677 #endif /* CONFIG_PROC_SYSCTL */ 4678 #endif /* CONFIG_SCHEDSTATS */ 4679 4680 #ifdef CONFIG_SYSCTL 4681 static struct ctl_table sched_core_sysctls[] = { 4682 #ifdef CONFIG_SCHEDSTATS 4683 { 4684 .procname = "sched_schedstats", 4685 .data = NULL, 4686 .maxlen = sizeof(unsigned int), 4687 .mode = 0644, 4688 .proc_handler = sysctl_schedstats, 4689 .extra1 = SYSCTL_ZERO, 4690 .extra2 = SYSCTL_ONE, 4691 }, 4692 #endif /* CONFIG_SCHEDSTATS */ 4693 #ifdef CONFIG_UCLAMP_TASK 4694 { 4695 .procname = "sched_util_clamp_min", 4696 .data = &sysctl_sched_uclamp_util_min, 4697 .maxlen = sizeof(unsigned int), 4698 .mode = 0644, 4699 .proc_handler = sysctl_sched_uclamp_handler, 4700 }, 4701 { 4702 .procname = "sched_util_clamp_max", 4703 .data = &sysctl_sched_uclamp_util_max, 4704 .maxlen = sizeof(unsigned int), 4705 .mode = 0644, 4706 .proc_handler = sysctl_sched_uclamp_handler, 4707 }, 4708 { 4709 .procname = "sched_util_clamp_min_rt_default", 4710 .data = &sysctl_sched_uclamp_util_min_rt_default, 4711 .maxlen = sizeof(unsigned int), 4712 .mode = 0644, 4713 .proc_handler = sysctl_sched_uclamp_handler, 4714 }, 4715 #endif /* CONFIG_UCLAMP_TASK */ 4716 #ifdef CONFIG_NUMA_BALANCING 4717 { 4718 .procname = "numa_balancing", 4719 .data = NULL, /* filled in by handler */ 4720 .maxlen = sizeof(unsigned int), 4721 .mode = 0644, 4722 .proc_handler = sysctl_numa_balancing, 4723 .extra1 = SYSCTL_ZERO, 4724 .extra2 = SYSCTL_FOUR, 4725 }, 4726 #endif /* CONFIG_NUMA_BALANCING */ 4727 {} 4728 }; 4729 static int __init sched_core_sysctl_init(void) 4730 { 4731 register_sysctl_init("kernel", sched_core_sysctls); 4732 return 0; 4733 } 4734 late_initcall(sched_core_sysctl_init); 4735 #endif /* CONFIG_SYSCTL */ 4736 4737 /* 4738 * fork()/clone()-time setup: 4739 */ 4740 int sched_fork(unsigned long clone_flags, struct task_struct *p) 4741 { 4742 __sched_fork(clone_flags, p); 4743 /* 4744 * We mark the process as NEW here. This guarantees that 4745 * nobody will actually run it, and a signal or other external 4746 * event cannot wake it up and insert it on the runqueue either. 4747 */ 4748 p->__state = TASK_NEW; 4749 4750 /* 4751 * Make sure we do not leak PI boosting priority to the child. 4752 */ 4753 p->prio = current->normal_prio; 4754 4755 uclamp_fork(p); 4756 4757 /* 4758 * Revert to default priority/policy on fork if requested. 4759 */ 4760 if (unlikely(p->sched_reset_on_fork)) { 4761 if (task_has_dl_policy(p) || task_has_rt_policy(p)) { 4762 p->policy = SCHED_NORMAL; 4763 p->static_prio = NICE_TO_PRIO(0); 4764 p->rt_priority = 0; 4765 } else if (PRIO_TO_NICE(p->static_prio) < 0) 4766 p->static_prio = NICE_TO_PRIO(0); 4767 4768 p->prio = p->normal_prio = p->static_prio; 4769 set_load_weight(p, false); 4770 4771 /* 4772 * We don't need the reset flag anymore after the fork. It has 4773 * fulfilled its duty: 4774 */ 4775 p->sched_reset_on_fork = 0; 4776 } 4777 4778 if (dl_prio(p->prio)) 4779 return -EAGAIN; 4780 else if (rt_prio(p->prio)) 4781 p->sched_class = &rt_sched_class; 4782 else 4783 p->sched_class = &fair_sched_class; 4784 4785 init_entity_runnable_average(&p->se); 4786 4787 4788 #ifdef CONFIG_SCHED_INFO 4789 if (likely(sched_info_on())) 4790 memset(&p->sched_info, 0, sizeof(p->sched_info)); 4791 #endif 4792 #if defined(CONFIG_SMP) 4793 p->on_cpu = 0; 4794 #endif 4795 init_task_preempt_count(p); 4796 #ifdef CONFIG_SMP 4797 plist_node_init(&p->pushable_tasks, MAX_PRIO); 4798 RB_CLEAR_NODE(&p->pushable_dl_tasks); 4799 #endif 4800 return 0; 4801 } 4802 4803 void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) 4804 { 4805 unsigned long flags; 4806 4807 /* 4808 * Because we're not yet on the pid-hash, p->pi_lock isn't strictly 4809 * required yet, but lockdep gets upset if rules are violated. 4810 */ 4811 raw_spin_lock_irqsave(&p->pi_lock, flags); 4812 #ifdef CONFIG_CGROUP_SCHED 4813 if (1) { 4814 struct task_group *tg; 4815 tg = container_of(kargs->cset->subsys[cpu_cgrp_id], 4816 struct task_group, css); 4817 tg = autogroup_task_group(p, tg); 4818 p->sched_task_group = tg; 4819 } 4820 #endif 4821 rseq_migrate(p); 4822 /* 4823 * We're setting the CPU for the first time, we don't migrate, 4824 * so use __set_task_cpu(). 4825 */ 4826 __set_task_cpu(p, smp_processor_id()); 4827 if (p->sched_class->task_fork) 4828 p->sched_class->task_fork(p); 4829 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 4830 } 4831 4832 void sched_post_fork(struct task_struct *p) 4833 { 4834 uclamp_post_fork(p); 4835 } 4836 4837 unsigned long to_ratio(u64 period, u64 runtime) 4838 { 4839 if (runtime == RUNTIME_INF) 4840 return BW_UNIT; 4841 4842 /* 4843 * Doing this here saves a lot of checks in all 4844 * the calling paths, and returning zero seems 4845 * safe for them anyway. 4846 */ 4847 if (period == 0) 4848 return 0; 4849 4850 return div64_u64(runtime << BW_SHIFT, period); 4851 } 4852 4853 /* 4854 * wake_up_new_task - wake up a newly created task for the first time. 4855 * 4856 * This function will do some initial scheduler statistics housekeeping 4857 * that must be done for every newly created context, then puts the task 4858 * on the runqueue and wakes it. 4859 */ 4860 void wake_up_new_task(struct task_struct *p) 4861 { 4862 struct rq_flags rf; 4863 struct rq *rq; 4864 4865 raw_spin_lock_irqsave(&p->pi_lock, rf.flags); 4866 WRITE_ONCE(p->__state, TASK_RUNNING); 4867 #ifdef CONFIG_SMP 4868 /* 4869 * Fork balancing, do it here and not earlier because: 4870 * - cpus_ptr can change in the fork path 4871 * - any previously selected CPU might disappear through hotplug 4872 * 4873 * Use __set_task_cpu() to avoid calling sched_class::migrate_task_rq, 4874 * as we're not fully set-up yet. 4875 */ 4876 p->recent_used_cpu = task_cpu(p); 4877 rseq_migrate(p); 4878 __set_task_cpu(p, select_task_rq(p, task_cpu(p), WF_FORK)); 4879 #endif 4880 rq = __task_rq_lock(p, &rf); 4881 update_rq_clock(rq); 4882 post_init_entity_util_avg(p); 4883 4884 activate_task(rq, p, ENQUEUE_NOCLOCK); 4885 trace_sched_wakeup_new(p); 4886 check_preempt_curr(rq, p, WF_FORK); 4887 #ifdef CONFIG_SMP 4888 if (p->sched_class->task_woken) { 4889 /* 4890 * Nothing relies on rq->lock after this, so it's fine to 4891 * drop it. 4892 */ 4893 rq_unpin_lock(rq, &rf); 4894 p->sched_class->task_woken(rq, p); 4895 rq_repin_lock(rq, &rf); 4896 } 4897 #endif 4898 task_rq_unlock(rq, p, &rf); 4899 } 4900 4901 #ifdef CONFIG_PREEMPT_NOTIFIERS 4902 4903 static DEFINE_STATIC_KEY_FALSE(preempt_notifier_key); 4904 4905 void preempt_notifier_inc(void) 4906 { 4907 static_branch_inc(&preempt_notifier_key); 4908 } 4909 EXPORT_SYMBOL_GPL(preempt_notifier_inc); 4910 4911 void preempt_notifier_dec(void) 4912 { 4913 static_branch_dec(&preempt_notifier_key); 4914 } 4915 EXPORT_SYMBOL_GPL(preempt_notifier_dec); 4916 4917 /** 4918 * preempt_notifier_register - tell me when current is being preempted & rescheduled 4919 * @notifier: notifier struct to register 4920 */ 4921 void preempt_notifier_register(struct preempt_notifier *notifier) 4922 { 4923 if (!static_branch_unlikely(&preempt_notifier_key)) 4924 WARN(1, "registering preempt_notifier while notifiers disabled\n"); 4925 4926 hlist_add_head(¬ifier->link, ¤t->preempt_notifiers); 4927 } 4928 EXPORT_SYMBOL_GPL(preempt_notifier_register); 4929 4930 /** 4931 * preempt_notifier_unregister - no longer interested in preemption notifications 4932 * @notifier: notifier struct to unregister 4933 * 4934 * This is *not* safe to call from within a preemption notifier. 4935 */ 4936 void preempt_notifier_unregister(struct preempt_notifier *notifier) 4937 { 4938 hlist_del(¬ifier->link); 4939 } 4940 EXPORT_SYMBOL_GPL(preempt_notifier_unregister); 4941 4942 static void __fire_sched_in_preempt_notifiers(struct task_struct *curr) 4943 { 4944 struct preempt_notifier *notifier; 4945 4946 hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) 4947 notifier->ops->sched_in(notifier, raw_smp_processor_id()); 4948 } 4949 4950 static __always_inline void fire_sched_in_preempt_notifiers(struct task_struct *curr) 4951 { 4952 if (static_branch_unlikely(&preempt_notifier_key)) 4953 __fire_sched_in_preempt_notifiers(curr); 4954 } 4955 4956 static void 4957 __fire_sched_out_preempt_notifiers(struct task_struct *curr, 4958 struct task_struct *next) 4959 { 4960 struct preempt_notifier *notifier; 4961 4962 hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) 4963 notifier->ops->sched_out(notifier, next); 4964 } 4965 4966 static __always_inline void 4967 fire_sched_out_preempt_notifiers(struct task_struct *curr, 4968 struct task_struct *next) 4969 { 4970 if (static_branch_unlikely(&preempt_notifier_key)) 4971 __fire_sched_out_preempt_notifiers(curr, next); 4972 } 4973 4974 #else /* !CONFIG_PREEMPT_NOTIFIERS */ 4975 4976 static inline void fire_sched_in_preempt_notifiers(struct task_struct *curr) 4977 { 4978 } 4979 4980 static inline void 4981 fire_sched_out_preempt_notifiers(struct task_struct *curr, 4982 struct task_struct *next) 4983 { 4984 } 4985 4986 #endif /* CONFIG_PREEMPT_NOTIFIERS */ 4987 4988 static inline void prepare_task(struct task_struct *next) 4989 { 4990 #ifdef CONFIG_SMP 4991 /* 4992 * Claim the task as running, we do this before switching to it 4993 * such that any running task will have this set. 4994 * 4995 * See the smp_load_acquire(&p->on_cpu) case in ttwu() and 4996 * its ordering comment. 4997 */ 4998 WRITE_ONCE(next->on_cpu, 1); 4999 #endif 5000 } 5001 5002 static inline void finish_task(struct task_struct *prev) 5003 { 5004 #ifdef CONFIG_SMP 5005 /* 5006 * This must be the very last reference to @prev from this CPU. After 5007 * p->on_cpu is cleared, the task can be moved to a different CPU. We 5008 * must ensure this doesn't happen until the switch is completely 5009 * finished. 5010 * 5011 * In particular, the load of prev->state in finish_task_switch() must 5012 * happen before this. 5013 * 5014 * Pairs with the smp_cond_load_acquire() in try_to_wake_up(). 5015 */ 5016 smp_store_release(&prev->on_cpu, 0); 5017 #endif 5018 } 5019 5020 #ifdef CONFIG_SMP 5021 5022 static void do_balance_callbacks(struct rq *rq, struct balance_callback *head) 5023 { 5024 void (*func)(struct rq *rq); 5025 struct balance_callback *next; 5026 5027 lockdep_assert_rq_held(rq); 5028 5029 while (head) { 5030 func = (void (*)(struct rq *))head->func; 5031 next = head->next; 5032 head->next = NULL; 5033 head = next; 5034 5035 func(rq); 5036 } 5037 } 5038 5039 static void balance_push(struct rq *rq); 5040 5041 /* 5042 * balance_push_callback is a right abuse of the callback interface and plays 5043 * by significantly different rules. 5044 * 5045 * Where the normal balance_callback's purpose is to be ran in the same context 5046 * that queued it (only later, when it's safe to drop rq->lock again), 5047 * balance_push_callback is specifically targeted at __schedule(). 5048 * 5049 * This abuse is tolerated because it places all the unlikely/odd cases behind 5050 * a single test, namely: rq->balance_callback == NULL. 5051 */ 5052 struct balance_callback balance_push_callback = { 5053 .next = NULL, 5054 .func = balance_push, 5055 }; 5056 5057 static inline struct balance_callback * 5058 __splice_balance_callbacks(struct rq *rq, bool split) 5059 { 5060 struct balance_callback *head = rq->balance_callback; 5061 5062 if (likely(!head)) 5063 return NULL; 5064 5065 lockdep_assert_rq_held(rq); 5066 /* 5067 * Must not take balance_push_callback off the list when 5068 * splice_balance_callbacks() and balance_callbacks() are not 5069 * in the same rq->lock section. 5070 * 5071 * In that case it would be possible for __schedule() to interleave 5072 * and observe the list empty. 5073 */ 5074 if (split && head == &balance_push_callback) 5075 head = NULL; 5076 else 5077 rq->balance_callback = NULL; 5078 5079 return head; 5080 } 5081 5082 static inline struct balance_callback *splice_balance_callbacks(struct rq *rq) 5083 { 5084 return __splice_balance_callbacks(rq, true); 5085 } 5086 5087 static void __balance_callbacks(struct rq *rq) 5088 { 5089 do_balance_callbacks(rq, __splice_balance_callbacks(rq, false)); 5090 } 5091 5092 static inline void balance_callbacks(struct rq *rq, struct balance_callback *head) 5093 { 5094 unsigned long flags; 5095 5096 if (unlikely(head)) { 5097 raw_spin_rq_lock_irqsave(rq, flags); 5098 do_balance_callbacks(rq, head); 5099 raw_spin_rq_unlock_irqrestore(rq, flags); 5100 } 5101 } 5102 5103 #else 5104 5105 static inline void __balance_callbacks(struct rq *rq) 5106 { 5107 } 5108 5109 static inline struct balance_callback *splice_balance_callbacks(struct rq *rq) 5110 { 5111 return NULL; 5112 } 5113 5114 static inline void balance_callbacks(struct rq *rq, struct balance_callback *head) 5115 { 5116 } 5117 5118 #endif 5119 5120 static inline void 5121 prepare_lock_switch(struct rq *rq, struct task_struct *next, struct rq_flags *rf) 5122 { 5123 /* 5124 * Since the runqueue lock will be released by the next 5125 * task (which is an invalid locking op but in the case 5126 * of the scheduler it's an obvious special-case), so we 5127 * do an early lockdep release here: 5128 */ 5129 rq_unpin_lock(rq, rf); 5130 spin_release(&__rq_lockp(rq)->dep_map, _THIS_IP_); 5131 #ifdef CONFIG_DEBUG_SPINLOCK 5132 /* this is a valid case when another task releases the spinlock */ 5133 rq_lockp(rq)->owner = next; 5134 #endif 5135 } 5136 5137 static inline void finish_lock_switch(struct rq *rq) 5138 { 5139 /* 5140 * If we are tracking spinlock dependencies then we have to 5141 * fix up the runqueue lock - which gets 'carried over' from 5142 * prev into current: 5143 */ 5144 spin_acquire(&__rq_lockp(rq)->dep_map, 0, 0, _THIS_IP_); 5145 __balance_callbacks(rq); 5146 raw_spin_rq_unlock_irq(rq); 5147 } 5148 5149 /* 5150 * NOP if the arch has not defined these: 5151 */ 5152 5153 #ifndef prepare_arch_switch 5154 # define prepare_arch_switch(next) do { } while (0) 5155 #endif 5156 5157 #ifndef finish_arch_post_lock_switch 5158 # define finish_arch_post_lock_switch() do { } while (0) 5159 #endif 5160 5161 static inline void kmap_local_sched_out(void) 5162 { 5163 #ifdef CONFIG_KMAP_LOCAL 5164 if (unlikely(current->kmap_ctrl.idx)) 5165 __kmap_local_sched_out(); 5166 #endif 5167 } 5168 5169 static inline void kmap_local_sched_in(void) 5170 { 5171 #ifdef CONFIG_KMAP_LOCAL 5172 if (unlikely(current->kmap_ctrl.idx)) 5173 __kmap_local_sched_in(); 5174 #endif 5175 } 5176 5177 /** 5178 * prepare_task_switch - prepare to switch tasks 5179 * @rq: the runqueue preparing to switch 5180 * @prev: the current task that is being switched out 5181 * @next: the task we are going to switch to. 5182 * 5183 * This is called with the rq lock held and interrupts off. It must 5184 * be paired with a subsequent finish_task_switch after the context 5185 * switch. 5186 * 5187 * prepare_task_switch sets up locking and calls architecture specific 5188 * hooks. 5189 */ 5190 static inline void 5191 prepare_task_switch(struct rq *rq, struct task_struct *prev, 5192 struct task_struct *next) 5193 { 5194 kcov_prepare_switch(prev); 5195 sched_info_switch(rq, prev, next); 5196 perf_event_task_sched_out(prev, next); 5197 rseq_preempt(prev); 5198 fire_sched_out_preempt_notifiers(prev, next); 5199 kmap_local_sched_out(); 5200 prepare_task(next); 5201 prepare_arch_switch(next); 5202 } 5203 5204 /** 5205 * finish_task_switch - clean up after a task-switch 5206 * @prev: the thread we just switched away from. 5207 * 5208 * finish_task_switch must be called after the context switch, paired 5209 * with a prepare_task_switch call before the context switch. 5210 * finish_task_switch will reconcile locking set up by prepare_task_switch, 5211 * and do any other architecture-specific cleanup actions. 5212 * 5213 * Note that we may have delayed dropping an mm in context_switch(). If 5214 * so, we finish that here outside of the runqueue lock. (Doing it 5215 * with the lock held can cause deadlocks; see schedule() for 5216 * details.) 5217 * 5218 * The context switch have flipped the stack from under us and restored the 5219 * local variables which were saved when this task called schedule() in the 5220 * past. prev == current is still correct but we need to recalculate this_rq 5221 * because prev may have moved to another CPU. 5222 */ 5223 static struct rq *finish_task_switch(struct task_struct *prev) 5224 __releases(rq->lock) 5225 { 5226 struct rq *rq = this_rq(); 5227 struct mm_struct *mm = rq->prev_mm; 5228 unsigned int prev_state; 5229 5230 /* 5231 * The previous task will have left us with a preempt_count of 2 5232 * because it left us after: 5233 * 5234 * schedule() 5235 * preempt_disable(); // 1 5236 * __schedule() 5237 * raw_spin_lock_irq(&rq->lock) // 2 5238 * 5239 * Also, see FORK_PREEMPT_COUNT. 5240 */ 5241 if (WARN_ONCE(preempt_count() != 2*PREEMPT_DISABLE_OFFSET, 5242 "corrupted preempt_count: %s/%d/0x%x\n", 5243 current->comm, current->pid, preempt_count())) 5244 preempt_count_set(FORK_PREEMPT_COUNT); 5245 5246 rq->prev_mm = NULL; 5247 5248 /* 5249 * A task struct has one reference for the use as "current". 5250 * If a task dies, then it sets TASK_DEAD in tsk->state and calls 5251 * schedule one last time. The schedule call will never return, and 5252 * the scheduled task must drop that reference. 5253 * 5254 * We must observe prev->state before clearing prev->on_cpu (in 5255 * finish_task), otherwise a concurrent wakeup can get prev 5256 * running on another CPU and we could rave with its RUNNING -> DEAD 5257 * transition, resulting in a double drop. 5258 */ 5259 prev_state = READ_ONCE(prev->__state); 5260 vtime_task_switch(prev); 5261 perf_event_task_sched_in(prev, current); 5262 finish_task(prev); 5263 tick_nohz_task_switch(); 5264 finish_lock_switch(rq); 5265 finish_arch_post_lock_switch(); 5266 kcov_finish_switch(current); 5267 /* 5268 * kmap_local_sched_out() is invoked with rq::lock held and 5269 * interrupts disabled. There is no requirement for that, but the 5270 * sched out code does not have an interrupt enabled section. 5271 * Restoring the maps on sched in does not require interrupts being 5272 * disabled either. 5273 */ 5274 kmap_local_sched_in(); 5275 5276 fire_sched_in_preempt_notifiers(current); 5277 /* 5278 * When switching through a kernel thread, the loop in 5279 * membarrier_{private,global}_expedited() may have observed that 5280 * kernel thread and not issued an IPI. It is therefore possible to 5281 * schedule between user->kernel->user threads without passing though 5282 * switch_mm(). Membarrier requires a barrier after storing to 5283 * rq->curr, before returning to userspace, so provide them here: 5284 * 5285 * - a full memory barrier for {PRIVATE,GLOBAL}_EXPEDITED, implicitly 5286 * provided by mmdrop_lazy_tlb(), 5287 * - a sync_core for SYNC_CORE. 5288 */ 5289 if (mm) { 5290 membarrier_mm_sync_core_before_usermode(mm); 5291 mmdrop_lazy_tlb_sched(mm); 5292 } 5293 5294 if (unlikely(prev_state == TASK_DEAD)) { 5295 if (prev->sched_class->task_dead) 5296 prev->sched_class->task_dead(prev); 5297 5298 /* Task is done with its stack. */ 5299 put_task_stack(prev); 5300 5301 put_task_struct_rcu_user(prev); 5302 } 5303 5304 return rq; 5305 } 5306 5307 /** 5308 * schedule_tail - first thing a freshly forked thread must call. 5309 * @prev: the thread we just switched away from. 5310 */ 5311 asmlinkage __visible void schedule_tail(struct task_struct *prev) 5312 __releases(rq->lock) 5313 { 5314 /* 5315 * New tasks start with FORK_PREEMPT_COUNT, see there and 5316 * finish_task_switch() for details. 5317 * 5318 * finish_task_switch() will drop rq->lock() and lower preempt_count 5319 * and the preempt_enable() will end up enabling preemption (on 5320 * PREEMPT_COUNT kernels). 5321 */ 5322 5323 finish_task_switch(prev); 5324 preempt_enable(); 5325 5326 if (current->set_child_tid) 5327 put_user(task_pid_vnr(current), current->set_child_tid); 5328 5329 calculate_sigpending(); 5330 } 5331 5332 /* 5333 * context_switch - switch to the new MM and the new thread's register state. 5334 */ 5335 static __always_inline struct rq * 5336 context_switch(struct rq *rq, struct task_struct *prev, 5337 struct task_struct *next, struct rq_flags *rf) 5338 { 5339 prepare_task_switch(rq, prev, next); 5340 5341 /* 5342 * For paravirt, this is coupled with an exit in switch_to to 5343 * combine the page table reload and the switch backend into 5344 * one hypercall. 5345 */ 5346 arch_start_context_switch(prev); 5347 5348 /* 5349 * kernel -> kernel lazy + transfer active 5350 * user -> kernel lazy + mmgrab_lazy_tlb() active 5351 * 5352 * kernel -> user switch + mmdrop_lazy_tlb() active 5353 * user -> user switch 5354 * 5355 * switch_mm_cid() needs to be updated if the barriers provided 5356 * by context_switch() are modified. 5357 */ 5358 if (!next->mm) { // to kernel 5359 enter_lazy_tlb(prev->active_mm, next); 5360 5361 next->active_mm = prev->active_mm; 5362 if (prev->mm) // from user 5363 mmgrab_lazy_tlb(prev->active_mm); 5364 else 5365 prev->active_mm = NULL; 5366 } else { // to user 5367 membarrier_switch_mm(rq, prev->active_mm, next->mm); 5368 /* 5369 * sys_membarrier() requires an smp_mb() between setting 5370 * rq->curr / membarrier_switch_mm() and returning to userspace. 5371 * 5372 * The below provides this either through switch_mm(), or in 5373 * case 'prev->active_mm == next->mm' through 5374 * finish_task_switch()'s mmdrop(). 5375 */ 5376 switch_mm_irqs_off(prev->active_mm, next->mm, next); 5377 lru_gen_use_mm(next->mm); 5378 5379 if (!prev->mm) { // from kernel 5380 /* will mmdrop_lazy_tlb() in finish_task_switch(). */ 5381 rq->prev_mm = prev->active_mm; 5382 prev->active_mm = NULL; 5383 } 5384 } 5385 5386 /* switch_mm_cid() requires the memory barriers above. */ 5387 switch_mm_cid(rq, prev, next); 5388 5389 rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP); 5390 5391 prepare_lock_switch(rq, next, rf); 5392 5393 /* Here we just switch the register state and the stack. */ 5394 switch_to(prev, next, prev); 5395 barrier(); 5396 5397 return finish_task_switch(prev); 5398 } 5399 5400 /* 5401 * nr_running and nr_context_switches: 5402 * 5403 * externally visible scheduler statistics: current number of runnable 5404 * threads, total number of context switches performed since bootup. 5405 */ 5406 unsigned int nr_running(void) 5407 { 5408 unsigned int i, sum = 0; 5409 5410 for_each_online_cpu(i) 5411 sum += cpu_rq(i)->nr_running; 5412 5413 return sum; 5414 } 5415 5416 /* 5417 * Check if only the current task is running on the CPU. 5418 * 5419 * Caution: this function does not check that the caller has disabled 5420 * preemption, thus the result might have a time-of-check-to-time-of-use 5421 * race. The caller is responsible to use it correctly, for example: 5422 * 5423 * - from a non-preemptible section (of course) 5424 * 5425 * - from a thread that is bound to a single CPU 5426 * 5427 * - in a loop with very short iterations (e.g. a polling loop) 5428 */ 5429 bool single_task_running(void) 5430 { 5431 return raw_rq()->nr_running == 1; 5432 } 5433 EXPORT_SYMBOL(single_task_running); 5434 5435 unsigned long long nr_context_switches_cpu(int cpu) 5436 { 5437 return cpu_rq(cpu)->nr_switches; 5438 } 5439 5440 unsigned long long nr_context_switches(void) 5441 { 5442 int i; 5443 unsigned long long sum = 0; 5444 5445 for_each_possible_cpu(i) 5446 sum += cpu_rq(i)->nr_switches; 5447 5448 return sum; 5449 } 5450 5451 /* 5452 * Consumers of these two interfaces, like for example the cpuidle menu 5453 * governor, are using nonsensical data. Preferring shallow idle state selection 5454 * for a CPU that has IO-wait which might not even end up running the task when 5455 * it does become runnable. 5456 */ 5457 5458 unsigned int nr_iowait_cpu(int cpu) 5459 { 5460 return atomic_read(&cpu_rq(cpu)->nr_iowait); 5461 } 5462 5463 /* 5464 * IO-wait accounting, and how it's mostly bollocks (on SMP). 5465 * 5466 * The idea behind IO-wait account is to account the idle time that we could 5467 * have spend running if it were not for IO. That is, if we were to improve the 5468 * storage performance, we'd have a proportional reduction in IO-wait time. 5469 * 5470 * This all works nicely on UP, where, when a task blocks on IO, we account 5471 * idle time as IO-wait, because if the storage were faster, it could've been 5472 * running and we'd not be idle. 5473 * 5474 * This has been extended to SMP, by doing the same for each CPU. This however 5475 * is broken. 5476 * 5477 * Imagine for instance the case where two tasks block on one CPU, only the one 5478 * CPU will have IO-wait accounted, while the other has regular idle. Even 5479 * though, if the storage were faster, both could've ran at the same time, 5480 * utilising both CPUs. 5481 * 5482 * This means, that when looking globally, the current IO-wait accounting on 5483 * SMP is a lower bound, by reason of under accounting. 5484 * 5485 * Worse, since the numbers are provided per CPU, they are sometimes 5486 * interpreted per CPU, and that is nonsensical. A blocked task isn't strictly 5487 * associated with any one particular CPU, it can wake to another CPU than it 5488 * blocked on. This means the per CPU IO-wait number is meaningless. 5489 * 5490 * Task CPU affinities can make all that even more 'interesting'. 5491 */ 5492 5493 unsigned int nr_iowait(void) 5494 { 5495 unsigned int i, sum = 0; 5496 5497 for_each_possible_cpu(i) 5498 sum += nr_iowait_cpu(i); 5499 5500 return sum; 5501 } 5502 5503 #ifdef CONFIG_SMP 5504 5505 /* 5506 * sched_exec - execve() is a valuable balancing opportunity, because at 5507 * this point the task has the smallest effective memory and cache footprint. 5508 */ 5509 void sched_exec(void) 5510 { 5511 struct task_struct *p = current; 5512 unsigned long flags; 5513 int dest_cpu; 5514 5515 raw_spin_lock_irqsave(&p->pi_lock, flags); 5516 dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), WF_EXEC); 5517 if (dest_cpu == smp_processor_id()) 5518 goto unlock; 5519 5520 if (likely(cpu_active(dest_cpu))) { 5521 struct migration_arg arg = { p, dest_cpu }; 5522 5523 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 5524 stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg); 5525 return; 5526 } 5527 unlock: 5528 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 5529 } 5530 5531 #endif 5532 5533 DEFINE_PER_CPU(struct kernel_stat, kstat); 5534 DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat); 5535 5536 EXPORT_PER_CPU_SYMBOL(kstat); 5537 EXPORT_PER_CPU_SYMBOL(kernel_cpustat); 5538 5539 /* 5540 * The function fair_sched_class.update_curr accesses the struct curr 5541 * and its field curr->exec_start; when called from task_sched_runtime(), 5542 * we observe a high rate of cache misses in practice. 5543 * Prefetching this data results in improved performance. 5544 */ 5545 static inline void prefetch_curr_exec_start(struct task_struct *p) 5546 { 5547 #ifdef CONFIG_FAIR_GROUP_SCHED 5548 struct sched_entity *curr = (&p->se)->cfs_rq->curr; 5549 #else 5550 struct sched_entity *curr = (&task_rq(p)->cfs)->curr; 5551 #endif 5552 prefetch(curr); 5553 prefetch(&curr->exec_start); 5554 } 5555 5556 /* 5557 * Return accounted runtime for the task. 5558 * In case the task is currently running, return the runtime plus current's 5559 * pending runtime that have not been accounted yet. 5560 */ 5561 unsigned long long task_sched_runtime(struct task_struct *p) 5562 { 5563 struct rq_flags rf; 5564 struct rq *rq; 5565 u64 ns; 5566 5567 #if defined(CONFIG_64BIT) && defined(CONFIG_SMP) 5568 /* 5569 * 64-bit doesn't need locks to atomically read a 64-bit value. 5570 * So we have a optimization chance when the task's delta_exec is 0. 5571 * Reading ->on_cpu is racy, but this is ok. 5572 * 5573 * If we race with it leaving CPU, we'll take a lock. So we're correct. 5574 * If we race with it entering CPU, unaccounted time is 0. This is 5575 * indistinguishable from the read occurring a few cycles earlier. 5576 * If we see ->on_cpu without ->on_rq, the task is leaving, and has 5577 * been accounted, so we're correct here as well. 5578 */ 5579 if (!p->on_cpu || !task_on_rq_queued(p)) 5580 return p->se.sum_exec_runtime; 5581 #endif 5582 5583 rq = task_rq_lock(p, &rf); 5584 /* 5585 * Must be ->curr _and_ ->on_rq. If dequeued, we would 5586 * project cycles that may never be accounted to this 5587 * thread, breaking clock_gettime(). 5588 */ 5589 if (task_current(rq, p) && task_on_rq_queued(p)) { 5590 prefetch_curr_exec_start(p); 5591 update_rq_clock(rq); 5592 p->sched_class->update_curr(rq); 5593 } 5594 ns = p->se.sum_exec_runtime; 5595 task_rq_unlock(rq, p, &rf); 5596 5597 return ns; 5598 } 5599 5600 #ifdef CONFIG_SCHED_DEBUG 5601 static u64 cpu_resched_latency(struct rq *rq) 5602 { 5603 int latency_warn_ms = READ_ONCE(sysctl_resched_latency_warn_ms); 5604 u64 resched_latency, now = rq_clock(rq); 5605 static bool warned_once; 5606 5607 if (sysctl_resched_latency_warn_once && warned_once) 5608 return 0; 5609 5610 if (!need_resched() || !latency_warn_ms) 5611 return 0; 5612 5613 if (system_state == SYSTEM_BOOTING) 5614 return 0; 5615 5616 if (!rq->last_seen_need_resched_ns) { 5617 rq->last_seen_need_resched_ns = now; 5618 rq->ticks_without_resched = 0; 5619 return 0; 5620 } 5621 5622 rq->ticks_without_resched++; 5623 resched_latency = now - rq->last_seen_need_resched_ns; 5624 if (resched_latency <= latency_warn_ms * NSEC_PER_MSEC) 5625 return 0; 5626 5627 warned_once = true; 5628 5629 return resched_latency; 5630 } 5631 5632 static int __init setup_resched_latency_warn_ms(char *str) 5633 { 5634 long val; 5635 5636 if ((kstrtol(str, 0, &val))) { 5637 pr_warn("Unable to set resched_latency_warn_ms\n"); 5638 return 1; 5639 } 5640 5641 sysctl_resched_latency_warn_ms = val; 5642 return 1; 5643 } 5644 __setup("resched_latency_warn_ms=", setup_resched_latency_warn_ms); 5645 #else 5646 static inline u64 cpu_resched_latency(struct rq *rq) { return 0; } 5647 #endif /* CONFIG_SCHED_DEBUG */ 5648 5649 /* 5650 * This function gets called by the timer code, with HZ frequency. 5651 * We call it with interrupts disabled. 5652 */ 5653 void scheduler_tick(void) 5654 { 5655 int cpu = smp_processor_id(); 5656 struct rq *rq = cpu_rq(cpu); 5657 struct task_struct *curr = rq->curr; 5658 struct rq_flags rf; 5659 unsigned long thermal_pressure; 5660 u64 resched_latency; 5661 5662 if (housekeeping_cpu(cpu, HK_TYPE_TICK)) 5663 arch_scale_freq_tick(); 5664 5665 sched_clock_tick(); 5666 5667 rq_lock(rq, &rf); 5668 5669 update_rq_clock(rq); 5670 thermal_pressure = arch_scale_thermal_pressure(cpu_of(rq)); 5671 update_thermal_load_avg(rq_clock_thermal(rq), rq, thermal_pressure); 5672 curr->sched_class->task_tick(rq, curr, 0); 5673 if (sched_feat(LATENCY_WARN)) 5674 resched_latency = cpu_resched_latency(rq); 5675 calc_global_load_tick(rq); 5676 sched_core_tick(rq); 5677 task_tick_mm_cid(rq, curr); 5678 5679 rq_unlock(rq, &rf); 5680 5681 if (sched_feat(LATENCY_WARN) && resched_latency) 5682 resched_latency_warn(cpu, resched_latency); 5683 5684 perf_event_task_tick(); 5685 5686 if (curr->flags & PF_WQ_WORKER) 5687 wq_worker_tick(curr); 5688 5689 #ifdef CONFIG_SMP 5690 rq->idle_balance = idle_cpu(cpu); 5691 trigger_load_balance(rq); 5692 #endif 5693 } 5694 5695 #ifdef CONFIG_NO_HZ_FULL 5696 5697 struct tick_work { 5698 int cpu; 5699 atomic_t state; 5700 struct delayed_work work; 5701 }; 5702 /* Values for ->state, see diagram below. */ 5703 #define TICK_SCHED_REMOTE_OFFLINE 0 5704 #define TICK_SCHED_REMOTE_OFFLINING 1 5705 #define TICK_SCHED_REMOTE_RUNNING 2 5706 5707 /* 5708 * State diagram for ->state: 5709 * 5710 * 5711 * TICK_SCHED_REMOTE_OFFLINE 5712 * | ^ 5713 * | | 5714 * | | sched_tick_remote() 5715 * | | 5716 * | | 5717 * +--TICK_SCHED_REMOTE_OFFLINING 5718 * | ^ 5719 * | | 5720 * sched_tick_start() | | sched_tick_stop() 5721 * | | 5722 * V | 5723 * TICK_SCHED_REMOTE_RUNNING 5724 * 5725 * 5726 * Other transitions get WARN_ON_ONCE(), except that sched_tick_remote() 5727 * and sched_tick_start() are happy to leave the state in RUNNING. 5728 */ 5729 5730 static struct tick_work __percpu *tick_work_cpu; 5731 5732 static void sched_tick_remote(struct work_struct *work) 5733 { 5734 struct delayed_work *dwork = to_delayed_work(work); 5735 struct tick_work *twork = container_of(dwork, struct tick_work, work); 5736 int cpu = twork->cpu; 5737 struct rq *rq = cpu_rq(cpu); 5738 struct task_struct *curr; 5739 struct rq_flags rf; 5740 u64 delta; 5741 int os; 5742 5743 /* 5744 * Handle the tick only if it appears the remote CPU is running in full 5745 * dynticks mode. The check is racy by nature, but missing a tick or 5746 * having one too much is no big deal because the scheduler tick updates 5747 * statistics and checks timeslices in a time-independent way, regardless 5748 * of when exactly it is running. 5749 */ 5750 if (!tick_nohz_tick_stopped_cpu(cpu)) 5751 goto out_requeue; 5752 5753 rq_lock_irq(rq, &rf); 5754 curr = rq->curr; 5755 if (cpu_is_offline(cpu)) 5756 goto out_unlock; 5757 5758 update_rq_clock(rq); 5759 5760 if (!is_idle_task(curr)) { 5761 /* 5762 * Make sure the next tick runs within a reasonable 5763 * amount of time. 5764 */ 5765 delta = rq_clock_task(rq) - curr->se.exec_start; 5766 WARN_ON_ONCE(delta > (u64)NSEC_PER_SEC * 3); 5767 } 5768 curr->sched_class->task_tick(rq, curr, 0); 5769 5770 calc_load_nohz_remote(rq); 5771 out_unlock: 5772 rq_unlock_irq(rq, &rf); 5773 out_requeue: 5774 5775 /* 5776 * Run the remote tick once per second (1Hz). This arbitrary 5777 * frequency is large enough to avoid overload but short enough 5778 * to keep scheduler internal stats reasonably up to date. But 5779 * first update state to reflect hotplug activity if required. 5780 */ 5781 os = atomic_fetch_add_unless(&twork->state, -1, TICK_SCHED_REMOTE_RUNNING); 5782 WARN_ON_ONCE(os == TICK_SCHED_REMOTE_OFFLINE); 5783 if (os == TICK_SCHED_REMOTE_RUNNING) 5784 queue_delayed_work(system_unbound_wq, dwork, HZ); 5785 } 5786 5787 static void sched_tick_start(int cpu) 5788 { 5789 int os; 5790 struct tick_work *twork; 5791 5792 if (housekeeping_cpu(cpu, HK_TYPE_TICK)) 5793 return; 5794 5795 WARN_ON_ONCE(!tick_work_cpu); 5796 5797 twork = per_cpu_ptr(tick_work_cpu, cpu); 5798 os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_RUNNING); 5799 WARN_ON_ONCE(os == TICK_SCHED_REMOTE_RUNNING); 5800 if (os == TICK_SCHED_REMOTE_OFFLINE) { 5801 twork->cpu = cpu; 5802 INIT_DELAYED_WORK(&twork->work, sched_tick_remote); 5803 queue_delayed_work(system_unbound_wq, &twork->work, HZ); 5804 } 5805 } 5806 5807 #ifdef CONFIG_HOTPLUG_CPU 5808 static void sched_tick_stop(int cpu) 5809 { 5810 struct tick_work *twork; 5811 int os; 5812 5813 if (housekeeping_cpu(cpu, HK_TYPE_TICK)) 5814 return; 5815 5816 WARN_ON_ONCE(!tick_work_cpu); 5817 5818 twork = per_cpu_ptr(tick_work_cpu, cpu); 5819 /* There cannot be competing actions, but don't rely on stop-machine. */ 5820 os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_OFFLINING); 5821 WARN_ON_ONCE(os != TICK_SCHED_REMOTE_RUNNING); 5822 /* Don't cancel, as this would mess up the state machine. */ 5823 } 5824 #endif /* CONFIG_HOTPLUG_CPU */ 5825 5826 int __init sched_tick_offload_init(void) 5827 { 5828 tick_work_cpu = alloc_percpu(struct tick_work); 5829 BUG_ON(!tick_work_cpu); 5830 return 0; 5831 } 5832 5833 #else /* !CONFIG_NO_HZ_FULL */ 5834 static inline void sched_tick_start(int cpu) { } 5835 static inline void sched_tick_stop(int cpu) { } 5836 #endif 5837 5838 #if defined(CONFIG_PREEMPTION) && (defined(CONFIG_DEBUG_PREEMPT) || \ 5839 defined(CONFIG_TRACE_PREEMPT_TOGGLE)) 5840 /* 5841 * If the value passed in is equal to the current preempt count 5842 * then we just disabled preemption. Start timing the latency. 5843 */ 5844 static inline void preempt_latency_start(int val) 5845 { 5846 if (preempt_count() == val) { 5847 unsigned long ip = get_lock_parent_ip(); 5848 #ifdef CONFIG_DEBUG_PREEMPT 5849 current->preempt_disable_ip = ip; 5850 #endif 5851 trace_preempt_off(CALLER_ADDR0, ip); 5852 } 5853 } 5854 5855 void preempt_count_add(int val) 5856 { 5857 #ifdef CONFIG_DEBUG_PREEMPT 5858 /* 5859 * Underflow? 5860 */ 5861 if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) 5862 return; 5863 #endif 5864 __preempt_count_add(val); 5865 #ifdef CONFIG_DEBUG_PREEMPT 5866 /* 5867 * Spinlock count overflowing soon? 5868 */ 5869 DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >= 5870 PREEMPT_MASK - 10); 5871 #endif 5872 preempt_latency_start(val); 5873 } 5874 EXPORT_SYMBOL(preempt_count_add); 5875 NOKPROBE_SYMBOL(preempt_count_add); 5876 5877 /* 5878 * If the value passed in equals to the current preempt count 5879 * then we just enabled preemption. Stop timing the latency. 5880 */ 5881 static inline void preempt_latency_stop(int val) 5882 { 5883 if (preempt_count() == val) 5884 trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip()); 5885 } 5886 5887 void preempt_count_sub(int val) 5888 { 5889 #ifdef CONFIG_DEBUG_PREEMPT 5890 /* 5891 * Underflow? 5892 */ 5893 if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) 5894 return; 5895 /* 5896 * Is the spinlock portion underflowing? 5897 */ 5898 if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) && 5899 !(preempt_count() & PREEMPT_MASK))) 5900 return; 5901 #endif 5902 5903 preempt_latency_stop(val); 5904 __preempt_count_sub(val); 5905 } 5906 EXPORT_SYMBOL(preempt_count_sub); 5907 NOKPROBE_SYMBOL(preempt_count_sub); 5908 5909 #else 5910 static inline void preempt_latency_start(int val) { } 5911 static inline void preempt_latency_stop(int val) { } 5912 #endif 5913 5914 static inline unsigned long get_preempt_disable_ip(struct task_struct *p) 5915 { 5916 #ifdef CONFIG_DEBUG_PREEMPT 5917 return p->preempt_disable_ip; 5918 #else 5919 return 0; 5920 #endif 5921 } 5922 5923 /* 5924 * Print scheduling while atomic bug: 5925 */ 5926 static noinline void __schedule_bug(struct task_struct *prev) 5927 { 5928 /* Save this before calling printk(), since that will clobber it */ 5929 unsigned long preempt_disable_ip = get_preempt_disable_ip(current); 5930 5931 if (oops_in_progress) 5932 return; 5933 5934 printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n", 5935 prev->comm, prev->pid, preempt_count()); 5936 5937 debug_show_held_locks(prev); 5938 print_modules(); 5939 if (irqs_disabled()) 5940 print_irqtrace_events(prev); 5941 if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) 5942 && in_atomic_preempt_off()) { 5943 pr_err("Preemption disabled at:"); 5944 print_ip_sym(KERN_ERR, preempt_disable_ip); 5945 } 5946 check_panic_on_warn("scheduling while atomic"); 5947 5948 dump_stack(); 5949 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 5950 } 5951 5952 /* 5953 * Various schedule()-time debugging checks and statistics: 5954 */ 5955 static inline void schedule_debug(struct task_struct *prev, bool preempt) 5956 { 5957 #ifdef CONFIG_SCHED_STACK_END_CHECK 5958 if (task_stack_end_corrupted(prev)) 5959 panic("corrupted stack end detected inside scheduler\n"); 5960 5961 if (task_scs_end_corrupted(prev)) 5962 panic("corrupted shadow stack detected inside scheduler\n"); 5963 #endif 5964 5965 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP 5966 if (!preempt && READ_ONCE(prev->__state) && prev->non_block_count) { 5967 printk(KERN_ERR "BUG: scheduling in a non-blocking section: %s/%d/%i\n", 5968 prev->comm, prev->pid, prev->non_block_count); 5969 dump_stack(); 5970 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 5971 } 5972 #endif 5973 5974 if (unlikely(in_atomic_preempt_off())) { 5975 __schedule_bug(prev); 5976 preempt_count_set(PREEMPT_DISABLED); 5977 } 5978 rcu_sleep_check(); 5979 SCHED_WARN_ON(ct_state() == CONTEXT_USER); 5980 5981 profile_hit(SCHED_PROFILING, __builtin_return_address(0)); 5982 5983 schedstat_inc(this_rq()->sched_count); 5984 } 5985 5986 static void put_prev_task_balance(struct rq *rq, struct task_struct *prev, 5987 struct rq_flags *rf) 5988 { 5989 #ifdef CONFIG_SMP 5990 const struct sched_class *class; 5991 /* 5992 * We must do the balancing pass before put_prev_task(), such 5993 * that when we release the rq->lock the task is in the same 5994 * state as before we took rq->lock. 5995 * 5996 * We can terminate the balance pass as soon as we know there is 5997 * a runnable task of @class priority or higher. 5998 */ 5999 for_class_range(class, prev->sched_class, &idle_sched_class) { 6000 if (class->balance(rq, prev, rf)) 6001 break; 6002 } 6003 #endif 6004 6005 put_prev_task(rq, prev); 6006 } 6007 6008 /* 6009 * Pick up the highest-prio task: 6010 */ 6011 static inline struct task_struct * 6012 __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 6013 { 6014 const struct sched_class *class; 6015 struct task_struct *p; 6016 6017 /* 6018 * Optimization: we know that if all tasks are in the fair class we can 6019 * call that function directly, but only if the @prev task wasn't of a 6020 * higher scheduling class, because otherwise those lose the 6021 * opportunity to pull in more work from other CPUs. 6022 */ 6023 if (likely(!sched_class_above(prev->sched_class, &fair_sched_class) && 6024 rq->nr_running == rq->cfs.h_nr_running)) { 6025 6026 p = pick_next_task_fair(rq, prev, rf); 6027 if (unlikely(p == RETRY_TASK)) 6028 goto restart; 6029 6030 /* Assume the next prioritized class is idle_sched_class */ 6031 if (!p) { 6032 put_prev_task(rq, prev); 6033 p = pick_next_task_idle(rq); 6034 } 6035 6036 return p; 6037 } 6038 6039 restart: 6040 put_prev_task_balance(rq, prev, rf); 6041 6042 for_each_class(class) { 6043 p = class->pick_next_task(rq); 6044 if (p) 6045 return p; 6046 } 6047 6048 BUG(); /* The idle class should always have a runnable task. */ 6049 } 6050 6051 #ifdef CONFIG_SCHED_CORE 6052 static inline bool is_task_rq_idle(struct task_struct *t) 6053 { 6054 return (task_rq(t)->idle == t); 6055 } 6056 6057 static inline bool cookie_equals(struct task_struct *a, unsigned long cookie) 6058 { 6059 return is_task_rq_idle(a) || (a->core_cookie == cookie); 6060 } 6061 6062 static inline bool cookie_match(struct task_struct *a, struct task_struct *b) 6063 { 6064 if (is_task_rq_idle(a) || is_task_rq_idle(b)) 6065 return true; 6066 6067 return a->core_cookie == b->core_cookie; 6068 } 6069 6070 static inline struct task_struct *pick_task(struct rq *rq) 6071 { 6072 const struct sched_class *class; 6073 struct task_struct *p; 6074 6075 for_each_class(class) { 6076 p = class->pick_task(rq); 6077 if (p) 6078 return p; 6079 } 6080 6081 BUG(); /* The idle class should always have a runnable task. */ 6082 } 6083 6084 extern void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi); 6085 6086 static void queue_core_balance(struct rq *rq); 6087 6088 static struct task_struct * 6089 pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 6090 { 6091 struct task_struct *next, *p, *max = NULL; 6092 const struct cpumask *smt_mask; 6093 bool fi_before = false; 6094 bool core_clock_updated = (rq == rq->core); 6095 unsigned long cookie; 6096 int i, cpu, occ = 0; 6097 struct rq *rq_i; 6098 bool need_sync; 6099 6100 if (!sched_core_enabled(rq)) 6101 return __pick_next_task(rq, prev, rf); 6102 6103 cpu = cpu_of(rq); 6104 6105 /* Stopper task is switching into idle, no need core-wide selection. */ 6106 if (cpu_is_offline(cpu)) { 6107 /* 6108 * Reset core_pick so that we don't enter the fastpath when 6109 * coming online. core_pick would already be migrated to 6110 * another cpu during offline. 6111 */ 6112 rq->core_pick = NULL; 6113 return __pick_next_task(rq, prev, rf); 6114 } 6115 6116 /* 6117 * If there were no {en,de}queues since we picked (IOW, the task 6118 * pointers are all still valid), and we haven't scheduled the last 6119 * pick yet, do so now. 6120 * 6121 * rq->core_pick can be NULL if no selection was made for a CPU because 6122 * it was either offline or went offline during a sibling's core-wide 6123 * selection. In this case, do a core-wide selection. 6124 */ 6125 if (rq->core->core_pick_seq == rq->core->core_task_seq && 6126 rq->core->core_pick_seq != rq->core_sched_seq && 6127 rq->core_pick) { 6128 WRITE_ONCE(rq->core_sched_seq, rq->core->core_pick_seq); 6129 6130 next = rq->core_pick; 6131 if (next != prev) { 6132 put_prev_task(rq, prev); 6133 set_next_task(rq, next); 6134 } 6135 6136 rq->core_pick = NULL; 6137 goto out; 6138 } 6139 6140 put_prev_task_balance(rq, prev, rf); 6141 6142 smt_mask = cpu_smt_mask(cpu); 6143 need_sync = !!rq->core->core_cookie; 6144 6145 /* reset state */ 6146 rq->core->core_cookie = 0UL; 6147 if (rq->core->core_forceidle_count) { 6148 if (!core_clock_updated) { 6149 update_rq_clock(rq->core); 6150 core_clock_updated = true; 6151 } 6152 sched_core_account_forceidle(rq); 6153 /* reset after accounting force idle */ 6154 rq->core->core_forceidle_start = 0; 6155 rq->core->core_forceidle_count = 0; 6156 rq->core->core_forceidle_occupation = 0; 6157 need_sync = true; 6158 fi_before = true; 6159 } 6160 6161 /* 6162 * core->core_task_seq, core->core_pick_seq, rq->core_sched_seq 6163 * 6164 * @task_seq guards the task state ({en,de}queues) 6165 * @pick_seq is the @task_seq we did a selection on 6166 * @sched_seq is the @pick_seq we scheduled 6167 * 6168 * However, preemptions can cause multiple picks on the same task set. 6169 * 'Fix' this by also increasing @task_seq for every pick. 6170 */ 6171 rq->core->core_task_seq++; 6172 6173 /* 6174 * Optimize for common case where this CPU has no cookies 6175 * and there are no cookied tasks running on siblings. 6176 */ 6177 if (!need_sync) { 6178 next = pick_task(rq); 6179 if (!next->core_cookie) { 6180 rq->core_pick = NULL; 6181 /* 6182 * For robustness, update the min_vruntime_fi for 6183 * unconstrained picks as well. 6184 */ 6185 WARN_ON_ONCE(fi_before); 6186 task_vruntime_update(rq, next, false); 6187 goto out_set_next; 6188 } 6189 } 6190 6191 /* 6192 * For each thread: do the regular task pick and find the max prio task 6193 * amongst them. 6194 * 6195 * Tie-break prio towards the current CPU 6196 */ 6197 for_each_cpu_wrap(i, smt_mask, cpu) { 6198 rq_i = cpu_rq(i); 6199 6200 /* 6201 * Current cpu always has its clock updated on entrance to 6202 * pick_next_task(). If the current cpu is not the core, 6203 * the core may also have been updated above. 6204 */ 6205 if (i != cpu && (rq_i != rq->core || !core_clock_updated)) 6206 update_rq_clock(rq_i); 6207 6208 p = rq_i->core_pick = pick_task(rq_i); 6209 if (!max || prio_less(max, p, fi_before)) 6210 max = p; 6211 } 6212 6213 cookie = rq->core->core_cookie = max->core_cookie; 6214 6215 /* 6216 * For each thread: try and find a runnable task that matches @max or 6217 * force idle. 6218 */ 6219 for_each_cpu(i, smt_mask) { 6220 rq_i = cpu_rq(i); 6221 p = rq_i->core_pick; 6222 6223 if (!cookie_equals(p, cookie)) { 6224 p = NULL; 6225 if (cookie) 6226 p = sched_core_find(rq_i, cookie); 6227 if (!p) 6228 p = idle_sched_class.pick_task(rq_i); 6229 } 6230 6231 rq_i->core_pick = p; 6232 6233 if (p == rq_i->idle) { 6234 if (rq_i->nr_running) { 6235 rq->core->core_forceidle_count++; 6236 if (!fi_before) 6237 rq->core->core_forceidle_seq++; 6238 } 6239 } else { 6240 occ++; 6241 } 6242 } 6243 6244 if (schedstat_enabled() && rq->core->core_forceidle_count) { 6245 rq->core->core_forceidle_start = rq_clock(rq->core); 6246 rq->core->core_forceidle_occupation = occ; 6247 } 6248 6249 rq->core->core_pick_seq = rq->core->core_task_seq; 6250 next = rq->core_pick; 6251 rq->core_sched_seq = rq->core->core_pick_seq; 6252 6253 /* Something should have been selected for current CPU */ 6254 WARN_ON_ONCE(!next); 6255 6256 /* 6257 * Reschedule siblings 6258 * 6259 * NOTE: L1TF -- at this point we're no longer running the old task and 6260 * sending an IPI (below) ensures the sibling will no longer be running 6261 * their task. This ensures there is no inter-sibling overlap between 6262 * non-matching user state. 6263 */ 6264 for_each_cpu(i, smt_mask) { 6265 rq_i = cpu_rq(i); 6266 6267 /* 6268 * An online sibling might have gone offline before a task 6269 * could be picked for it, or it might be offline but later 6270 * happen to come online, but its too late and nothing was 6271 * picked for it. That's Ok - it will pick tasks for itself, 6272 * so ignore it. 6273 */ 6274 if (!rq_i->core_pick) 6275 continue; 6276 6277 /* 6278 * Update for new !FI->FI transitions, or if continuing to be in !FI: 6279 * fi_before fi update? 6280 * 0 0 1 6281 * 0 1 1 6282 * 1 0 1 6283 * 1 1 0 6284 */ 6285 if (!(fi_before && rq->core->core_forceidle_count)) 6286 task_vruntime_update(rq_i, rq_i->core_pick, !!rq->core->core_forceidle_count); 6287 6288 rq_i->core_pick->core_occupation = occ; 6289 6290 if (i == cpu) { 6291 rq_i->core_pick = NULL; 6292 continue; 6293 } 6294 6295 /* Did we break L1TF mitigation requirements? */ 6296 WARN_ON_ONCE(!cookie_match(next, rq_i->core_pick)); 6297 6298 if (rq_i->curr == rq_i->core_pick) { 6299 rq_i->core_pick = NULL; 6300 continue; 6301 } 6302 6303 resched_curr(rq_i); 6304 } 6305 6306 out_set_next: 6307 set_next_task(rq, next); 6308 out: 6309 if (rq->core->core_forceidle_count && next == rq->idle) 6310 queue_core_balance(rq); 6311 6312 return next; 6313 } 6314 6315 static bool try_steal_cookie(int this, int that) 6316 { 6317 struct rq *dst = cpu_rq(this), *src = cpu_rq(that); 6318 struct task_struct *p; 6319 unsigned long cookie; 6320 bool success = false; 6321 6322 local_irq_disable(); 6323 double_rq_lock(dst, src); 6324 6325 cookie = dst->core->core_cookie; 6326 if (!cookie) 6327 goto unlock; 6328 6329 if (dst->curr != dst->idle) 6330 goto unlock; 6331 6332 p = sched_core_find(src, cookie); 6333 if (!p) 6334 goto unlock; 6335 6336 do { 6337 if (p == src->core_pick || p == src->curr) 6338 goto next; 6339 6340 if (!is_cpu_allowed(p, this)) 6341 goto next; 6342 6343 if (p->core_occupation > dst->idle->core_occupation) 6344 goto next; 6345 /* 6346 * sched_core_find() and sched_core_next() will ensure that task @p 6347 * is not throttled now, we also need to check whether the runqueue 6348 * of the destination CPU is being throttled. 6349 */ 6350 if (sched_task_is_throttled(p, this)) 6351 goto next; 6352 6353 deactivate_task(src, p, 0); 6354 set_task_cpu(p, this); 6355 activate_task(dst, p, 0); 6356 6357 resched_curr(dst); 6358 6359 success = true; 6360 break; 6361 6362 next: 6363 p = sched_core_next(p, cookie); 6364 } while (p); 6365 6366 unlock: 6367 double_rq_unlock(dst, src); 6368 local_irq_enable(); 6369 6370 return success; 6371 } 6372 6373 static bool steal_cookie_task(int cpu, struct sched_domain *sd) 6374 { 6375 int i; 6376 6377 for_each_cpu_wrap(i, sched_domain_span(sd), cpu + 1) { 6378 if (i == cpu) 6379 continue; 6380 6381 if (need_resched()) 6382 break; 6383 6384 if (try_steal_cookie(cpu, i)) 6385 return true; 6386 } 6387 6388 return false; 6389 } 6390 6391 static void sched_core_balance(struct rq *rq) 6392 { 6393 struct sched_domain *sd; 6394 int cpu = cpu_of(rq); 6395 6396 preempt_disable(); 6397 rcu_read_lock(); 6398 raw_spin_rq_unlock_irq(rq); 6399 for_each_domain(cpu, sd) { 6400 if (need_resched()) 6401 break; 6402 6403 if (steal_cookie_task(cpu, sd)) 6404 break; 6405 } 6406 raw_spin_rq_lock_irq(rq); 6407 rcu_read_unlock(); 6408 preempt_enable(); 6409 } 6410 6411 static DEFINE_PER_CPU(struct balance_callback, core_balance_head); 6412 6413 static void queue_core_balance(struct rq *rq) 6414 { 6415 if (!sched_core_enabled(rq)) 6416 return; 6417 6418 if (!rq->core->core_cookie) 6419 return; 6420 6421 if (!rq->nr_running) /* not forced idle */ 6422 return; 6423 6424 queue_balance_callback(rq, &per_cpu(core_balance_head, rq->cpu), sched_core_balance); 6425 } 6426 6427 static void sched_core_cpu_starting(unsigned int cpu) 6428 { 6429 const struct cpumask *smt_mask = cpu_smt_mask(cpu); 6430 struct rq *rq = cpu_rq(cpu), *core_rq = NULL; 6431 unsigned long flags; 6432 int t; 6433 6434 sched_core_lock(cpu, &flags); 6435 6436 WARN_ON_ONCE(rq->core != rq); 6437 6438 /* if we're the first, we'll be our own leader */ 6439 if (cpumask_weight(smt_mask) == 1) 6440 goto unlock; 6441 6442 /* find the leader */ 6443 for_each_cpu(t, smt_mask) { 6444 if (t == cpu) 6445 continue; 6446 rq = cpu_rq(t); 6447 if (rq->core == rq) { 6448 core_rq = rq; 6449 break; 6450 } 6451 } 6452 6453 if (WARN_ON_ONCE(!core_rq)) /* whoopsie */ 6454 goto unlock; 6455 6456 /* install and validate core_rq */ 6457 for_each_cpu(t, smt_mask) { 6458 rq = cpu_rq(t); 6459 6460 if (t == cpu) 6461 rq->core = core_rq; 6462 6463 WARN_ON_ONCE(rq->core != core_rq); 6464 } 6465 6466 unlock: 6467 sched_core_unlock(cpu, &flags); 6468 } 6469 6470 static void sched_core_cpu_deactivate(unsigned int cpu) 6471 { 6472 const struct cpumask *smt_mask = cpu_smt_mask(cpu); 6473 struct rq *rq = cpu_rq(cpu), *core_rq = NULL; 6474 unsigned long flags; 6475 int t; 6476 6477 sched_core_lock(cpu, &flags); 6478 6479 /* if we're the last man standing, nothing to do */ 6480 if (cpumask_weight(smt_mask) == 1) { 6481 WARN_ON_ONCE(rq->core != rq); 6482 goto unlock; 6483 } 6484 6485 /* if we're not the leader, nothing to do */ 6486 if (rq->core != rq) 6487 goto unlock; 6488 6489 /* find a new leader */ 6490 for_each_cpu(t, smt_mask) { 6491 if (t == cpu) 6492 continue; 6493 core_rq = cpu_rq(t); 6494 break; 6495 } 6496 6497 if (WARN_ON_ONCE(!core_rq)) /* impossible */ 6498 goto unlock; 6499 6500 /* copy the shared state to the new leader */ 6501 core_rq->core_task_seq = rq->core_task_seq; 6502 core_rq->core_pick_seq = rq->core_pick_seq; 6503 core_rq->core_cookie = rq->core_cookie; 6504 core_rq->core_forceidle_count = rq->core_forceidle_count; 6505 core_rq->core_forceidle_seq = rq->core_forceidle_seq; 6506 core_rq->core_forceidle_occupation = rq->core_forceidle_occupation; 6507 6508 /* 6509 * Accounting edge for forced idle is handled in pick_next_task(). 6510 * Don't need another one here, since the hotplug thread shouldn't 6511 * have a cookie. 6512 */ 6513 core_rq->core_forceidle_start = 0; 6514 6515 /* install new leader */ 6516 for_each_cpu(t, smt_mask) { 6517 rq = cpu_rq(t); 6518 rq->core = core_rq; 6519 } 6520 6521 unlock: 6522 sched_core_unlock(cpu, &flags); 6523 } 6524 6525 static inline void sched_core_cpu_dying(unsigned int cpu) 6526 { 6527 struct rq *rq = cpu_rq(cpu); 6528 6529 if (rq->core != rq) 6530 rq->core = rq; 6531 } 6532 6533 #else /* !CONFIG_SCHED_CORE */ 6534 6535 static inline void sched_core_cpu_starting(unsigned int cpu) {} 6536 static inline void sched_core_cpu_deactivate(unsigned int cpu) {} 6537 static inline void sched_core_cpu_dying(unsigned int cpu) {} 6538 6539 static struct task_struct * 6540 pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 6541 { 6542 return __pick_next_task(rq, prev, rf); 6543 } 6544 6545 #endif /* CONFIG_SCHED_CORE */ 6546 6547 /* 6548 * Constants for the sched_mode argument of __schedule(). 6549 * 6550 * The mode argument allows RT enabled kernels to differentiate a 6551 * preemption from blocking on an 'sleeping' spin/rwlock. Note that 6552 * SM_MASK_PREEMPT for !RT has all bits set, which allows the compiler to 6553 * optimize the AND operation out and just check for zero. 6554 */ 6555 #define SM_NONE 0x0 6556 #define SM_PREEMPT 0x1 6557 #define SM_RTLOCK_WAIT 0x2 6558 6559 #ifndef CONFIG_PREEMPT_RT 6560 # define SM_MASK_PREEMPT (~0U) 6561 #else 6562 # define SM_MASK_PREEMPT SM_PREEMPT 6563 #endif 6564 6565 /* 6566 * __schedule() is the main scheduler function. 6567 * 6568 * The main means of driving the scheduler and thus entering this function are: 6569 * 6570 * 1. Explicit blocking: mutex, semaphore, waitqueue, etc. 6571 * 6572 * 2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return 6573 * paths. For example, see arch/x86/entry_64.S. 6574 * 6575 * To drive preemption between tasks, the scheduler sets the flag in timer 6576 * interrupt handler scheduler_tick(). 6577 * 6578 * 3. Wakeups don't really cause entry into schedule(). They add a 6579 * task to the run-queue and that's it. 6580 * 6581 * Now, if the new task added to the run-queue preempts the current 6582 * task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets 6583 * called on the nearest possible occasion: 6584 * 6585 * - If the kernel is preemptible (CONFIG_PREEMPTION=y): 6586 * 6587 * - in syscall or exception context, at the next outmost 6588 * preempt_enable(). (this might be as soon as the wake_up()'s 6589 * spin_unlock()!) 6590 * 6591 * - in IRQ context, return from interrupt-handler to 6592 * preemptible context 6593 * 6594 * - If the kernel is not preemptible (CONFIG_PREEMPTION is not set) 6595 * then at the next: 6596 * 6597 * - cond_resched() call 6598 * - explicit schedule() call 6599 * - return from syscall or exception to user-space 6600 * - return from interrupt-handler to user-space 6601 * 6602 * WARNING: must be called with preemption disabled! 6603 */ 6604 static void __sched notrace __schedule(unsigned int sched_mode) 6605 { 6606 struct task_struct *prev, *next; 6607 unsigned long *switch_count; 6608 unsigned long prev_state; 6609 struct rq_flags rf; 6610 struct rq *rq; 6611 int cpu; 6612 6613 cpu = smp_processor_id(); 6614 rq = cpu_rq(cpu); 6615 prev = rq->curr; 6616 6617 schedule_debug(prev, !!sched_mode); 6618 6619 if (sched_feat(HRTICK) || sched_feat(HRTICK_DL)) 6620 hrtick_clear(rq); 6621 6622 local_irq_disable(); 6623 rcu_note_context_switch(!!sched_mode); 6624 6625 /* 6626 * Make sure that signal_pending_state()->signal_pending() below 6627 * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE) 6628 * done by the caller to avoid the race with signal_wake_up(): 6629 * 6630 * __set_current_state(@state) signal_wake_up() 6631 * schedule() set_tsk_thread_flag(p, TIF_SIGPENDING) 6632 * wake_up_state(p, state) 6633 * LOCK rq->lock LOCK p->pi_state 6634 * smp_mb__after_spinlock() smp_mb__after_spinlock() 6635 * if (signal_pending_state()) if (p->state & @state) 6636 * 6637 * Also, the membarrier system call requires a full memory barrier 6638 * after coming from user-space, before storing to rq->curr. 6639 */ 6640 rq_lock(rq, &rf); 6641 smp_mb__after_spinlock(); 6642 6643 /* Promote REQ to ACT */ 6644 rq->clock_update_flags <<= 1; 6645 update_rq_clock(rq); 6646 6647 switch_count = &prev->nivcsw; 6648 6649 /* 6650 * We must load prev->state once (task_struct::state is volatile), such 6651 * that we form a control dependency vs deactivate_task() below. 6652 */ 6653 prev_state = READ_ONCE(prev->__state); 6654 if (!(sched_mode & SM_MASK_PREEMPT) && prev_state) { 6655 if (signal_pending_state(prev_state, prev)) { 6656 WRITE_ONCE(prev->__state, TASK_RUNNING); 6657 } else { 6658 prev->sched_contributes_to_load = 6659 (prev_state & TASK_UNINTERRUPTIBLE) && 6660 !(prev_state & TASK_NOLOAD) && 6661 !(prev_state & TASK_FROZEN); 6662 6663 if (prev->sched_contributes_to_load) 6664 rq->nr_uninterruptible++; 6665 6666 /* 6667 * __schedule() ttwu() 6668 * prev_state = prev->state; if (p->on_rq && ...) 6669 * if (prev_state) goto out; 6670 * p->on_rq = 0; smp_acquire__after_ctrl_dep(); 6671 * p->state = TASK_WAKING 6672 * 6673 * Where __schedule() and ttwu() have matching control dependencies. 6674 * 6675 * After this, schedule() must not care about p->state any more. 6676 */ 6677 deactivate_task(rq, prev, DEQUEUE_SLEEP | DEQUEUE_NOCLOCK); 6678 6679 if (prev->in_iowait) { 6680 atomic_inc(&rq->nr_iowait); 6681 delayacct_blkio_start(); 6682 } 6683 } 6684 switch_count = &prev->nvcsw; 6685 } 6686 6687 next = pick_next_task(rq, prev, &rf); 6688 clear_tsk_need_resched(prev); 6689 clear_preempt_need_resched(); 6690 #ifdef CONFIG_SCHED_DEBUG 6691 rq->last_seen_need_resched_ns = 0; 6692 #endif 6693 6694 if (likely(prev != next)) { 6695 rq->nr_switches++; 6696 /* 6697 * RCU users of rcu_dereference(rq->curr) may not see 6698 * changes to task_struct made by pick_next_task(). 6699 */ 6700 RCU_INIT_POINTER(rq->curr, next); 6701 /* 6702 * The membarrier system call requires each architecture 6703 * to have a full memory barrier after updating 6704 * rq->curr, before returning to user-space. 6705 * 6706 * Here are the schemes providing that barrier on the 6707 * various architectures: 6708 * - mm ? switch_mm() : mmdrop() for x86, s390, sparc, PowerPC. 6709 * switch_mm() rely on membarrier_arch_switch_mm() on PowerPC. 6710 * - finish_lock_switch() for weakly-ordered 6711 * architectures where spin_unlock is a full barrier, 6712 * - switch_to() for arm64 (weakly-ordered, spin_unlock 6713 * is a RELEASE barrier), 6714 */ 6715 ++*switch_count; 6716 6717 migrate_disable_switch(rq, prev); 6718 psi_sched_switch(prev, next, !task_on_rq_queued(prev)); 6719 6720 trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); 6721 6722 /* Also unlocks the rq: */ 6723 rq = context_switch(rq, prev, next, &rf); 6724 } else { 6725 rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP); 6726 6727 rq_unpin_lock(rq, &rf); 6728 __balance_callbacks(rq); 6729 raw_spin_rq_unlock_irq(rq); 6730 } 6731 } 6732 6733 void __noreturn do_task_dead(void) 6734 { 6735 /* Causes final put_task_struct in finish_task_switch(): */ 6736 set_special_state(TASK_DEAD); 6737 6738 /* Tell freezer to ignore us: */ 6739 current->flags |= PF_NOFREEZE; 6740 6741 __schedule(SM_NONE); 6742 BUG(); 6743 6744 /* Avoid "noreturn function does return" - but don't continue if BUG() is a NOP: */ 6745 for (;;) 6746 cpu_relax(); 6747 } 6748 6749 static inline void sched_submit_work(struct task_struct *tsk) 6750 { 6751 unsigned int task_flags; 6752 6753 if (task_is_running(tsk)) 6754 return; 6755 6756 task_flags = tsk->flags; 6757 /* 6758 * If a worker goes to sleep, notify and ask workqueue whether it 6759 * wants to wake up a task to maintain concurrency. 6760 */ 6761 if (task_flags & (PF_WQ_WORKER | PF_IO_WORKER)) { 6762 if (task_flags & PF_WQ_WORKER) 6763 wq_worker_sleeping(tsk); 6764 else 6765 io_wq_worker_sleeping(tsk); 6766 } 6767 6768 /* 6769 * spinlock and rwlock must not flush block requests. This will 6770 * deadlock if the callback attempts to acquire a lock which is 6771 * already acquired. 6772 */ 6773 SCHED_WARN_ON(current->__state & TASK_RTLOCK_WAIT); 6774 6775 /* 6776 * If we are going to sleep and we have plugged IO queued, 6777 * make sure to submit it to avoid deadlocks. 6778 */ 6779 blk_flush_plug(tsk->plug, true); 6780 } 6781 6782 static void sched_update_worker(struct task_struct *tsk) 6783 { 6784 if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER)) { 6785 if (tsk->flags & PF_WQ_WORKER) 6786 wq_worker_running(tsk); 6787 else 6788 io_wq_worker_running(tsk); 6789 } 6790 } 6791 6792 asmlinkage __visible void __sched schedule(void) 6793 { 6794 struct task_struct *tsk = current; 6795 6796 sched_submit_work(tsk); 6797 do { 6798 preempt_disable(); 6799 __schedule(SM_NONE); 6800 sched_preempt_enable_no_resched(); 6801 } while (need_resched()); 6802 sched_update_worker(tsk); 6803 } 6804 EXPORT_SYMBOL(schedule); 6805 6806 /* 6807 * synchronize_rcu_tasks() makes sure that no task is stuck in preempted 6808 * state (have scheduled out non-voluntarily) by making sure that all 6809 * tasks have either left the run queue or have gone into user space. 6810 * As idle tasks do not do either, they must not ever be preempted 6811 * (schedule out non-voluntarily). 6812 * 6813 * schedule_idle() is similar to schedule_preempt_disable() except that it 6814 * never enables preemption because it does not call sched_submit_work(). 6815 */ 6816 void __sched schedule_idle(void) 6817 { 6818 /* 6819 * As this skips calling sched_submit_work(), which the idle task does 6820 * regardless because that function is a nop when the task is in a 6821 * TASK_RUNNING state, make sure this isn't used someplace that the 6822 * current task can be in any other state. Note, idle is always in the 6823 * TASK_RUNNING state. 6824 */ 6825 WARN_ON_ONCE(current->__state); 6826 do { 6827 __schedule(SM_NONE); 6828 } while (need_resched()); 6829 } 6830 6831 #if defined(CONFIG_CONTEXT_TRACKING_USER) && !defined(CONFIG_HAVE_CONTEXT_TRACKING_USER_OFFSTACK) 6832 asmlinkage __visible void __sched schedule_user(void) 6833 { 6834 /* 6835 * If we come here after a random call to set_need_resched(), 6836 * or we have been woken up remotely but the IPI has not yet arrived, 6837 * we haven't yet exited the RCU idle mode. Do it here manually until 6838 * we find a better solution. 6839 * 6840 * NB: There are buggy callers of this function. Ideally we 6841 * should warn if prev_state != CONTEXT_USER, but that will trigger 6842 * too frequently to make sense yet. 6843 */ 6844 enum ctx_state prev_state = exception_enter(); 6845 schedule(); 6846 exception_exit(prev_state); 6847 } 6848 #endif 6849 6850 /** 6851 * schedule_preempt_disabled - called with preemption disabled 6852 * 6853 * Returns with preemption disabled. Note: preempt_count must be 1 6854 */ 6855 void __sched schedule_preempt_disabled(void) 6856 { 6857 sched_preempt_enable_no_resched(); 6858 schedule(); 6859 preempt_disable(); 6860 } 6861 6862 #ifdef CONFIG_PREEMPT_RT 6863 void __sched notrace schedule_rtlock(void) 6864 { 6865 do { 6866 preempt_disable(); 6867 __schedule(SM_RTLOCK_WAIT); 6868 sched_preempt_enable_no_resched(); 6869 } while (need_resched()); 6870 } 6871 NOKPROBE_SYMBOL(schedule_rtlock); 6872 #endif 6873 6874 static void __sched notrace preempt_schedule_common(void) 6875 { 6876 do { 6877 /* 6878 * Because the function tracer can trace preempt_count_sub() 6879 * and it also uses preempt_enable/disable_notrace(), if 6880 * NEED_RESCHED is set, the preempt_enable_notrace() called 6881 * by the function tracer will call this function again and 6882 * cause infinite recursion. 6883 * 6884 * Preemption must be disabled here before the function 6885 * tracer can trace. Break up preempt_disable() into two 6886 * calls. One to disable preemption without fear of being 6887 * traced. The other to still record the preemption latency, 6888 * which can also be traced by the function tracer. 6889 */ 6890 preempt_disable_notrace(); 6891 preempt_latency_start(1); 6892 __schedule(SM_PREEMPT); 6893 preempt_latency_stop(1); 6894 preempt_enable_no_resched_notrace(); 6895 6896 /* 6897 * Check again in case we missed a preemption opportunity 6898 * between schedule and now. 6899 */ 6900 } while (need_resched()); 6901 } 6902 6903 #ifdef CONFIG_PREEMPTION 6904 /* 6905 * This is the entry point to schedule() from in-kernel preemption 6906 * off of preempt_enable. 6907 */ 6908 asmlinkage __visible void __sched notrace preempt_schedule(void) 6909 { 6910 /* 6911 * If there is a non-zero preempt_count or interrupts are disabled, 6912 * we do not want to preempt the current task. Just return.. 6913 */ 6914 if (likely(!preemptible())) 6915 return; 6916 preempt_schedule_common(); 6917 } 6918 NOKPROBE_SYMBOL(preempt_schedule); 6919 EXPORT_SYMBOL(preempt_schedule); 6920 6921 #ifdef CONFIG_PREEMPT_DYNAMIC 6922 #if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 6923 #ifndef preempt_schedule_dynamic_enabled 6924 #define preempt_schedule_dynamic_enabled preempt_schedule 6925 #define preempt_schedule_dynamic_disabled NULL 6926 #endif 6927 DEFINE_STATIC_CALL(preempt_schedule, preempt_schedule_dynamic_enabled); 6928 EXPORT_STATIC_CALL_TRAMP(preempt_schedule); 6929 #elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 6930 static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule); 6931 void __sched notrace dynamic_preempt_schedule(void) 6932 { 6933 if (!static_branch_unlikely(&sk_dynamic_preempt_schedule)) 6934 return; 6935 preempt_schedule(); 6936 } 6937 NOKPROBE_SYMBOL(dynamic_preempt_schedule); 6938 EXPORT_SYMBOL(dynamic_preempt_schedule); 6939 #endif 6940 #endif 6941 6942 /** 6943 * preempt_schedule_notrace - preempt_schedule called by tracing 6944 * 6945 * The tracing infrastructure uses preempt_enable_notrace to prevent 6946 * recursion and tracing preempt enabling caused by the tracing 6947 * infrastructure itself. But as tracing can happen in areas coming 6948 * from userspace or just about to enter userspace, a preempt enable 6949 * can occur before user_exit() is called. This will cause the scheduler 6950 * to be called when the system is still in usermode. 6951 * 6952 * To prevent this, the preempt_enable_notrace will use this function 6953 * instead of preempt_schedule() to exit user context if needed before 6954 * calling the scheduler. 6955 */ 6956 asmlinkage __visible void __sched notrace preempt_schedule_notrace(void) 6957 { 6958 enum ctx_state prev_ctx; 6959 6960 if (likely(!preemptible())) 6961 return; 6962 6963 do { 6964 /* 6965 * Because the function tracer can trace preempt_count_sub() 6966 * and it also uses preempt_enable/disable_notrace(), if 6967 * NEED_RESCHED is set, the preempt_enable_notrace() called 6968 * by the function tracer will call this function again and 6969 * cause infinite recursion. 6970 * 6971 * Preemption must be disabled here before the function 6972 * tracer can trace. Break up preempt_disable() into two 6973 * calls. One to disable preemption without fear of being 6974 * traced. The other to still record the preemption latency, 6975 * which can also be traced by the function tracer. 6976 */ 6977 preempt_disable_notrace(); 6978 preempt_latency_start(1); 6979 /* 6980 * Needs preempt disabled in case user_exit() is traced 6981 * and the tracer calls preempt_enable_notrace() causing 6982 * an infinite recursion. 6983 */ 6984 prev_ctx = exception_enter(); 6985 __schedule(SM_PREEMPT); 6986 exception_exit(prev_ctx); 6987 6988 preempt_latency_stop(1); 6989 preempt_enable_no_resched_notrace(); 6990 } while (need_resched()); 6991 } 6992 EXPORT_SYMBOL_GPL(preempt_schedule_notrace); 6993 6994 #ifdef CONFIG_PREEMPT_DYNAMIC 6995 #if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 6996 #ifndef preempt_schedule_notrace_dynamic_enabled 6997 #define preempt_schedule_notrace_dynamic_enabled preempt_schedule_notrace 6998 #define preempt_schedule_notrace_dynamic_disabled NULL 6999 #endif 7000 DEFINE_STATIC_CALL(preempt_schedule_notrace, preempt_schedule_notrace_dynamic_enabled); 7001 EXPORT_STATIC_CALL_TRAMP(preempt_schedule_notrace); 7002 #elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 7003 static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule_notrace); 7004 void __sched notrace dynamic_preempt_schedule_notrace(void) 7005 { 7006 if (!static_branch_unlikely(&sk_dynamic_preempt_schedule_notrace)) 7007 return; 7008 preempt_schedule_notrace(); 7009 } 7010 NOKPROBE_SYMBOL(dynamic_preempt_schedule_notrace); 7011 EXPORT_SYMBOL(dynamic_preempt_schedule_notrace); 7012 #endif 7013 #endif 7014 7015 #endif /* CONFIG_PREEMPTION */ 7016 7017 /* 7018 * This is the entry point to schedule() from kernel preemption 7019 * off of irq context. 7020 * Note, that this is called and return with irqs disabled. This will 7021 * protect us against recursive calling from irq. 7022 */ 7023 asmlinkage __visible void __sched preempt_schedule_irq(void) 7024 { 7025 enum ctx_state prev_state; 7026 7027 /* Catch callers which need to be fixed */ 7028 BUG_ON(preempt_count() || !irqs_disabled()); 7029 7030 prev_state = exception_enter(); 7031 7032 do { 7033 preempt_disable(); 7034 local_irq_enable(); 7035 __schedule(SM_PREEMPT); 7036 local_irq_disable(); 7037 sched_preempt_enable_no_resched(); 7038 } while (need_resched()); 7039 7040 exception_exit(prev_state); 7041 } 7042 7043 int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flags, 7044 void *key) 7045 { 7046 WARN_ON_ONCE(IS_ENABLED(CONFIG_SCHED_DEBUG) && wake_flags & ~WF_SYNC); 7047 return try_to_wake_up(curr->private, mode, wake_flags); 7048 } 7049 EXPORT_SYMBOL(default_wake_function); 7050 7051 static void __setscheduler_prio(struct task_struct *p, int prio) 7052 { 7053 if (dl_prio(prio)) 7054 p->sched_class = &dl_sched_class; 7055 else if (rt_prio(prio)) 7056 p->sched_class = &rt_sched_class; 7057 else 7058 p->sched_class = &fair_sched_class; 7059 7060 p->prio = prio; 7061 } 7062 7063 #ifdef CONFIG_RT_MUTEXES 7064 7065 static inline int __rt_effective_prio(struct task_struct *pi_task, int prio) 7066 { 7067 if (pi_task) 7068 prio = min(prio, pi_task->prio); 7069 7070 return prio; 7071 } 7072 7073 static inline int rt_effective_prio(struct task_struct *p, int prio) 7074 { 7075 struct task_struct *pi_task = rt_mutex_get_top_task(p); 7076 7077 return __rt_effective_prio(pi_task, prio); 7078 } 7079 7080 /* 7081 * rt_mutex_setprio - set the current priority of a task 7082 * @p: task to boost 7083 * @pi_task: donor task 7084 * 7085 * This function changes the 'effective' priority of a task. It does 7086 * not touch ->normal_prio like __setscheduler(). 7087 * 7088 * Used by the rt_mutex code to implement priority inheritance 7089 * logic. Call site only calls if the priority of the task changed. 7090 */ 7091 void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) 7092 { 7093 int prio, oldprio, queued, running, queue_flag = 7094 DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; 7095 const struct sched_class *prev_class; 7096 struct rq_flags rf; 7097 struct rq *rq; 7098 7099 /* XXX used to be waiter->prio, not waiter->task->prio */ 7100 prio = __rt_effective_prio(pi_task, p->normal_prio); 7101 7102 /* 7103 * If nothing changed; bail early. 7104 */ 7105 if (p->pi_top_task == pi_task && prio == p->prio && !dl_prio(prio)) 7106 return; 7107 7108 rq = __task_rq_lock(p, &rf); 7109 update_rq_clock(rq); 7110 /* 7111 * Set under pi_lock && rq->lock, such that the value can be used under 7112 * either lock. 7113 * 7114 * Note that there is loads of tricky to make this pointer cache work 7115 * right. rt_mutex_slowunlock()+rt_mutex_postunlock() work together to 7116 * ensure a task is de-boosted (pi_task is set to NULL) before the 7117 * task is allowed to run again (and can exit). This ensures the pointer 7118 * points to a blocked task -- which guarantees the task is present. 7119 */ 7120 p->pi_top_task = pi_task; 7121 7122 /* 7123 * For FIFO/RR we only need to set prio, if that matches we're done. 7124 */ 7125 if (prio == p->prio && !dl_prio(prio)) 7126 goto out_unlock; 7127 7128 /* 7129 * Idle task boosting is a nono in general. There is one 7130 * exception, when PREEMPT_RT and NOHZ is active: 7131 * 7132 * The idle task calls get_next_timer_interrupt() and holds 7133 * the timer wheel base->lock on the CPU and another CPU wants 7134 * to access the timer (probably to cancel it). We can safely 7135 * ignore the boosting request, as the idle CPU runs this code 7136 * with interrupts disabled and will complete the lock 7137 * protected section without being interrupted. So there is no 7138 * real need to boost. 7139 */ 7140 if (unlikely(p == rq->idle)) { 7141 WARN_ON(p != rq->curr); 7142 WARN_ON(p->pi_blocked_on); 7143 goto out_unlock; 7144 } 7145 7146 trace_sched_pi_setprio(p, pi_task); 7147 oldprio = p->prio; 7148 7149 if (oldprio == prio) 7150 queue_flag &= ~DEQUEUE_MOVE; 7151 7152 prev_class = p->sched_class; 7153 queued = task_on_rq_queued(p); 7154 running = task_current(rq, p); 7155 if (queued) 7156 dequeue_task(rq, p, queue_flag); 7157 if (running) 7158 put_prev_task(rq, p); 7159 7160 /* 7161 * Boosting condition are: 7162 * 1. -rt task is running and holds mutex A 7163 * --> -dl task blocks on mutex A 7164 * 7165 * 2. -dl task is running and holds mutex A 7166 * --> -dl task blocks on mutex A and could preempt the 7167 * running task 7168 */ 7169 if (dl_prio(prio)) { 7170 if (!dl_prio(p->normal_prio) || 7171 (pi_task && dl_prio(pi_task->prio) && 7172 dl_entity_preempt(&pi_task->dl, &p->dl))) { 7173 p->dl.pi_se = pi_task->dl.pi_se; 7174 queue_flag |= ENQUEUE_REPLENISH; 7175 } else { 7176 p->dl.pi_se = &p->dl; 7177 } 7178 } else if (rt_prio(prio)) { 7179 if (dl_prio(oldprio)) 7180 p->dl.pi_se = &p->dl; 7181 if (oldprio < prio) 7182 queue_flag |= ENQUEUE_HEAD; 7183 } else { 7184 if (dl_prio(oldprio)) 7185 p->dl.pi_se = &p->dl; 7186 if (rt_prio(oldprio)) 7187 p->rt.timeout = 0; 7188 } 7189 7190 __setscheduler_prio(p, prio); 7191 7192 if (queued) 7193 enqueue_task(rq, p, queue_flag); 7194 if (running) 7195 set_next_task(rq, p); 7196 7197 check_class_changed(rq, p, prev_class, oldprio); 7198 out_unlock: 7199 /* Avoid rq from going away on us: */ 7200 preempt_disable(); 7201 7202 rq_unpin_lock(rq, &rf); 7203 __balance_callbacks(rq); 7204 raw_spin_rq_unlock(rq); 7205 7206 preempt_enable(); 7207 } 7208 #else 7209 static inline int rt_effective_prio(struct task_struct *p, int prio) 7210 { 7211 return prio; 7212 } 7213 #endif 7214 7215 void set_user_nice(struct task_struct *p, long nice) 7216 { 7217 bool queued, running; 7218 int old_prio; 7219 struct rq_flags rf; 7220 struct rq *rq; 7221 7222 if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE) 7223 return; 7224 /* 7225 * We have to be careful, if called from sys_setpriority(), 7226 * the task might be in the middle of scheduling on another CPU. 7227 */ 7228 rq = task_rq_lock(p, &rf); 7229 update_rq_clock(rq); 7230 7231 /* 7232 * The RT priorities are set via sched_setscheduler(), but we still 7233 * allow the 'normal' nice value to be set - but as expected 7234 * it won't have any effect on scheduling until the task is 7235 * SCHED_DEADLINE, SCHED_FIFO or SCHED_RR: 7236 */ 7237 if (task_has_dl_policy(p) || task_has_rt_policy(p)) { 7238 p->static_prio = NICE_TO_PRIO(nice); 7239 goto out_unlock; 7240 } 7241 queued = task_on_rq_queued(p); 7242 running = task_current(rq, p); 7243 if (queued) 7244 dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); 7245 if (running) 7246 put_prev_task(rq, p); 7247 7248 p->static_prio = NICE_TO_PRIO(nice); 7249 set_load_weight(p, true); 7250 old_prio = p->prio; 7251 p->prio = effective_prio(p); 7252 7253 if (queued) 7254 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); 7255 if (running) 7256 set_next_task(rq, p); 7257 7258 /* 7259 * If the task increased its priority or is running and 7260 * lowered its priority, then reschedule its CPU: 7261 */ 7262 p->sched_class->prio_changed(rq, p, old_prio); 7263 7264 out_unlock: 7265 task_rq_unlock(rq, p, &rf); 7266 } 7267 EXPORT_SYMBOL(set_user_nice); 7268 7269 /* 7270 * is_nice_reduction - check if nice value is an actual reduction 7271 * 7272 * Similar to can_nice() but does not perform a capability check. 7273 * 7274 * @p: task 7275 * @nice: nice value 7276 */ 7277 static bool is_nice_reduction(const struct task_struct *p, const int nice) 7278 { 7279 /* Convert nice value [19,-20] to rlimit style value [1,40]: */ 7280 int nice_rlim = nice_to_rlimit(nice); 7281 7282 return (nice_rlim <= task_rlimit(p, RLIMIT_NICE)); 7283 } 7284 7285 /* 7286 * can_nice - check if a task can reduce its nice value 7287 * @p: task 7288 * @nice: nice value 7289 */ 7290 int can_nice(const struct task_struct *p, const int nice) 7291 { 7292 return is_nice_reduction(p, nice) || capable(CAP_SYS_NICE); 7293 } 7294 7295 #ifdef __ARCH_WANT_SYS_NICE 7296 7297 /* 7298 * sys_nice - change the priority of the current process. 7299 * @increment: priority increment 7300 * 7301 * sys_setpriority is a more generic, but much slower function that 7302 * does similar things. 7303 */ 7304 SYSCALL_DEFINE1(nice, int, increment) 7305 { 7306 long nice, retval; 7307 7308 /* 7309 * Setpriority might change our priority at the same moment. 7310 * We don't have to worry. Conceptually one call occurs first 7311 * and we have a single winner. 7312 */ 7313 increment = clamp(increment, -NICE_WIDTH, NICE_WIDTH); 7314 nice = task_nice(current) + increment; 7315 7316 nice = clamp_val(nice, MIN_NICE, MAX_NICE); 7317 if (increment < 0 && !can_nice(current, nice)) 7318 return -EPERM; 7319 7320 retval = security_task_setnice(current, nice); 7321 if (retval) 7322 return retval; 7323 7324 set_user_nice(current, nice); 7325 return 0; 7326 } 7327 7328 #endif 7329 7330 /** 7331 * task_prio - return the priority value of a given task. 7332 * @p: the task in question. 7333 * 7334 * Return: The priority value as seen by users in /proc. 7335 * 7336 * sched policy return value kernel prio user prio/nice 7337 * 7338 * normal, batch, idle [0 ... 39] [100 ... 139] 0/[-20 ... 19] 7339 * fifo, rr [-2 ... -100] [98 ... 0] [1 ... 99] 7340 * deadline -101 -1 0 7341 */ 7342 int task_prio(const struct task_struct *p) 7343 { 7344 return p->prio - MAX_RT_PRIO; 7345 } 7346 7347 /** 7348 * idle_cpu - is a given CPU idle currently? 7349 * @cpu: the processor in question. 7350 * 7351 * Return: 1 if the CPU is currently idle. 0 otherwise. 7352 */ 7353 int idle_cpu(int cpu) 7354 { 7355 struct rq *rq = cpu_rq(cpu); 7356 7357 if (rq->curr != rq->idle) 7358 return 0; 7359 7360 if (rq->nr_running) 7361 return 0; 7362 7363 #ifdef CONFIG_SMP 7364 if (rq->ttwu_pending) 7365 return 0; 7366 #endif 7367 7368 return 1; 7369 } 7370 7371 /** 7372 * available_idle_cpu - is a given CPU idle for enqueuing work. 7373 * @cpu: the CPU in question. 7374 * 7375 * Return: 1 if the CPU is currently idle. 0 otherwise. 7376 */ 7377 int available_idle_cpu(int cpu) 7378 { 7379 if (!idle_cpu(cpu)) 7380 return 0; 7381 7382 if (vcpu_is_preempted(cpu)) 7383 return 0; 7384 7385 return 1; 7386 } 7387 7388 /** 7389 * idle_task - return the idle task for a given CPU. 7390 * @cpu: the processor in question. 7391 * 7392 * Return: The idle task for the CPU @cpu. 7393 */ 7394 struct task_struct *idle_task(int cpu) 7395 { 7396 return cpu_rq(cpu)->idle; 7397 } 7398 7399 #ifdef CONFIG_SCHED_CORE 7400 int sched_core_idle_cpu(int cpu) 7401 { 7402 struct rq *rq = cpu_rq(cpu); 7403 7404 if (sched_core_enabled(rq) && rq->curr == rq->idle) 7405 return 1; 7406 7407 return idle_cpu(cpu); 7408 } 7409 7410 #endif 7411 7412 #ifdef CONFIG_SMP 7413 /* 7414 * This function computes an effective utilization for the given CPU, to be 7415 * used for frequency selection given the linear relation: f = u * f_max. 7416 * 7417 * The scheduler tracks the following metrics: 7418 * 7419 * cpu_util_{cfs,rt,dl,irq}() 7420 * cpu_bw_dl() 7421 * 7422 * Where the cfs,rt and dl util numbers are tracked with the same metric and 7423 * synchronized windows and are thus directly comparable. 7424 * 7425 * The cfs,rt,dl utilization are the running times measured with rq->clock_task 7426 * which excludes things like IRQ and steal-time. These latter are then accrued 7427 * in the irq utilization. 7428 * 7429 * The DL bandwidth number otoh is not a measured metric but a value computed 7430 * based on the task model parameters and gives the minimal utilization 7431 * required to meet deadlines. 7432 */ 7433 unsigned long effective_cpu_util(int cpu, unsigned long util_cfs, 7434 enum cpu_util_type type, 7435 struct task_struct *p) 7436 { 7437 unsigned long dl_util, util, irq, max; 7438 struct rq *rq = cpu_rq(cpu); 7439 7440 max = arch_scale_cpu_capacity(cpu); 7441 7442 if (!uclamp_is_used() && 7443 type == FREQUENCY_UTIL && rt_rq_is_runnable(&rq->rt)) { 7444 return max; 7445 } 7446 7447 /* 7448 * Early check to see if IRQ/steal time saturates the CPU, can be 7449 * because of inaccuracies in how we track these -- see 7450 * update_irq_load_avg(). 7451 */ 7452 irq = cpu_util_irq(rq); 7453 if (unlikely(irq >= max)) 7454 return max; 7455 7456 /* 7457 * Because the time spend on RT/DL tasks is visible as 'lost' time to 7458 * CFS tasks and we use the same metric to track the effective 7459 * utilization (PELT windows are synchronized) we can directly add them 7460 * to obtain the CPU's actual utilization. 7461 * 7462 * CFS and RT utilization can be boosted or capped, depending on 7463 * utilization clamp constraints requested by currently RUNNABLE 7464 * tasks. 7465 * When there are no CFS RUNNABLE tasks, clamps are released and 7466 * frequency will be gracefully reduced with the utilization decay. 7467 */ 7468 util = util_cfs + cpu_util_rt(rq); 7469 if (type == FREQUENCY_UTIL) 7470 util = uclamp_rq_util_with(rq, util, p); 7471 7472 dl_util = cpu_util_dl(rq); 7473 7474 /* 7475 * For frequency selection we do not make cpu_util_dl() a permanent part 7476 * of this sum because we want to use cpu_bw_dl() later on, but we need 7477 * to check if the CFS+RT+DL sum is saturated (ie. no idle time) such 7478 * that we select f_max when there is no idle time. 7479 * 7480 * NOTE: numerical errors or stop class might cause us to not quite hit 7481 * saturation when we should -- something for later. 7482 */ 7483 if (util + dl_util >= max) 7484 return max; 7485 7486 /* 7487 * OTOH, for energy computation we need the estimated running time, so 7488 * include util_dl and ignore dl_bw. 7489 */ 7490 if (type == ENERGY_UTIL) 7491 util += dl_util; 7492 7493 /* 7494 * There is still idle time; further improve the number by using the 7495 * irq metric. Because IRQ/steal time is hidden from the task clock we 7496 * need to scale the task numbers: 7497 * 7498 * max - irq 7499 * U' = irq + --------- * U 7500 * max 7501 */ 7502 util = scale_irq_capacity(util, irq, max); 7503 util += irq; 7504 7505 /* 7506 * Bandwidth required by DEADLINE must always be granted while, for 7507 * FAIR and RT, we use blocked utilization of IDLE CPUs as a mechanism 7508 * to gracefully reduce the frequency when no tasks show up for longer 7509 * periods of time. 7510 * 7511 * Ideally we would like to set bw_dl as min/guaranteed freq and util + 7512 * bw_dl as requested freq. However, cpufreq is not yet ready for such 7513 * an interface. So, we only do the latter for now. 7514 */ 7515 if (type == FREQUENCY_UTIL) 7516 util += cpu_bw_dl(rq); 7517 7518 return min(max, util); 7519 } 7520 7521 unsigned long sched_cpu_util(int cpu) 7522 { 7523 return effective_cpu_util(cpu, cpu_util_cfs(cpu), ENERGY_UTIL, NULL); 7524 } 7525 #endif /* CONFIG_SMP */ 7526 7527 /** 7528 * find_process_by_pid - find a process with a matching PID value. 7529 * @pid: the pid in question. 7530 * 7531 * The task of @pid, if found. %NULL otherwise. 7532 */ 7533 static struct task_struct *find_process_by_pid(pid_t pid) 7534 { 7535 return pid ? find_task_by_vpid(pid) : current; 7536 } 7537 7538 /* 7539 * sched_setparam() passes in -1 for its policy, to let the functions 7540 * it calls know not to change it. 7541 */ 7542 #define SETPARAM_POLICY -1 7543 7544 static void __setscheduler_params(struct task_struct *p, 7545 const struct sched_attr *attr) 7546 { 7547 int policy = attr->sched_policy; 7548 7549 if (policy == SETPARAM_POLICY) 7550 policy = p->policy; 7551 7552 p->policy = policy; 7553 7554 if (dl_policy(policy)) 7555 __setparam_dl(p, attr); 7556 else if (fair_policy(policy)) 7557 p->static_prio = NICE_TO_PRIO(attr->sched_nice); 7558 7559 /* 7560 * __sched_setscheduler() ensures attr->sched_priority == 0 when 7561 * !rt_policy. Always setting this ensures that things like 7562 * getparam()/getattr() don't report silly values for !rt tasks. 7563 */ 7564 p->rt_priority = attr->sched_priority; 7565 p->normal_prio = normal_prio(p); 7566 set_load_weight(p, true); 7567 } 7568 7569 /* 7570 * Check the target process has a UID that matches the current process's: 7571 */ 7572 static bool check_same_owner(struct task_struct *p) 7573 { 7574 const struct cred *cred = current_cred(), *pcred; 7575 bool match; 7576 7577 rcu_read_lock(); 7578 pcred = __task_cred(p); 7579 match = (uid_eq(cred->euid, pcred->euid) || 7580 uid_eq(cred->euid, pcred->uid)); 7581 rcu_read_unlock(); 7582 return match; 7583 } 7584 7585 /* 7586 * Allow unprivileged RT tasks to decrease priority. 7587 * Only issue a capable test if needed and only once to avoid an audit 7588 * event on permitted non-privileged operations: 7589 */ 7590 static int user_check_sched_setscheduler(struct task_struct *p, 7591 const struct sched_attr *attr, 7592 int policy, int reset_on_fork) 7593 { 7594 if (fair_policy(policy)) { 7595 if (attr->sched_nice < task_nice(p) && 7596 !is_nice_reduction(p, attr->sched_nice)) 7597 goto req_priv; 7598 } 7599 7600 if (rt_policy(policy)) { 7601 unsigned long rlim_rtprio = task_rlimit(p, RLIMIT_RTPRIO); 7602 7603 /* Can't set/change the rt policy: */ 7604 if (policy != p->policy && !rlim_rtprio) 7605 goto req_priv; 7606 7607 /* Can't increase priority: */ 7608 if (attr->sched_priority > p->rt_priority && 7609 attr->sched_priority > rlim_rtprio) 7610 goto req_priv; 7611 } 7612 7613 /* 7614 * Can't set/change SCHED_DEADLINE policy at all for now 7615 * (safest behavior); in the future we would like to allow 7616 * unprivileged DL tasks to increase their relative deadline 7617 * or reduce their runtime (both ways reducing utilization) 7618 */ 7619 if (dl_policy(policy)) 7620 goto req_priv; 7621 7622 /* 7623 * Treat SCHED_IDLE as nice 20. Only allow a switch to 7624 * SCHED_NORMAL if the RLIMIT_NICE would normally permit it. 7625 */ 7626 if (task_has_idle_policy(p) && !idle_policy(policy)) { 7627 if (!is_nice_reduction(p, task_nice(p))) 7628 goto req_priv; 7629 } 7630 7631 /* Can't change other user's priorities: */ 7632 if (!check_same_owner(p)) 7633 goto req_priv; 7634 7635 /* Normal users shall not reset the sched_reset_on_fork flag: */ 7636 if (p->sched_reset_on_fork && !reset_on_fork) 7637 goto req_priv; 7638 7639 return 0; 7640 7641 req_priv: 7642 if (!capable(CAP_SYS_NICE)) 7643 return -EPERM; 7644 7645 return 0; 7646 } 7647 7648 static int __sched_setscheduler(struct task_struct *p, 7649 const struct sched_attr *attr, 7650 bool user, bool pi) 7651 { 7652 int oldpolicy = -1, policy = attr->sched_policy; 7653 int retval, oldprio, newprio, queued, running; 7654 const struct sched_class *prev_class; 7655 struct balance_callback *head; 7656 struct rq_flags rf; 7657 int reset_on_fork; 7658 int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; 7659 struct rq *rq; 7660 bool cpuset_locked = false; 7661 7662 /* The pi code expects interrupts enabled */ 7663 BUG_ON(pi && in_interrupt()); 7664 recheck: 7665 /* Double check policy once rq lock held: */ 7666 if (policy < 0) { 7667 reset_on_fork = p->sched_reset_on_fork; 7668 policy = oldpolicy = p->policy; 7669 } else { 7670 reset_on_fork = !!(attr->sched_flags & SCHED_FLAG_RESET_ON_FORK); 7671 7672 if (!valid_policy(policy)) 7673 return -EINVAL; 7674 } 7675 7676 if (attr->sched_flags & ~(SCHED_FLAG_ALL | SCHED_FLAG_SUGOV)) 7677 return -EINVAL; 7678 7679 /* 7680 * Valid priorities for SCHED_FIFO and SCHED_RR are 7681 * 1..MAX_RT_PRIO-1, valid priority for SCHED_NORMAL, 7682 * SCHED_BATCH and SCHED_IDLE is 0. 7683 */ 7684 if (attr->sched_priority > MAX_RT_PRIO-1) 7685 return -EINVAL; 7686 if ((dl_policy(policy) && !__checkparam_dl(attr)) || 7687 (rt_policy(policy) != (attr->sched_priority != 0))) 7688 return -EINVAL; 7689 7690 if (user) { 7691 retval = user_check_sched_setscheduler(p, attr, policy, reset_on_fork); 7692 if (retval) 7693 return retval; 7694 7695 if (attr->sched_flags & SCHED_FLAG_SUGOV) 7696 return -EINVAL; 7697 7698 retval = security_task_setscheduler(p); 7699 if (retval) 7700 return retval; 7701 } 7702 7703 /* Update task specific "requested" clamps */ 7704 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP) { 7705 retval = uclamp_validate(p, attr); 7706 if (retval) 7707 return retval; 7708 } 7709 7710 /* 7711 * SCHED_DEADLINE bandwidth accounting relies on stable cpusets 7712 * information. 7713 */ 7714 if (dl_policy(policy) || dl_policy(p->policy)) { 7715 cpuset_locked = true; 7716 cpuset_lock(); 7717 } 7718 7719 /* 7720 * Make sure no PI-waiters arrive (or leave) while we are 7721 * changing the priority of the task: 7722 * 7723 * To be able to change p->policy safely, the appropriate 7724 * runqueue lock must be held. 7725 */ 7726 rq = task_rq_lock(p, &rf); 7727 update_rq_clock(rq); 7728 7729 /* 7730 * Changing the policy of the stop threads its a very bad idea: 7731 */ 7732 if (p == rq->stop) { 7733 retval = -EINVAL; 7734 goto unlock; 7735 } 7736 7737 /* 7738 * If not changing anything there's no need to proceed further, 7739 * but store a possible modification of reset_on_fork. 7740 */ 7741 if (unlikely(policy == p->policy)) { 7742 if (fair_policy(policy) && attr->sched_nice != task_nice(p)) 7743 goto change; 7744 if (rt_policy(policy) && attr->sched_priority != p->rt_priority) 7745 goto change; 7746 if (dl_policy(policy) && dl_param_changed(p, attr)) 7747 goto change; 7748 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP) 7749 goto change; 7750 7751 p->sched_reset_on_fork = reset_on_fork; 7752 retval = 0; 7753 goto unlock; 7754 } 7755 change: 7756 7757 if (user) { 7758 #ifdef CONFIG_RT_GROUP_SCHED 7759 /* 7760 * Do not allow realtime tasks into groups that have no runtime 7761 * assigned. 7762 */ 7763 if (rt_bandwidth_enabled() && rt_policy(policy) && 7764 task_group(p)->rt_bandwidth.rt_runtime == 0 && 7765 !task_group_is_autogroup(task_group(p))) { 7766 retval = -EPERM; 7767 goto unlock; 7768 } 7769 #endif 7770 #ifdef CONFIG_SMP 7771 if (dl_bandwidth_enabled() && dl_policy(policy) && 7772 !(attr->sched_flags & SCHED_FLAG_SUGOV)) { 7773 cpumask_t *span = rq->rd->span; 7774 7775 /* 7776 * Don't allow tasks with an affinity mask smaller than 7777 * the entire root_domain to become SCHED_DEADLINE. We 7778 * will also fail if there's no bandwidth available. 7779 */ 7780 if (!cpumask_subset(span, p->cpus_ptr) || 7781 rq->rd->dl_bw.bw == 0) { 7782 retval = -EPERM; 7783 goto unlock; 7784 } 7785 } 7786 #endif 7787 } 7788 7789 /* Re-check policy now with rq lock held: */ 7790 if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { 7791 policy = oldpolicy = -1; 7792 task_rq_unlock(rq, p, &rf); 7793 if (cpuset_locked) 7794 cpuset_unlock(); 7795 goto recheck; 7796 } 7797 7798 /* 7799 * If setscheduling to SCHED_DEADLINE (or changing the parameters 7800 * of a SCHED_DEADLINE task) we need to check if enough bandwidth 7801 * is available. 7802 */ 7803 if ((dl_policy(policy) || dl_task(p)) && sched_dl_overflow(p, policy, attr)) { 7804 retval = -EBUSY; 7805 goto unlock; 7806 } 7807 7808 p->sched_reset_on_fork = reset_on_fork; 7809 oldprio = p->prio; 7810 7811 newprio = __normal_prio(policy, attr->sched_priority, attr->sched_nice); 7812 if (pi) { 7813 /* 7814 * Take priority boosted tasks into account. If the new 7815 * effective priority is unchanged, we just store the new 7816 * normal parameters and do not touch the scheduler class and 7817 * the runqueue. This will be done when the task deboost 7818 * itself. 7819 */ 7820 newprio = rt_effective_prio(p, newprio); 7821 if (newprio == oldprio) 7822 queue_flags &= ~DEQUEUE_MOVE; 7823 } 7824 7825 queued = task_on_rq_queued(p); 7826 running = task_current(rq, p); 7827 if (queued) 7828 dequeue_task(rq, p, queue_flags); 7829 if (running) 7830 put_prev_task(rq, p); 7831 7832 prev_class = p->sched_class; 7833 7834 if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { 7835 __setscheduler_params(p, attr); 7836 __setscheduler_prio(p, newprio); 7837 } 7838 __setscheduler_uclamp(p, attr); 7839 7840 if (queued) { 7841 /* 7842 * We enqueue to tail when the priority of a task is 7843 * increased (user space view). 7844 */ 7845 if (oldprio < p->prio) 7846 queue_flags |= ENQUEUE_HEAD; 7847 7848 enqueue_task(rq, p, queue_flags); 7849 } 7850 if (running) 7851 set_next_task(rq, p); 7852 7853 check_class_changed(rq, p, prev_class, oldprio); 7854 7855 /* Avoid rq from going away on us: */ 7856 preempt_disable(); 7857 head = splice_balance_callbacks(rq); 7858 task_rq_unlock(rq, p, &rf); 7859 7860 if (pi) { 7861 if (cpuset_locked) 7862 cpuset_unlock(); 7863 rt_mutex_adjust_pi(p); 7864 } 7865 7866 /* Run balance callbacks after we've adjusted the PI chain: */ 7867 balance_callbacks(rq, head); 7868 preempt_enable(); 7869 7870 return 0; 7871 7872 unlock: 7873 task_rq_unlock(rq, p, &rf); 7874 if (cpuset_locked) 7875 cpuset_unlock(); 7876 return retval; 7877 } 7878 7879 static int _sched_setscheduler(struct task_struct *p, int policy, 7880 const struct sched_param *param, bool check) 7881 { 7882 struct sched_attr attr = { 7883 .sched_policy = policy, 7884 .sched_priority = param->sched_priority, 7885 .sched_nice = PRIO_TO_NICE(p->static_prio), 7886 }; 7887 7888 /* Fixup the legacy SCHED_RESET_ON_FORK hack. */ 7889 if ((policy != SETPARAM_POLICY) && (policy & SCHED_RESET_ON_FORK)) { 7890 attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK; 7891 policy &= ~SCHED_RESET_ON_FORK; 7892 attr.sched_policy = policy; 7893 } 7894 7895 return __sched_setscheduler(p, &attr, check, true); 7896 } 7897 /** 7898 * sched_setscheduler - change the scheduling policy and/or RT priority of a thread. 7899 * @p: the task in question. 7900 * @policy: new policy. 7901 * @param: structure containing the new RT priority. 7902 * 7903 * Use sched_set_fifo(), read its comment. 7904 * 7905 * Return: 0 on success. An error code otherwise. 7906 * 7907 * NOTE that the task may be already dead. 7908 */ 7909 int sched_setscheduler(struct task_struct *p, int policy, 7910 const struct sched_param *param) 7911 { 7912 return _sched_setscheduler(p, policy, param, true); 7913 } 7914 7915 int sched_setattr(struct task_struct *p, const struct sched_attr *attr) 7916 { 7917 return __sched_setscheduler(p, attr, true, true); 7918 } 7919 7920 int sched_setattr_nocheck(struct task_struct *p, const struct sched_attr *attr) 7921 { 7922 return __sched_setscheduler(p, attr, false, true); 7923 } 7924 EXPORT_SYMBOL_GPL(sched_setattr_nocheck); 7925 7926 /** 7927 * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace. 7928 * @p: the task in question. 7929 * @policy: new policy. 7930 * @param: structure containing the new RT priority. 7931 * 7932 * Just like sched_setscheduler, only don't bother checking if the 7933 * current context has permission. For example, this is needed in 7934 * stop_machine(): we create temporary high priority worker threads, 7935 * but our caller might not have that capability. 7936 * 7937 * Return: 0 on success. An error code otherwise. 7938 */ 7939 int sched_setscheduler_nocheck(struct task_struct *p, int policy, 7940 const struct sched_param *param) 7941 { 7942 return _sched_setscheduler(p, policy, param, false); 7943 } 7944 7945 /* 7946 * SCHED_FIFO is a broken scheduler model; that is, it is fundamentally 7947 * incapable of resource management, which is the one thing an OS really should 7948 * be doing. 7949 * 7950 * This is of course the reason it is limited to privileged users only. 7951 * 7952 * Worse still; it is fundamentally impossible to compose static priority 7953 * workloads. You cannot take two correctly working static prio workloads 7954 * and smash them together and still expect them to work. 7955 * 7956 * For this reason 'all' FIFO tasks the kernel creates are basically at: 7957 * 7958 * MAX_RT_PRIO / 2 7959 * 7960 * The administrator _MUST_ configure the system, the kernel simply doesn't 7961 * know enough information to make a sensible choice. 7962 */ 7963 void sched_set_fifo(struct task_struct *p) 7964 { 7965 struct sched_param sp = { .sched_priority = MAX_RT_PRIO / 2 }; 7966 WARN_ON_ONCE(sched_setscheduler_nocheck(p, SCHED_FIFO, &sp) != 0); 7967 } 7968 EXPORT_SYMBOL_GPL(sched_set_fifo); 7969 7970 /* 7971 * For when you don't much care about FIFO, but want to be above SCHED_NORMAL. 7972 */ 7973 void sched_set_fifo_low(struct task_struct *p) 7974 { 7975 struct sched_param sp = { .sched_priority = 1 }; 7976 WARN_ON_ONCE(sched_setscheduler_nocheck(p, SCHED_FIFO, &sp) != 0); 7977 } 7978 EXPORT_SYMBOL_GPL(sched_set_fifo_low); 7979 7980 void sched_set_normal(struct task_struct *p, int nice) 7981 { 7982 struct sched_attr attr = { 7983 .sched_policy = SCHED_NORMAL, 7984 .sched_nice = nice, 7985 }; 7986 WARN_ON_ONCE(sched_setattr_nocheck(p, &attr) != 0); 7987 } 7988 EXPORT_SYMBOL_GPL(sched_set_normal); 7989 7990 static int 7991 do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param) 7992 { 7993 struct sched_param lparam; 7994 struct task_struct *p; 7995 int retval; 7996 7997 if (!param || pid < 0) 7998 return -EINVAL; 7999 if (copy_from_user(&lparam, param, sizeof(struct sched_param))) 8000 return -EFAULT; 8001 8002 rcu_read_lock(); 8003 retval = -ESRCH; 8004 p = find_process_by_pid(pid); 8005 if (likely(p)) 8006 get_task_struct(p); 8007 rcu_read_unlock(); 8008 8009 if (likely(p)) { 8010 retval = sched_setscheduler(p, policy, &lparam); 8011 put_task_struct(p); 8012 } 8013 8014 return retval; 8015 } 8016 8017 /* 8018 * Mimics kernel/events/core.c perf_copy_attr(). 8019 */ 8020 static int sched_copy_attr(struct sched_attr __user *uattr, struct sched_attr *attr) 8021 { 8022 u32 size; 8023 int ret; 8024 8025 /* Zero the full structure, so that a short copy will be nice: */ 8026 memset(attr, 0, sizeof(*attr)); 8027 8028 ret = get_user(size, &uattr->size); 8029 if (ret) 8030 return ret; 8031 8032 /* ABI compatibility quirk: */ 8033 if (!size) 8034 size = SCHED_ATTR_SIZE_VER0; 8035 if (size < SCHED_ATTR_SIZE_VER0 || size > PAGE_SIZE) 8036 goto err_size; 8037 8038 ret = copy_struct_from_user(attr, sizeof(*attr), uattr, size); 8039 if (ret) { 8040 if (ret == -E2BIG) 8041 goto err_size; 8042 return ret; 8043 } 8044 8045 if ((attr->sched_flags & SCHED_FLAG_UTIL_CLAMP) && 8046 size < SCHED_ATTR_SIZE_VER1) 8047 return -EINVAL; 8048 8049 /* 8050 * XXX: Do we want to be lenient like existing syscalls; or do we want 8051 * to be strict and return an error on out-of-bounds values? 8052 */ 8053 attr->sched_nice = clamp(attr->sched_nice, MIN_NICE, MAX_NICE); 8054 8055 return 0; 8056 8057 err_size: 8058 put_user(sizeof(*attr), &uattr->size); 8059 return -E2BIG; 8060 } 8061 8062 static void get_params(struct task_struct *p, struct sched_attr *attr) 8063 { 8064 if (task_has_dl_policy(p)) 8065 __getparam_dl(p, attr); 8066 else if (task_has_rt_policy(p)) 8067 attr->sched_priority = p->rt_priority; 8068 else 8069 attr->sched_nice = task_nice(p); 8070 } 8071 8072 /** 8073 * sys_sched_setscheduler - set/change the scheduler policy and RT priority 8074 * @pid: the pid in question. 8075 * @policy: new policy. 8076 * @param: structure containing the new RT priority. 8077 * 8078 * Return: 0 on success. An error code otherwise. 8079 */ 8080 SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy, struct sched_param __user *, param) 8081 { 8082 if (policy < 0) 8083 return -EINVAL; 8084 8085 return do_sched_setscheduler(pid, policy, param); 8086 } 8087 8088 /** 8089 * sys_sched_setparam - set/change the RT priority of a thread 8090 * @pid: the pid in question. 8091 * @param: structure containing the new RT priority. 8092 * 8093 * Return: 0 on success. An error code otherwise. 8094 */ 8095 SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param) 8096 { 8097 return do_sched_setscheduler(pid, SETPARAM_POLICY, param); 8098 } 8099 8100 /** 8101 * sys_sched_setattr - same as above, but with extended sched_attr 8102 * @pid: the pid in question. 8103 * @uattr: structure containing the extended parameters. 8104 * @flags: for future extension. 8105 */ 8106 SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr, 8107 unsigned int, flags) 8108 { 8109 struct sched_attr attr; 8110 struct task_struct *p; 8111 int retval; 8112 8113 if (!uattr || pid < 0 || flags) 8114 return -EINVAL; 8115 8116 retval = sched_copy_attr(uattr, &attr); 8117 if (retval) 8118 return retval; 8119 8120 if ((int)attr.sched_policy < 0) 8121 return -EINVAL; 8122 if (attr.sched_flags & SCHED_FLAG_KEEP_POLICY) 8123 attr.sched_policy = SETPARAM_POLICY; 8124 8125 rcu_read_lock(); 8126 retval = -ESRCH; 8127 p = find_process_by_pid(pid); 8128 if (likely(p)) 8129 get_task_struct(p); 8130 rcu_read_unlock(); 8131 8132 if (likely(p)) { 8133 if (attr.sched_flags & SCHED_FLAG_KEEP_PARAMS) 8134 get_params(p, &attr); 8135 retval = sched_setattr(p, &attr); 8136 put_task_struct(p); 8137 } 8138 8139 return retval; 8140 } 8141 8142 /** 8143 * sys_sched_getscheduler - get the policy (scheduling class) of a thread 8144 * @pid: the pid in question. 8145 * 8146 * Return: On success, the policy of the thread. Otherwise, a negative error 8147 * code. 8148 */ 8149 SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid) 8150 { 8151 struct task_struct *p; 8152 int retval; 8153 8154 if (pid < 0) 8155 return -EINVAL; 8156 8157 retval = -ESRCH; 8158 rcu_read_lock(); 8159 p = find_process_by_pid(pid); 8160 if (p) { 8161 retval = security_task_getscheduler(p); 8162 if (!retval) 8163 retval = p->policy 8164 | (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0); 8165 } 8166 rcu_read_unlock(); 8167 return retval; 8168 } 8169 8170 /** 8171 * sys_sched_getparam - get the RT priority of a thread 8172 * @pid: the pid in question. 8173 * @param: structure containing the RT priority. 8174 * 8175 * Return: On success, 0 and the RT priority is in @param. Otherwise, an error 8176 * code. 8177 */ 8178 SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param) 8179 { 8180 struct sched_param lp = { .sched_priority = 0 }; 8181 struct task_struct *p; 8182 int retval; 8183 8184 if (!param || pid < 0) 8185 return -EINVAL; 8186 8187 rcu_read_lock(); 8188 p = find_process_by_pid(pid); 8189 retval = -ESRCH; 8190 if (!p) 8191 goto out_unlock; 8192 8193 retval = security_task_getscheduler(p); 8194 if (retval) 8195 goto out_unlock; 8196 8197 if (task_has_rt_policy(p)) 8198 lp.sched_priority = p->rt_priority; 8199 rcu_read_unlock(); 8200 8201 /* 8202 * This one might sleep, we cannot do it with a spinlock held ... 8203 */ 8204 retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0; 8205 8206 return retval; 8207 8208 out_unlock: 8209 rcu_read_unlock(); 8210 return retval; 8211 } 8212 8213 /* 8214 * Copy the kernel size attribute structure (which might be larger 8215 * than what user-space knows about) to user-space. 8216 * 8217 * Note that all cases are valid: user-space buffer can be larger or 8218 * smaller than the kernel-space buffer. The usual case is that both 8219 * have the same size. 8220 */ 8221 static int 8222 sched_attr_copy_to_user(struct sched_attr __user *uattr, 8223 struct sched_attr *kattr, 8224 unsigned int usize) 8225 { 8226 unsigned int ksize = sizeof(*kattr); 8227 8228 if (!access_ok(uattr, usize)) 8229 return -EFAULT; 8230 8231 /* 8232 * sched_getattr() ABI forwards and backwards compatibility: 8233 * 8234 * If usize == ksize then we just copy everything to user-space and all is good. 8235 * 8236 * If usize < ksize then we only copy as much as user-space has space for, 8237 * this keeps ABI compatibility as well. We skip the rest. 8238 * 8239 * If usize > ksize then user-space is using a newer version of the ABI, 8240 * which part the kernel doesn't know about. Just ignore it - tooling can 8241 * detect the kernel's knowledge of attributes from the attr->size value 8242 * which is set to ksize in this case. 8243 */ 8244 kattr->size = min(usize, ksize); 8245 8246 if (copy_to_user(uattr, kattr, kattr->size)) 8247 return -EFAULT; 8248 8249 return 0; 8250 } 8251 8252 /** 8253 * sys_sched_getattr - similar to sched_getparam, but with sched_attr 8254 * @pid: the pid in question. 8255 * @uattr: structure containing the extended parameters. 8256 * @usize: sizeof(attr) for fwd/bwd comp. 8257 * @flags: for future extension. 8258 */ 8259 SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr, 8260 unsigned int, usize, unsigned int, flags) 8261 { 8262 struct sched_attr kattr = { }; 8263 struct task_struct *p; 8264 int retval; 8265 8266 if (!uattr || pid < 0 || usize > PAGE_SIZE || 8267 usize < SCHED_ATTR_SIZE_VER0 || flags) 8268 return -EINVAL; 8269 8270 rcu_read_lock(); 8271 p = find_process_by_pid(pid); 8272 retval = -ESRCH; 8273 if (!p) 8274 goto out_unlock; 8275 8276 retval = security_task_getscheduler(p); 8277 if (retval) 8278 goto out_unlock; 8279 8280 kattr.sched_policy = p->policy; 8281 if (p->sched_reset_on_fork) 8282 kattr.sched_flags |= SCHED_FLAG_RESET_ON_FORK; 8283 get_params(p, &kattr); 8284 kattr.sched_flags &= SCHED_FLAG_ALL; 8285 8286 #ifdef CONFIG_UCLAMP_TASK 8287 /* 8288 * This could race with another potential updater, but this is fine 8289 * because it'll correctly read the old or the new value. We don't need 8290 * to guarantee who wins the race as long as it doesn't return garbage. 8291 */ 8292 kattr.sched_util_min = p->uclamp_req[UCLAMP_MIN].value; 8293 kattr.sched_util_max = p->uclamp_req[UCLAMP_MAX].value; 8294 #endif 8295 8296 rcu_read_unlock(); 8297 8298 return sched_attr_copy_to_user(uattr, &kattr, usize); 8299 8300 out_unlock: 8301 rcu_read_unlock(); 8302 return retval; 8303 } 8304 8305 #ifdef CONFIG_SMP 8306 int dl_task_check_affinity(struct task_struct *p, const struct cpumask *mask) 8307 { 8308 int ret = 0; 8309 8310 /* 8311 * If the task isn't a deadline task or admission control is 8312 * disabled then we don't care about affinity changes. 8313 */ 8314 if (!task_has_dl_policy(p) || !dl_bandwidth_enabled()) 8315 return 0; 8316 8317 /* 8318 * Since bandwidth control happens on root_domain basis, 8319 * if admission test is enabled, we only admit -deadline 8320 * tasks allowed to run on all the CPUs in the task's 8321 * root_domain. 8322 */ 8323 rcu_read_lock(); 8324 if (!cpumask_subset(task_rq(p)->rd->span, mask)) 8325 ret = -EBUSY; 8326 rcu_read_unlock(); 8327 return ret; 8328 } 8329 #endif 8330 8331 static int 8332 __sched_setaffinity(struct task_struct *p, struct affinity_context *ctx) 8333 { 8334 int retval; 8335 cpumask_var_t cpus_allowed, new_mask; 8336 8337 if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) 8338 return -ENOMEM; 8339 8340 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) { 8341 retval = -ENOMEM; 8342 goto out_free_cpus_allowed; 8343 } 8344 8345 cpuset_cpus_allowed(p, cpus_allowed); 8346 cpumask_and(new_mask, ctx->new_mask, cpus_allowed); 8347 8348 ctx->new_mask = new_mask; 8349 ctx->flags |= SCA_CHECK; 8350 8351 retval = dl_task_check_affinity(p, new_mask); 8352 if (retval) 8353 goto out_free_new_mask; 8354 8355 retval = __set_cpus_allowed_ptr(p, ctx); 8356 if (retval) 8357 goto out_free_new_mask; 8358 8359 cpuset_cpus_allowed(p, cpus_allowed); 8360 if (!cpumask_subset(new_mask, cpus_allowed)) { 8361 /* 8362 * We must have raced with a concurrent cpuset update. 8363 * Just reset the cpumask to the cpuset's cpus_allowed. 8364 */ 8365 cpumask_copy(new_mask, cpus_allowed); 8366 8367 /* 8368 * If SCA_USER is set, a 2nd call to __set_cpus_allowed_ptr() 8369 * will restore the previous user_cpus_ptr value. 8370 * 8371 * In the unlikely event a previous user_cpus_ptr exists, 8372 * we need to further restrict the mask to what is allowed 8373 * by that old user_cpus_ptr. 8374 */ 8375 if (unlikely((ctx->flags & SCA_USER) && ctx->user_mask)) { 8376 bool empty = !cpumask_and(new_mask, new_mask, 8377 ctx->user_mask); 8378 8379 if (WARN_ON_ONCE(empty)) 8380 cpumask_copy(new_mask, cpus_allowed); 8381 } 8382 __set_cpus_allowed_ptr(p, ctx); 8383 retval = -EINVAL; 8384 } 8385 8386 out_free_new_mask: 8387 free_cpumask_var(new_mask); 8388 out_free_cpus_allowed: 8389 free_cpumask_var(cpus_allowed); 8390 return retval; 8391 } 8392 8393 long sched_setaffinity(pid_t pid, const struct cpumask *in_mask) 8394 { 8395 struct affinity_context ac; 8396 struct cpumask *user_mask; 8397 struct task_struct *p; 8398 int retval; 8399 8400 rcu_read_lock(); 8401 8402 p = find_process_by_pid(pid); 8403 if (!p) { 8404 rcu_read_unlock(); 8405 return -ESRCH; 8406 } 8407 8408 /* Prevent p going away */ 8409 get_task_struct(p); 8410 rcu_read_unlock(); 8411 8412 if (p->flags & PF_NO_SETAFFINITY) { 8413 retval = -EINVAL; 8414 goto out_put_task; 8415 } 8416 8417 if (!check_same_owner(p)) { 8418 rcu_read_lock(); 8419 if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) { 8420 rcu_read_unlock(); 8421 retval = -EPERM; 8422 goto out_put_task; 8423 } 8424 rcu_read_unlock(); 8425 } 8426 8427 retval = security_task_setscheduler(p); 8428 if (retval) 8429 goto out_put_task; 8430 8431 /* 8432 * With non-SMP configs, user_cpus_ptr/user_mask isn't used and 8433 * alloc_user_cpus_ptr() returns NULL. 8434 */ 8435 user_mask = alloc_user_cpus_ptr(NUMA_NO_NODE); 8436 if (user_mask) { 8437 cpumask_copy(user_mask, in_mask); 8438 } else if (IS_ENABLED(CONFIG_SMP)) { 8439 retval = -ENOMEM; 8440 goto out_put_task; 8441 } 8442 8443 ac = (struct affinity_context){ 8444 .new_mask = in_mask, 8445 .user_mask = user_mask, 8446 .flags = SCA_USER, 8447 }; 8448 8449 retval = __sched_setaffinity(p, &ac); 8450 kfree(ac.user_mask); 8451 8452 out_put_task: 8453 put_task_struct(p); 8454 return retval; 8455 } 8456 8457 static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len, 8458 struct cpumask *new_mask) 8459 { 8460 if (len < cpumask_size()) 8461 cpumask_clear(new_mask); 8462 else if (len > cpumask_size()) 8463 len = cpumask_size(); 8464 8465 return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0; 8466 } 8467 8468 /** 8469 * sys_sched_setaffinity - set the CPU affinity of a process 8470 * @pid: pid of the process 8471 * @len: length in bytes of the bitmask pointed to by user_mask_ptr 8472 * @user_mask_ptr: user-space pointer to the new CPU mask 8473 * 8474 * Return: 0 on success. An error code otherwise. 8475 */ 8476 SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len, 8477 unsigned long __user *, user_mask_ptr) 8478 { 8479 cpumask_var_t new_mask; 8480 int retval; 8481 8482 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) 8483 return -ENOMEM; 8484 8485 retval = get_user_cpu_mask(user_mask_ptr, len, new_mask); 8486 if (retval == 0) 8487 retval = sched_setaffinity(pid, new_mask); 8488 free_cpumask_var(new_mask); 8489 return retval; 8490 } 8491 8492 long sched_getaffinity(pid_t pid, struct cpumask *mask) 8493 { 8494 struct task_struct *p; 8495 unsigned long flags; 8496 int retval; 8497 8498 rcu_read_lock(); 8499 8500 retval = -ESRCH; 8501 p = find_process_by_pid(pid); 8502 if (!p) 8503 goto out_unlock; 8504 8505 retval = security_task_getscheduler(p); 8506 if (retval) 8507 goto out_unlock; 8508 8509 raw_spin_lock_irqsave(&p->pi_lock, flags); 8510 cpumask_and(mask, &p->cpus_mask, cpu_active_mask); 8511 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 8512 8513 out_unlock: 8514 rcu_read_unlock(); 8515 8516 return retval; 8517 } 8518 8519 /** 8520 * sys_sched_getaffinity - get the CPU affinity of a process 8521 * @pid: pid of the process 8522 * @len: length in bytes of the bitmask pointed to by user_mask_ptr 8523 * @user_mask_ptr: user-space pointer to hold the current CPU mask 8524 * 8525 * Return: size of CPU mask copied to user_mask_ptr on success. An 8526 * error code otherwise. 8527 */ 8528 SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len, 8529 unsigned long __user *, user_mask_ptr) 8530 { 8531 int ret; 8532 cpumask_var_t mask; 8533 8534 if ((len * BITS_PER_BYTE) < nr_cpu_ids) 8535 return -EINVAL; 8536 if (len & (sizeof(unsigned long)-1)) 8537 return -EINVAL; 8538 8539 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) 8540 return -ENOMEM; 8541 8542 ret = sched_getaffinity(pid, mask); 8543 if (ret == 0) { 8544 unsigned int retlen = min(len, cpumask_size()); 8545 8546 if (copy_to_user(user_mask_ptr, cpumask_bits(mask), retlen)) 8547 ret = -EFAULT; 8548 else 8549 ret = retlen; 8550 } 8551 free_cpumask_var(mask); 8552 8553 return ret; 8554 } 8555 8556 static void do_sched_yield(void) 8557 { 8558 struct rq_flags rf; 8559 struct rq *rq; 8560 8561 rq = this_rq_lock_irq(&rf); 8562 8563 schedstat_inc(rq->yld_count); 8564 current->sched_class->yield_task(rq); 8565 8566 preempt_disable(); 8567 rq_unlock_irq(rq, &rf); 8568 sched_preempt_enable_no_resched(); 8569 8570 schedule(); 8571 } 8572 8573 /** 8574 * sys_sched_yield - yield the current processor to other threads. 8575 * 8576 * This function yields the current CPU to other tasks. If there are no 8577 * other threads running on this CPU then this function will return. 8578 * 8579 * Return: 0. 8580 */ 8581 SYSCALL_DEFINE0(sched_yield) 8582 { 8583 do_sched_yield(); 8584 return 0; 8585 } 8586 8587 #if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC) 8588 int __sched __cond_resched(void) 8589 { 8590 if (should_resched(0)) { 8591 preempt_schedule_common(); 8592 return 1; 8593 } 8594 /* 8595 * In preemptible kernels, ->rcu_read_lock_nesting tells the tick 8596 * whether the current CPU is in an RCU read-side critical section, 8597 * so the tick can report quiescent states even for CPUs looping 8598 * in kernel context. In contrast, in non-preemptible kernels, 8599 * RCU readers leave no in-memory hints, which means that CPU-bound 8600 * processes executing in kernel context might never report an 8601 * RCU quiescent state. Therefore, the following code causes 8602 * cond_resched() to report a quiescent state, but only when RCU 8603 * is in urgent need of one. 8604 */ 8605 #ifndef CONFIG_PREEMPT_RCU 8606 rcu_all_qs(); 8607 #endif 8608 return 0; 8609 } 8610 EXPORT_SYMBOL(__cond_resched); 8611 #endif 8612 8613 #ifdef CONFIG_PREEMPT_DYNAMIC 8614 #if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 8615 #define cond_resched_dynamic_enabled __cond_resched 8616 #define cond_resched_dynamic_disabled ((void *)&__static_call_return0) 8617 DEFINE_STATIC_CALL_RET0(cond_resched, __cond_resched); 8618 EXPORT_STATIC_CALL_TRAMP(cond_resched); 8619 8620 #define might_resched_dynamic_enabled __cond_resched 8621 #define might_resched_dynamic_disabled ((void *)&__static_call_return0) 8622 DEFINE_STATIC_CALL_RET0(might_resched, __cond_resched); 8623 EXPORT_STATIC_CALL_TRAMP(might_resched); 8624 #elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 8625 static DEFINE_STATIC_KEY_FALSE(sk_dynamic_cond_resched); 8626 int __sched dynamic_cond_resched(void) 8627 { 8628 klp_sched_try_switch(); 8629 if (!static_branch_unlikely(&sk_dynamic_cond_resched)) 8630 return 0; 8631 return __cond_resched(); 8632 } 8633 EXPORT_SYMBOL(dynamic_cond_resched); 8634 8635 static DEFINE_STATIC_KEY_FALSE(sk_dynamic_might_resched); 8636 int __sched dynamic_might_resched(void) 8637 { 8638 if (!static_branch_unlikely(&sk_dynamic_might_resched)) 8639 return 0; 8640 return __cond_resched(); 8641 } 8642 EXPORT_SYMBOL(dynamic_might_resched); 8643 #endif 8644 #endif 8645 8646 /* 8647 * __cond_resched_lock() - if a reschedule is pending, drop the given lock, 8648 * call schedule, and on return reacquire the lock. 8649 * 8650 * This works OK both with and without CONFIG_PREEMPTION. We do strange low-level 8651 * operations here to prevent schedule() from being called twice (once via 8652 * spin_unlock(), once by hand). 8653 */ 8654 int __cond_resched_lock(spinlock_t *lock) 8655 { 8656 int resched = should_resched(PREEMPT_LOCK_OFFSET); 8657 int ret = 0; 8658 8659 lockdep_assert_held(lock); 8660 8661 if (spin_needbreak(lock) || resched) { 8662 spin_unlock(lock); 8663 if (!_cond_resched()) 8664 cpu_relax(); 8665 ret = 1; 8666 spin_lock(lock); 8667 } 8668 return ret; 8669 } 8670 EXPORT_SYMBOL(__cond_resched_lock); 8671 8672 int __cond_resched_rwlock_read(rwlock_t *lock) 8673 { 8674 int resched = should_resched(PREEMPT_LOCK_OFFSET); 8675 int ret = 0; 8676 8677 lockdep_assert_held_read(lock); 8678 8679 if (rwlock_needbreak(lock) || resched) { 8680 read_unlock(lock); 8681 if (!_cond_resched()) 8682 cpu_relax(); 8683 ret = 1; 8684 read_lock(lock); 8685 } 8686 return ret; 8687 } 8688 EXPORT_SYMBOL(__cond_resched_rwlock_read); 8689 8690 int __cond_resched_rwlock_write(rwlock_t *lock) 8691 { 8692 int resched = should_resched(PREEMPT_LOCK_OFFSET); 8693 int ret = 0; 8694 8695 lockdep_assert_held_write(lock); 8696 8697 if (rwlock_needbreak(lock) || resched) { 8698 write_unlock(lock); 8699 if (!_cond_resched()) 8700 cpu_relax(); 8701 ret = 1; 8702 write_lock(lock); 8703 } 8704 return ret; 8705 } 8706 EXPORT_SYMBOL(__cond_resched_rwlock_write); 8707 8708 #ifdef CONFIG_PREEMPT_DYNAMIC 8709 8710 #ifdef CONFIG_GENERIC_ENTRY 8711 #include <linux/entry-common.h> 8712 #endif 8713 8714 /* 8715 * SC:cond_resched 8716 * SC:might_resched 8717 * SC:preempt_schedule 8718 * SC:preempt_schedule_notrace 8719 * SC:irqentry_exit_cond_resched 8720 * 8721 * 8722 * NONE: 8723 * cond_resched <- __cond_resched 8724 * might_resched <- RET0 8725 * preempt_schedule <- NOP 8726 * preempt_schedule_notrace <- NOP 8727 * irqentry_exit_cond_resched <- NOP 8728 * 8729 * VOLUNTARY: 8730 * cond_resched <- __cond_resched 8731 * might_resched <- __cond_resched 8732 * preempt_schedule <- NOP 8733 * preempt_schedule_notrace <- NOP 8734 * irqentry_exit_cond_resched <- NOP 8735 * 8736 * FULL: 8737 * cond_resched <- RET0 8738 * might_resched <- RET0 8739 * preempt_schedule <- preempt_schedule 8740 * preempt_schedule_notrace <- preempt_schedule_notrace 8741 * irqentry_exit_cond_resched <- irqentry_exit_cond_resched 8742 */ 8743 8744 enum { 8745 preempt_dynamic_undefined = -1, 8746 preempt_dynamic_none, 8747 preempt_dynamic_voluntary, 8748 preempt_dynamic_full, 8749 }; 8750 8751 int preempt_dynamic_mode = preempt_dynamic_undefined; 8752 8753 int sched_dynamic_mode(const char *str) 8754 { 8755 if (!strcmp(str, "none")) 8756 return preempt_dynamic_none; 8757 8758 if (!strcmp(str, "voluntary")) 8759 return preempt_dynamic_voluntary; 8760 8761 if (!strcmp(str, "full")) 8762 return preempt_dynamic_full; 8763 8764 return -EINVAL; 8765 } 8766 8767 #if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 8768 #define preempt_dynamic_enable(f) static_call_update(f, f##_dynamic_enabled) 8769 #define preempt_dynamic_disable(f) static_call_update(f, f##_dynamic_disabled) 8770 #elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 8771 #define preempt_dynamic_enable(f) static_key_enable(&sk_dynamic_##f.key) 8772 #define preempt_dynamic_disable(f) static_key_disable(&sk_dynamic_##f.key) 8773 #else 8774 #error "Unsupported PREEMPT_DYNAMIC mechanism" 8775 #endif 8776 8777 static DEFINE_MUTEX(sched_dynamic_mutex); 8778 static bool klp_override; 8779 8780 static void __sched_dynamic_update(int mode) 8781 { 8782 /* 8783 * Avoid {NONE,VOLUNTARY} -> FULL transitions from ever ending up in 8784 * the ZERO state, which is invalid. 8785 */ 8786 if (!klp_override) 8787 preempt_dynamic_enable(cond_resched); 8788 preempt_dynamic_enable(might_resched); 8789 preempt_dynamic_enable(preempt_schedule); 8790 preempt_dynamic_enable(preempt_schedule_notrace); 8791 preempt_dynamic_enable(irqentry_exit_cond_resched); 8792 8793 switch (mode) { 8794 case preempt_dynamic_none: 8795 if (!klp_override) 8796 preempt_dynamic_enable(cond_resched); 8797 preempt_dynamic_disable(might_resched); 8798 preempt_dynamic_disable(preempt_schedule); 8799 preempt_dynamic_disable(preempt_schedule_notrace); 8800 preempt_dynamic_disable(irqentry_exit_cond_resched); 8801 if (mode != preempt_dynamic_mode) 8802 pr_info("Dynamic Preempt: none\n"); 8803 break; 8804 8805 case preempt_dynamic_voluntary: 8806 if (!klp_override) 8807 preempt_dynamic_enable(cond_resched); 8808 preempt_dynamic_enable(might_resched); 8809 preempt_dynamic_disable(preempt_schedule); 8810 preempt_dynamic_disable(preempt_schedule_notrace); 8811 preempt_dynamic_disable(irqentry_exit_cond_resched); 8812 if (mode != preempt_dynamic_mode) 8813 pr_info("Dynamic Preempt: voluntary\n"); 8814 break; 8815 8816 case preempt_dynamic_full: 8817 if (!klp_override) 8818 preempt_dynamic_disable(cond_resched); 8819 preempt_dynamic_disable(might_resched); 8820 preempt_dynamic_enable(preempt_schedule); 8821 preempt_dynamic_enable(preempt_schedule_notrace); 8822 preempt_dynamic_enable(irqentry_exit_cond_resched); 8823 if (mode != preempt_dynamic_mode) 8824 pr_info("Dynamic Preempt: full\n"); 8825 break; 8826 } 8827 8828 preempt_dynamic_mode = mode; 8829 } 8830 8831 void sched_dynamic_update(int mode) 8832 { 8833 mutex_lock(&sched_dynamic_mutex); 8834 __sched_dynamic_update(mode); 8835 mutex_unlock(&sched_dynamic_mutex); 8836 } 8837 8838 #ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL 8839 8840 static int klp_cond_resched(void) 8841 { 8842 __klp_sched_try_switch(); 8843 return __cond_resched(); 8844 } 8845 8846 void sched_dynamic_klp_enable(void) 8847 { 8848 mutex_lock(&sched_dynamic_mutex); 8849 8850 klp_override = true; 8851 static_call_update(cond_resched, klp_cond_resched); 8852 8853 mutex_unlock(&sched_dynamic_mutex); 8854 } 8855 8856 void sched_dynamic_klp_disable(void) 8857 { 8858 mutex_lock(&sched_dynamic_mutex); 8859 8860 klp_override = false; 8861 __sched_dynamic_update(preempt_dynamic_mode); 8862 8863 mutex_unlock(&sched_dynamic_mutex); 8864 } 8865 8866 #endif /* CONFIG_HAVE_PREEMPT_DYNAMIC_CALL */ 8867 8868 static int __init setup_preempt_mode(char *str) 8869 { 8870 int mode = sched_dynamic_mode(str); 8871 if (mode < 0) { 8872 pr_warn("Dynamic Preempt: unsupported mode: %s\n", str); 8873 return 0; 8874 } 8875 8876 sched_dynamic_update(mode); 8877 return 1; 8878 } 8879 __setup("preempt=", setup_preempt_mode); 8880 8881 static void __init preempt_dynamic_init(void) 8882 { 8883 if (preempt_dynamic_mode == preempt_dynamic_undefined) { 8884 if (IS_ENABLED(CONFIG_PREEMPT_NONE)) { 8885 sched_dynamic_update(preempt_dynamic_none); 8886 } else if (IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY)) { 8887 sched_dynamic_update(preempt_dynamic_voluntary); 8888 } else { 8889 /* Default static call setting, nothing to do */ 8890 WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT)); 8891 preempt_dynamic_mode = preempt_dynamic_full; 8892 pr_info("Dynamic Preempt: full\n"); 8893 } 8894 } 8895 } 8896 8897 #define PREEMPT_MODEL_ACCESSOR(mode) \ 8898 bool preempt_model_##mode(void) \ 8899 { \ 8900 WARN_ON_ONCE(preempt_dynamic_mode == preempt_dynamic_undefined); \ 8901 return preempt_dynamic_mode == preempt_dynamic_##mode; \ 8902 } \ 8903 EXPORT_SYMBOL_GPL(preempt_model_##mode) 8904 8905 PREEMPT_MODEL_ACCESSOR(none); 8906 PREEMPT_MODEL_ACCESSOR(voluntary); 8907 PREEMPT_MODEL_ACCESSOR(full); 8908 8909 #else /* !CONFIG_PREEMPT_DYNAMIC */ 8910 8911 static inline void preempt_dynamic_init(void) { } 8912 8913 #endif /* #ifdef CONFIG_PREEMPT_DYNAMIC */ 8914 8915 /** 8916 * yield - yield the current processor to other threads. 8917 * 8918 * Do not ever use this function, there's a 99% chance you're doing it wrong. 8919 * 8920 * The scheduler is at all times free to pick the calling task as the most 8921 * eligible task to run, if removing the yield() call from your code breaks 8922 * it, it's already broken. 8923 * 8924 * Typical broken usage is: 8925 * 8926 * while (!event) 8927 * yield(); 8928 * 8929 * where one assumes that yield() will let 'the other' process run that will 8930 * make event true. If the current task is a SCHED_FIFO task that will never 8931 * happen. Never use yield() as a progress guarantee!! 8932 * 8933 * If you want to use yield() to wait for something, use wait_event(). 8934 * If you want to use yield() to be 'nice' for others, use cond_resched(). 8935 * If you still want to use yield(), do not! 8936 */ 8937 void __sched yield(void) 8938 { 8939 set_current_state(TASK_RUNNING); 8940 do_sched_yield(); 8941 } 8942 EXPORT_SYMBOL(yield); 8943 8944 /** 8945 * yield_to - yield the current processor to another thread in 8946 * your thread group, or accelerate that thread toward the 8947 * processor it's on. 8948 * @p: target task 8949 * @preempt: whether task preemption is allowed or not 8950 * 8951 * It's the caller's job to ensure that the target task struct 8952 * can't go away on us before we can do any checks. 8953 * 8954 * Return: 8955 * true (>0) if we indeed boosted the target task. 8956 * false (0) if we failed to boost the target. 8957 * -ESRCH if there's no task to yield to. 8958 */ 8959 int __sched yield_to(struct task_struct *p, bool preempt) 8960 { 8961 struct task_struct *curr = current; 8962 struct rq *rq, *p_rq; 8963 unsigned long flags; 8964 int yielded = 0; 8965 8966 local_irq_save(flags); 8967 rq = this_rq(); 8968 8969 again: 8970 p_rq = task_rq(p); 8971 /* 8972 * If we're the only runnable task on the rq and target rq also 8973 * has only one task, there's absolutely no point in yielding. 8974 */ 8975 if (rq->nr_running == 1 && p_rq->nr_running == 1) { 8976 yielded = -ESRCH; 8977 goto out_irq; 8978 } 8979 8980 double_rq_lock(rq, p_rq); 8981 if (task_rq(p) != p_rq) { 8982 double_rq_unlock(rq, p_rq); 8983 goto again; 8984 } 8985 8986 if (!curr->sched_class->yield_to_task) 8987 goto out_unlock; 8988 8989 if (curr->sched_class != p->sched_class) 8990 goto out_unlock; 8991 8992 if (task_on_cpu(p_rq, p) || !task_is_running(p)) 8993 goto out_unlock; 8994 8995 yielded = curr->sched_class->yield_to_task(rq, p); 8996 if (yielded) { 8997 schedstat_inc(rq->yld_count); 8998 /* 8999 * Make p's CPU reschedule; pick_next_entity takes care of 9000 * fairness. 9001 */ 9002 if (preempt && rq != p_rq) 9003 resched_curr(p_rq); 9004 } 9005 9006 out_unlock: 9007 double_rq_unlock(rq, p_rq); 9008 out_irq: 9009 local_irq_restore(flags); 9010 9011 if (yielded > 0) 9012 schedule(); 9013 9014 return yielded; 9015 } 9016 EXPORT_SYMBOL_GPL(yield_to); 9017 9018 int io_schedule_prepare(void) 9019 { 9020 int old_iowait = current->in_iowait; 9021 9022 current->in_iowait = 1; 9023 blk_flush_plug(current->plug, true); 9024 return old_iowait; 9025 } 9026 9027 void io_schedule_finish(int token) 9028 { 9029 current->in_iowait = token; 9030 } 9031 9032 /* 9033 * This task is about to go to sleep on IO. Increment rq->nr_iowait so 9034 * that process accounting knows that this is a task in IO wait state. 9035 */ 9036 long __sched io_schedule_timeout(long timeout) 9037 { 9038 int token; 9039 long ret; 9040 9041 token = io_schedule_prepare(); 9042 ret = schedule_timeout(timeout); 9043 io_schedule_finish(token); 9044 9045 return ret; 9046 } 9047 EXPORT_SYMBOL(io_schedule_timeout); 9048 9049 void __sched io_schedule(void) 9050 { 9051 int token; 9052 9053 token = io_schedule_prepare(); 9054 schedule(); 9055 io_schedule_finish(token); 9056 } 9057 EXPORT_SYMBOL(io_schedule); 9058 9059 /** 9060 * sys_sched_get_priority_max - return maximum RT priority. 9061 * @policy: scheduling class. 9062 * 9063 * Return: On success, this syscall returns the maximum 9064 * rt_priority that can be used by a given scheduling class. 9065 * On failure, a negative error code is returned. 9066 */ 9067 SYSCALL_DEFINE1(sched_get_priority_max, int, policy) 9068 { 9069 int ret = -EINVAL; 9070 9071 switch (policy) { 9072 case SCHED_FIFO: 9073 case SCHED_RR: 9074 ret = MAX_RT_PRIO-1; 9075 break; 9076 case SCHED_DEADLINE: 9077 case SCHED_NORMAL: 9078 case SCHED_BATCH: 9079 case SCHED_IDLE: 9080 ret = 0; 9081 break; 9082 } 9083 return ret; 9084 } 9085 9086 /** 9087 * sys_sched_get_priority_min - return minimum RT priority. 9088 * @policy: scheduling class. 9089 * 9090 * Return: On success, this syscall returns the minimum 9091 * rt_priority that can be used by a given scheduling class. 9092 * On failure, a negative error code is returned. 9093 */ 9094 SYSCALL_DEFINE1(sched_get_priority_min, int, policy) 9095 { 9096 int ret = -EINVAL; 9097 9098 switch (policy) { 9099 case SCHED_FIFO: 9100 case SCHED_RR: 9101 ret = 1; 9102 break; 9103 case SCHED_DEADLINE: 9104 case SCHED_NORMAL: 9105 case SCHED_BATCH: 9106 case SCHED_IDLE: 9107 ret = 0; 9108 } 9109 return ret; 9110 } 9111 9112 static int sched_rr_get_interval(pid_t pid, struct timespec64 *t) 9113 { 9114 struct task_struct *p; 9115 unsigned int time_slice; 9116 struct rq_flags rf; 9117 struct rq *rq; 9118 int retval; 9119 9120 if (pid < 0) 9121 return -EINVAL; 9122 9123 retval = -ESRCH; 9124 rcu_read_lock(); 9125 p = find_process_by_pid(pid); 9126 if (!p) 9127 goto out_unlock; 9128 9129 retval = security_task_getscheduler(p); 9130 if (retval) 9131 goto out_unlock; 9132 9133 rq = task_rq_lock(p, &rf); 9134 time_slice = 0; 9135 if (p->sched_class->get_rr_interval) 9136 time_slice = p->sched_class->get_rr_interval(rq, p); 9137 task_rq_unlock(rq, p, &rf); 9138 9139 rcu_read_unlock(); 9140 jiffies_to_timespec64(time_slice, t); 9141 return 0; 9142 9143 out_unlock: 9144 rcu_read_unlock(); 9145 return retval; 9146 } 9147 9148 /** 9149 * sys_sched_rr_get_interval - return the default timeslice of a process. 9150 * @pid: pid of the process. 9151 * @interval: userspace pointer to the timeslice value. 9152 * 9153 * this syscall writes the default timeslice value of a given process 9154 * into the user-space timespec buffer. A value of '0' means infinity. 9155 * 9156 * Return: On success, 0 and the timeslice is in @interval. Otherwise, 9157 * an error code. 9158 */ 9159 SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid, 9160 struct __kernel_timespec __user *, interval) 9161 { 9162 struct timespec64 t; 9163 int retval = sched_rr_get_interval(pid, &t); 9164 9165 if (retval == 0) 9166 retval = put_timespec64(&t, interval); 9167 9168 return retval; 9169 } 9170 9171 #ifdef CONFIG_COMPAT_32BIT_TIME 9172 SYSCALL_DEFINE2(sched_rr_get_interval_time32, pid_t, pid, 9173 struct old_timespec32 __user *, interval) 9174 { 9175 struct timespec64 t; 9176 int retval = sched_rr_get_interval(pid, &t); 9177 9178 if (retval == 0) 9179 retval = put_old_timespec32(&t, interval); 9180 return retval; 9181 } 9182 #endif 9183 9184 void sched_show_task(struct task_struct *p) 9185 { 9186 unsigned long free = 0; 9187 int ppid; 9188 9189 if (!try_get_task_stack(p)) 9190 return; 9191 9192 pr_info("task:%-15.15s state:%c", p->comm, task_state_to_char(p)); 9193 9194 if (task_is_running(p)) 9195 pr_cont(" running task "); 9196 #ifdef CONFIG_DEBUG_STACK_USAGE 9197 free = stack_not_used(p); 9198 #endif 9199 ppid = 0; 9200 rcu_read_lock(); 9201 if (pid_alive(p)) 9202 ppid = task_pid_nr(rcu_dereference(p->real_parent)); 9203 rcu_read_unlock(); 9204 pr_cont(" stack:%-5lu pid:%-5d ppid:%-6d flags:0x%08lx\n", 9205 free, task_pid_nr(p), ppid, 9206 read_task_thread_flags(p)); 9207 9208 print_worker_info(KERN_INFO, p); 9209 print_stop_info(KERN_INFO, p); 9210 show_stack(p, NULL, KERN_INFO); 9211 put_task_stack(p); 9212 } 9213 EXPORT_SYMBOL_GPL(sched_show_task); 9214 9215 static inline bool 9216 state_filter_match(unsigned long state_filter, struct task_struct *p) 9217 { 9218 unsigned int state = READ_ONCE(p->__state); 9219 9220 /* no filter, everything matches */ 9221 if (!state_filter) 9222 return true; 9223 9224 /* filter, but doesn't match */ 9225 if (!(state & state_filter)) 9226 return false; 9227 9228 /* 9229 * When looking for TASK_UNINTERRUPTIBLE skip TASK_IDLE (allows 9230 * TASK_KILLABLE). 9231 */ 9232 if (state_filter == TASK_UNINTERRUPTIBLE && (state & TASK_NOLOAD)) 9233 return false; 9234 9235 return true; 9236 } 9237 9238 9239 void show_state_filter(unsigned int state_filter) 9240 { 9241 struct task_struct *g, *p; 9242 9243 rcu_read_lock(); 9244 for_each_process_thread(g, p) { 9245 /* 9246 * reset the NMI-timeout, listing all files on a slow 9247 * console might take a lot of time: 9248 * Also, reset softlockup watchdogs on all CPUs, because 9249 * another CPU might be blocked waiting for us to process 9250 * an IPI. 9251 */ 9252 touch_nmi_watchdog(); 9253 touch_all_softlockup_watchdogs(); 9254 if (state_filter_match(state_filter, p)) 9255 sched_show_task(p); 9256 } 9257 9258 #ifdef CONFIG_SCHED_DEBUG 9259 if (!state_filter) 9260 sysrq_sched_debug_show(); 9261 #endif 9262 rcu_read_unlock(); 9263 /* 9264 * Only show locks if all tasks are dumped: 9265 */ 9266 if (!state_filter) 9267 debug_show_all_locks(); 9268 } 9269 9270 /** 9271 * init_idle - set up an idle thread for a given CPU 9272 * @idle: task in question 9273 * @cpu: CPU the idle task belongs to 9274 * 9275 * NOTE: this function does not set the idle thread's NEED_RESCHED 9276 * flag, to make booting more robust. 9277 */ 9278 void __init init_idle(struct task_struct *idle, int cpu) 9279 { 9280 #ifdef CONFIG_SMP 9281 struct affinity_context ac = (struct affinity_context) { 9282 .new_mask = cpumask_of(cpu), 9283 .flags = 0, 9284 }; 9285 #endif 9286 struct rq *rq = cpu_rq(cpu); 9287 unsigned long flags; 9288 9289 __sched_fork(0, idle); 9290 9291 raw_spin_lock_irqsave(&idle->pi_lock, flags); 9292 raw_spin_rq_lock(rq); 9293 9294 idle->__state = TASK_RUNNING; 9295 idle->se.exec_start = sched_clock(); 9296 /* 9297 * PF_KTHREAD should already be set at this point; regardless, make it 9298 * look like a proper per-CPU kthread. 9299 */ 9300 idle->flags |= PF_IDLE | PF_KTHREAD | PF_NO_SETAFFINITY; 9301 kthread_set_per_cpu(idle, cpu); 9302 9303 #ifdef CONFIG_SMP 9304 /* 9305 * It's possible that init_idle() gets called multiple times on a task, 9306 * in that case do_set_cpus_allowed() will not do the right thing. 9307 * 9308 * And since this is boot we can forgo the serialization. 9309 */ 9310 set_cpus_allowed_common(idle, &ac); 9311 #endif 9312 /* 9313 * We're having a chicken and egg problem, even though we are 9314 * holding rq->lock, the CPU isn't yet set to this CPU so the 9315 * lockdep check in task_group() will fail. 9316 * 9317 * Similar case to sched_fork(). / Alternatively we could 9318 * use task_rq_lock() here and obtain the other rq->lock. 9319 * 9320 * Silence PROVE_RCU 9321 */ 9322 rcu_read_lock(); 9323 __set_task_cpu(idle, cpu); 9324 rcu_read_unlock(); 9325 9326 rq->idle = idle; 9327 rcu_assign_pointer(rq->curr, idle); 9328 idle->on_rq = TASK_ON_RQ_QUEUED; 9329 #ifdef CONFIG_SMP 9330 idle->on_cpu = 1; 9331 #endif 9332 raw_spin_rq_unlock(rq); 9333 raw_spin_unlock_irqrestore(&idle->pi_lock, flags); 9334 9335 /* Set the preempt count _outside_ the spinlocks! */ 9336 init_idle_preempt_count(idle, cpu); 9337 9338 /* 9339 * The idle tasks have their own, simple scheduling class: 9340 */ 9341 idle->sched_class = &idle_sched_class; 9342 ftrace_graph_init_idle_task(idle, cpu); 9343 vtime_init_idle(idle, cpu); 9344 #ifdef CONFIG_SMP 9345 sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu); 9346 #endif 9347 } 9348 9349 #ifdef CONFIG_SMP 9350 9351 int cpuset_cpumask_can_shrink(const struct cpumask *cur, 9352 const struct cpumask *trial) 9353 { 9354 int ret = 1; 9355 9356 if (cpumask_empty(cur)) 9357 return ret; 9358 9359 ret = dl_cpuset_cpumask_can_shrink(cur, trial); 9360 9361 return ret; 9362 } 9363 9364 int task_can_attach(struct task_struct *p) 9365 { 9366 int ret = 0; 9367 9368 /* 9369 * Kthreads which disallow setaffinity shouldn't be moved 9370 * to a new cpuset; we don't want to change their CPU 9371 * affinity and isolating such threads by their set of 9372 * allowed nodes is unnecessary. Thus, cpusets are not 9373 * applicable for such threads. This prevents checking for 9374 * success of set_cpus_allowed_ptr() on all attached tasks 9375 * before cpus_mask may be changed. 9376 */ 9377 if (p->flags & PF_NO_SETAFFINITY) 9378 ret = -EINVAL; 9379 9380 return ret; 9381 } 9382 9383 bool sched_smp_initialized __read_mostly; 9384 9385 #ifdef CONFIG_NUMA_BALANCING 9386 /* Migrate current task p to target_cpu */ 9387 int migrate_task_to(struct task_struct *p, int target_cpu) 9388 { 9389 struct migration_arg arg = { p, target_cpu }; 9390 int curr_cpu = task_cpu(p); 9391 9392 if (curr_cpu == target_cpu) 9393 return 0; 9394 9395 if (!cpumask_test_cpu(target_cpu, p->cpus_ptr)) 9396 return -EINVAL; 9397 9398 /* TODO: This is not properly updating schedstats */ 9399 9400 trace_sched_move_numa(p, curr_cpu, target_cpu); 9401 return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg); 9402 } 9403 9404 /* 9405 * Requeue a task on a given node and accurately track the number of NUMA 9406 * tasks on the runqueues 9407 */ 9408 void sched_setnuma(struct task_struct *p, int nid) 9409 { 9410 bool queued, running; 9411 struct rq_flags rf; 9412 struct rq *rq; 9413 9414 rq = task_rq_lock(p, &rf); 9415 queued = task_on_rq_queued(p); 9416 running = task_current(rq, p); 9417 9418 if (queued) 9419 dequeue_task(rq, p, DEQUEUE_SAVE); 9420 if (running) 9421 put_prev_task(rq, p); 9422 9423 p->numa_preferred_nid = nid; 9424 9425 if (queued) 9426 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); 9427 if (running) 9428 set_next_task(rq, p); 9429 task_rq_unlock(rq, p, &rf); 9430 } 9431 #endif /* CONFIG_NUMA_BALANCING */ 9432 9433 #ifdef CONFIG_HOTPLUG_CPU 9434 /* 9435 * Ensure that the idle task is using init_mm right before its CPU goes 9436 * offline. 9437 */ 9438 void idle_task_exit(void) 9439 { 9440 struct mm_struct *mm = current->active_mm; 9441 9442 BUG_ON(cpu_online(smp_processor_id())); 9443 BUG_ON(current != this_rq()->idle); 9444 9445 if (mm != &init_mm) { 9446 switch_mm(mm, &init_mm, current); 9447 finish_arch_post_lock_switch(); 9448 } 9449 9450 /* finish_cpu(), as ran on the BP, will clean up the active_mm state */ 9451 } 9452 9453 static int __balance_push_cpu_stop(void *arg) 9454 { 9455 struct task_struct *p = arg; 9456 struct rq *rq = this_rq(); 9457 struct rq_flags rf; 9458 int cpu; 9459 9460 raw_spin_lock_irq(&p->pi_lock); 9461 rq_lock(rq, &rf); 9462 9463 update_rq_clock(rq); 9464 9465 if (task_rq(p) == rq && task_on_rq_queued(p)) { 9466 cpu = select_fallback_rq(rq->cpu, p); 9467 rq = __migrate_task(rq, &rf, p, cpu); 9468 } 9469 9470 rq_unlock(rq, &rf); 9471 raw_spin_unlock_irq(&p->pi_lock); 9472 9473 put_task_struct(p); 9474 9475 return 0; 9476 } 9477 9478 static DEFINE_PER_CPU(struct cpu_stop_work, push_work); 9479 9480 /* 9481 * Ensure we only run per-cpu kthreads once the CPU goes !active. 9482 * 9483 * This is enabled below SCHED_AP_ACTIVE; when !cpu_active(), but only 9484 * effective when the hotplug motion is down. 9485 */ 9486 static void balance_push(struct rq *rq) 9487 { 9488 struct task_struct *push_task = rq->curr; 9489 9490 lockdep_assert_rq_held(rq); 9491 9492 /* 9493 * Ensure the thing is persistent until balance_push_set(.on = false); 9494 */ 9495 rq->balance_callback = &balance_push_callback; 9496 9497 /* 9498 * Only active while going offline and when invoked on the outgoing 9499 * CPU. 9500 */ 9501 if (!cpu_dying(rq->cpu) || rq != this_rq()) 9502 return; 9503 9504 /* 9505 * Both the cpu-hotplug and stop task are in this case and are 9506 * required to complete the hotplug process. 9507 */ 9508 if (kthread_is_per_cpu(push_task) || 9509 is_migration_disabled(push_task)) { 9510 9511 /* 9512 * If this is the idle task on the outgoing CPU try to wake 9513 * up the hotplug control thread which might wait for the 9514 * last task to vanish. The rcuwait_active() check is 9515 * accurate here because the waiter is pinned on this CPU 9516 * and can't obviously be running in parallel. 9517 * 9518 * On RT kernels this also has to check whether there are 9519 * pinned and scheduled out tasks on the runqueue. They 9520 * need to leave the migrate disabled section first. 9521 */ 9522 if (!rq->nr_running && !rq_has_pinned_tasks(rq) && 9523 rcuwait_active(&rq->hotplug_wait)) { 9524 raw_spin_rq_unlock(rq); 9525 rcuwait_wake_up(&rq->hotplug_wait); 9526 raw_spin_rq_lock(rq); 9527 } 9528 return; 9529 } 9530 9531 get_task_struct(push_task); 9532 /* 9533 * Temporarily drop rq->lock such that we can wake-up the stop task. 9534 * Both preemption and IRQs are still disabled. 9535 */ 9536 raw_spin_rq_unlock(rq); 9537 stop_one_cpu_nowait(rq->cpu, __balance_push_cpu_stop, push_task, 9538 this_cpu_ptr(&push_work)); 9539 /* 9540 * At this point need_resched() is true and we'll take the loop in 9541 * schedule(). The next pick is obviously going to be the stop task 9542 * which kthread_is_per_cpu() and will push this task away. 9543 */ 9544 raw_spin_rq_lock(rq); 9545 } 9546 9547 static void balance_push_set(int cpu, bool on) 9548 { 9549 struct rq *rq = cpu_rq(cpu); 9550 struct rq_flags rf; 9551 9552 rq_lock_irqsave(rq, &rf); 9553 if (on) { 9554 WARN_ON_ONCE(rq->balance_callback); 9555 rq->balance_callback = &balance_push_callback; 9556 } else if (rq->balance_callback == &balance_push_callback) { 9557 rq->balance_callback = NULL; 9558 } 9559 rq_unlock_irqrestore(rq, &rf); 9560 } 9561 9562 /* 9563 * Invoked from a CPUs hotplug control thread after the CPU has been marked 9564 * inactive. All tasks which are not per CPU kernel threads are either 9565 * pushed off this CPU now via balance_push() or placed on a different CPU 9566 * during wakeup. Wait until the CPU is quiescent. 9567 */ 9568 static void balance_hotplug_wait(void) 9569 { 9570 struct rq *rq = this_rq(); 9571 9572 rcuwait_wait_event(&rq->hotplug_wait, 9573 rq->nr_running == 1 && !rq_has_pinned_tasks(rq), 9574 TASK_UNINTERRUPTIBLE); 9575 } 9576 9577 #else 9578 9579 static inline void balance_push(struct rq *rq) 9580 { 9581 } 9582 9583 static inline void balance_push_set(int cpu, bool on) 9584 { 9585 } 9586 9587 static inline void balance_hotplug_wait(void) 9588 { 9589 } 9590 9591 #endif /* CONFIG_HOTPLUG_CPU */ 9592 9593 void set_rq_online(struct rq *rq) 9594 { 9595 if (!rq->online) { 9596 const struct sched_class *class; 9597 9598 cpumask_set_cpu(rq->cpu, rq->rd->online); 9599 rq->online = 1; 9600 9601 for_each_class(class) { 9602 if (class->rq_online) 9603 class->rq_online(rq); 9604 } 9605 } 9606 } 9607 9608 void set_rq_offline(struct rq *rq) 9609 { 9610 if (rq->online) { 9611 const struct sched_class *class; 9612 9613 update_rq_clock(rq); 9614 for_each_class(class) { 9615 if (class->rq_offline) 9616 class->rq_offline(rq); 9617 } 9618 9619 cpumask_clear_cpu(rq->cpu, rq->rd->online); 9620 rq->online = 0; 9621 } 9622 } 9623 9624 /* 9625 * used to mark begin/end of suspend/resume: 9626 */ 9627 static int num_cpus_frozen; 9628 9629 /* 9630 * Update cpusets according to cpu_active mask. If cpusets are 9631 * disabled, cpuset_update_active_cpus() becomes a simple wrapper 9632 * around partition_sched_domains(). 9633 * 9634 * If we come here as part of a suspend/resume, don't touch cpusets because we 9635 * want to restore it back to its original state upon resume anyway. 9636 */ 9637 static void cpuset_cpu_active(void) 9638 { 9639 if (cpuhp_tasks_frozen) { 9640 /* 9641 * num_cpus_frozen tracks how many CPUs are involved in suspend 9642 * resume sequence. As long as this is not the last online 9643 * operation in the resume sequence, just build a single sched 9644 * domain, ignoring cpusets. 9645 */ 9646 partition_sched_domains(1, NULL, NULL); 9647 if (--num_cpus_frozen) 9648 return; 9649 /* 9650 * This is the last CPU online operation. So fall through and 9651 * restore the original sched domains by considering the 9652 * cpuset configurations. 9653 */ 9654 cpuset_force_rebuild(); 9655 } 9656 cpuset_update_active_cpus(); 9657 } 9658 9659 static int cpuset_cpu_inactive(unsigned int cpu) 9660 { 9661 if (!cpuhp_tasks_frozen) { 9662 int ret = dl_bw_check_overflow(cpu); 9663 9664 if (ret) 9665 return ret; 9666 cpuset_update_active_cpus(); 9667 } else { 9668 num_cpus_frozen++; 9669 partition_sched_domains(1, NULL, NULL); 9670 } 9671 return 0; 9672 } 9673 9674 int sched_cpu_activate(unsigned int cpu) 9675 { 9676 struct rq *rq = cpu_rq(cpu); 9677 struct rq_flags rf; 9678 9679 /* 9680 * Clear the balance_push callback and prepare to schedule 9681 * regular tasks. 9682 */ 9683 balance_push_set(cpu, false); 9684 9685 #ifdef CONFIG_SCHED_SMT 9686 /* 9687 * When going up, increment the number of cores with SMT present. 9688 */ 9689 if (cpumask_weight(cpu_smt_mask(cpu)) == 2) 9690 static_branch_inc_cpuslocked(&sched_smt_present); 9691 #endif 9692 set_cpu_active(cpu, true); 9693 9694 if (sched_smp_initialized) { 9695 sched_update_numa(cpu, true); 9696 sched_domains_numa_masks_set(cpu); 9697 cpuset_cpu_active(); 9698 } 9699 9700 /* 9701 * Put the rq online, if not already. This happens: 9702 * 9703 * 1) In the early boot process, because we build the real domains 9704 * after all CPUs have been brought up. 9705 * 9706 * 2) At runtime, if cpuset_cpu_active() fails to rebuild the 9707 * domains. 9708 */ 9709 rq_lock_irqsave(rq, &rf); 9710 if (rq->rd) { 9711 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); 9712 set_rq_online(rq); 9713 } 9714 rq_unlock_irqrestore(rq, &rf); 9715 9716 return 0; 9717 } 9718 9719 int sched_cpu_deactivate(unsigned int cpu) 9720 { 9721 struct rq *rq = cpu_rq(cpu); 9722 struct rq_flags rf; 9723 int ret; 9724 9725 /* 9726 * Remove CPU from nohz.idle_cpus_mask to prevent participating in 9727 * load balancing when not active 9728 */ 9729 nohz_balance_exit_idle(rq); 9730 9731 set_cpu_active(cpu, false); 9732 9733 /* 9734 * From this point forward, this CPU will refuse to run any task that 9735 * is not: migrate_disable() or KTHREAD_IS_PER_CPU, and will actively 9736 * push those tasks away until this gets cleared, see 9737 * sched_cpu_dying(). 9738 */ 9739 balance_push_set(cpu, true); 9740 9741 /* 9742 * We've cleared cpu_active_mask / set balance_push, wait for all 9743 * preempt-disabled and RCU users of this state to go away such that 9744 * all new such users will observe it. 9745 * 9746 * Specifically, we rely on ttwu to no longer target this CPU, see 9747 * ttwu_queue_cond() and is_cpu_allowed(). 9748 * 9749 * Do sync before park smpboot threads to take care the rcu boost case. 9750 */ 9751 synchronize_rcu(); 9752 9753 rq_lock_irqsave(rq, &rf); 9754 if (rq->rd) { 9755 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); 9756 set_rq_offline(rq); 9757 } 9758 rq_unlock_irqrestore(rq, &rf); 9759 9760 #ifdef CONFIG_SCHED_SMT 9761 /* 9762 * When going down, decrement the number of cores with SMT present. 9763 */ 9764 if (cpumask_weight(cpu_smt_mask(cpu)) == 2) 9765 static_branch_dec_cpuslocked(&sched_smt_present); 9766 9767 sched_core_cpu_deactivate(cpu); 9768 #endif 9769 9770 if (!sched_smp_initialized) 9771 return 0; 9772 9773 sched_update_numa(cpu, false); 9774 ret = cpuset_cpu_inactive(cpu); 9775 if (ret) { 9776 balance_push_set(cpu, false); 9777 set_cpu_active(cpu, true); 9778 sched_update_numa(cpu, true); 9779 return ret; 9780 } 9781 sched_domains_numa_masks_clear(cpu); 9782 return 0; 9783 } 9784 9785 static void sched_rq_cpu_starting(unsigned int cpu) 9786 { 9787 struct rq *rq = cpu_rq(cpu); 9788 9789 rq->calc_load_update = calc_load_update; 9790 update_max_interval(); 9791 } 9792 9793 int sched_cpu_starting(unsigned int cpu) 9794 { 9795 sched_core_cpu_starting(cpu); 9796 sched_rq_cpu_starting(cpu); 9797 sched_tick_start(cpu); 9798 return 0; 9799 } 9800 9801 #ifdef CONFIG_HOTPLUG_CPU 9802 9803 /* 9804 * Invoked immediately before the stopper thread is invoked to bring the 9805 * CPU down completely. At this point all per CPU kthreads except the 9806 * hotplug thread (current) and the stopper thread (inactive) have been 9807 * either parked or have been unbound from the outgoing CPU. Ensure that 9808 * any of those which might be on the way out are gone. 9809 * 9810 * If after this point a bound task is being woken on this CPU then the 9811 * responsible hotplug callback has failed to do it's job. 9812 * sched_cpu_dying() will catch it with the appropriate fireworks. 9813 */ 9814 int sched_cpu_wait_empty(unsigned int cpu) 9815 { 9816 balance_hotplug_wait(); 9817 return 0; 9818 } 9819 9820 /* 9821 * Since this CPU is going 'away' for a while, fold any nr_active delta we 9822 * might have. Called from the CPU stopper task after ensuring that the 9823 * stopper is the last running task on the CPU, so nr_active count is 9824 * stable. We need to take the teardown thread which is calling this into 9825 * account, so we hand in adjust = 1 to the load calculation. 9826 * 9827 * Also see the comment "Global load-average calculations". 9828 */ 9829 static void calc_load_migrate(struct rq *rq) 9830 { 9831 long delta = calc_load_fold_active(rq, 1); 9832 9833 if (delta) 9834 atomic_long_add(delta, &calc_load_tasks); 9835 } 9836 9837 static void dump_rq_tasks(struct rq *rq, const char *loglvl) 9838 { 9839 struct task_struct *g, *p; 9840 int cpu = cpu_of(rq); 9841 9842 lockdep_assert_rq_held(rq); 9843 9844 printk("%sCPU%d enqueued tasks (%u total):\n", loglvl, cpu, rq->nr_running); 9845 for_each_process_thread(g, p) { 9846 if (task_cpu(p) != cpu) 9847 continue; 9848 9849 if (!task_on_rq_queued(p)) 9850 continue; 9851 9852 printk("%s\tpid: %d, name: %s\n", loglvl, p->pid, p->comm); 9853 } 9854 } 9855 9856 int sched_cpu_dying(unsigned int cpu) 9857 { 9858 struct rq *rq = cpu_rq(cpu); 9859 struct rq_flags rf; 9860 9861 /* Handle pending wakeups and then migrate everything off */ 9862 sched_tick_stop(cpu); 9863 9864 rq_lock_irqsave(rq, &rf); 9865 if (rq->nr_running != 1 || rq_has_pinned_tasks(rq)) { 9866 WARN(true, "Dying CPU not properly vacated!"); 9867 dump_rq_tasks(rq, KERN_WARNING); 9868 } 9869 rq_unlock_irqrestore(rq, &rf); 9870 9871 calc_load_migrate(rq); 9872 update_max_interval(); 9873 hrtick_clear(rq); 9874 sched_core_cpu_dying(cpu); 9875 return 0; 9876 } 9877 #endif 9878 9879 void __init sched_init_smp(void) 9880 { 9881 sched_init_numa(NUMA_NO_NODE); 9882 9883 /* 9884 * There's no userspace yet to cause hotplug operations; hence all the 9885 * CPU masks are stable and all blatant races in the below code cannot 9886 * happen. 9887 */ 9888 mutex_lock(&sched_domains_mutex); 9889 sched_init_domains(cpu_active_mask); 9890 mutex_unlock(&sched_domains_mutex); 9891 9892 /* Move init over to a non-isolated CPU */ 9893 if (set_cpus_allowed_ptr(current, housekeeping_cpumask(HK_TYPE_DOMAIN)) < 0) 9894 BUG(); 9895 current->flags &= ~PF_NO_SETAFFINITY; 9896 sched_init_granularity(); 9897 9898 init_sched_rt_class(); 9899 init_sched_dl_class(); 9900 9901 sched_smp_initialized = true; 9902 } 9903 9904 static int __init migration_init(void) 9905 { 9906 sched_cpu_starting(smp_processor_id()); 9907 return 0; 9908 } 9909 early_initcall(migration_init); 9910 9911 #else 9912 void __init sched_init_smp(void) 9913 { 9914 sched_init_granularity(); 9915 } 9916 #endif /* CONFIG_SMP */ 9917 9918 int in_sched_functions(unsigned long addr) 9919 { 9920 return in_lock_functions(addr) || 9921 (addr >= (unsigned long)__sched_text_start 9922 && addr < (unsigned long)__sched_text_end); 9923 } 9924 9925 #ifdef CONFIG_CGROUP_SCHED 9926 /* 9927 * Default task group. 9928 * Every task in system belongs to this group at bootup. 9929 */ 9930 struct task_group root_task_group; 9931 LIST_HEAD(task_groups); 9932 9933 /* Cacheline aligned slab cache for task_group */ 9934 static struct kmem_cache *task_group_cache __read_mostly; 9935 #endif 9936 9937 void __init sched_init(void) 9938 { 9939 unsigned long ptr = 0; 9940 int i; 9941 9942 /* Make sure the linker didn't screw up */ 9943 BUG_ON(&idle_sched_class != &fair_sched_class + 1 || 9944 &fair_sched_class != &rt_sched_class + 1 || 9945 &rt_sched_class != &dl_sched_class + 1); 9946 #ifdef CONFIG_SMP 9947 BUG_ON(&dl_sched_class != &stop_sched_class + 1); 9948 #endif 9949 9950 wait_bit_init(); 9951 9952 #ifdef CONFIG_FAIR_GROUP_SCHED 9953 ptr += 2 * nr_cpu_ids * sizeof(void **); 9954 #endif 9955 #ifdef CONFIG_RT_GROUP_SCHED 9956 ptr += 2 * nr_cpu_ids * sizeof(void **); 9957 #endif 9958 if (ptr) { 9959 ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT); 9960 9961 #ifdef CONFIG_FAIR_GROUP_SCHED 9962 root_task_group.se = (struct sched_entity **)ptr; 9963 ptr += nr_cpu_ids * sizeof(void **); 9964 9965 root_task_group.cfs_rq = (struct cfs_rq **)ptr; 9966 ptr += nr_cpu_ids * sizeof(void **); 9967 9968 root_task_group.shares = ROOT_TASK_GROUP_LOAD; 9969 init_cfs_bandwidth(&root_task_group.cfs_bandwidth, NULL); 9970 #endif /* CONFIG_FAIR_GROUP_SCHED */ 9971 #ifdef CONFIG_RT_GROUP_SCHED 9972 root_task_group.rt_se = (struct sched_rt_entity **)ptr; 9973 ptr += nr_cpu_ids * sizeof(void **); 9974 9975 root_task_group.rt_rq = (struct rt_rq **)ptr; 9976 ptr += nr_cpu_ids * sizeof(void **); 9977 9978 #endif /* CONFIG_RT_GROUP_SCHED */ 9979 } 9980 9981 init_rt_bandwidth(&def_rt_bandwidth, global_rt_period(), global_rt_runtime()); 9982 9983 #ifdef CONFIG_SMP 9984 init_defrootdomain(); 9985 #endif 9986 9987 #ifdef CONFIG_RT_GROUP_SCHED 9988 init_rt_bandwidth(&root_task_group.rt_bandwidth, 9989 global_rt_period(), global_rt_runtime()); 9990 #endif /* CONFIG_RT_GROUP_SCHED */ 9991 9992 #ifdef CONFIG_CGROUP_SCHED 9993 task_group_cache = KMEM_CACHE(task_group, 0); 9994 9995 list_add(&root_task_group.list, &task_groups); 9996 INIT_LIST_HEAD(&root_task_group.children); 9997 INIT_LIST_HEAD(&root_task_group.siblings); 9998 autogroup_init(&init_task); 9999 #endif /* CONFIG_CGROUP_SCHED */ 10000 10001 for_each_possible_cpu(i) { 10002 struct rq *rq; 10003 10004 rq = cpu_rq(i); 10005 raw_spin_lock_init(&rq->__lock); 10006 rq->nr_running = 0; 10007 rq->calc_load_active = 0; 10008 rq->calc_load_update = jiffies + LOAD_FREQ; 10009 init_cfs_rq(&rq->cfs); 10010 init_rt_rq(&rq->rt); 10011 init_dl_rq(&rq->dl); 10012 #ifdef CONFIG_FAIR_GROUP_SCHED 10013 INIT_LIST_HEAD(&rq->leaf_cfs_rq_list); 10014 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 10015 /* 10016 * How much CPU bandwidth does root_task_group get? 10017 * 10018 * In case of task-groups formed thr' the cgroup filesystem, it 10019 * gets 100% of the CPU resources in the system. This overall 10020 * system CPU resource is divided among the tasks of 10021 * root_task_group and its child task-groups in a fair manner, 10022 * based on each entity's (task or task-group's) weight 10023 * (se->load.weight). 10024 * 10025 * In other words, if root_task_group has 10 tasks of weight 10026 * 1024) and two child groups A0 and A1 (of weight 1024 each), 10027 * then A0's share of the CPU resource is: 10028 * 10029 * A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33% 10030 * 10031 * We achieve this by letting root_task_group's tasks sit 10032 * directly in rq->cfs (i.e root_task_group->se[] = NULL). 10033 */ 10034 init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL); 10035 #endif /* CONFIG_FAIR_GROUP_SCHED */ 10036 10037 rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime; 10038 #ifdef CONFIG_RT_GROUP_SCHED 10039 init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL); 10040 #endif 10041 #ifdef CONFIG_SMP 10042 rq->sd = NULL; 10043 rq->rd = NULL; 10044 rq->cpu_capacity = rq->cpu_capacity_orig = SCHED_CAPACITY_SCALE; 10045 rq->balance_callback = &balance_push_callback; 10046 rq->active_balance = 0; 10047 rq->next_balance = jiffies; 10048 rq->push_cpu = 0; 10049 rq->cpu = i; 10050 rq->online = 0; 10051 rq->idle_stamp = 0; 10052 rq->avg_idle = 2*sysctl_sched_migration_cost; 10053 rq->wake_stamp = jiffies; 10054 rq->wake_avg_idle = rq->avg_idle; 10055 rq->max_idle_balance_cost = sysctl_sched_migration_cost; 10056 10057 INIT_LIST_HEAD(&rq->cfs_tasks); 10058 10059 rq_attach_root(rq, &def_root_domain); 10060 #ifdef CONFIG_NO_HZ_COMMON 10061 rq->last_blocked_load_update_tick = jiffies; 10062 atomic_set(&rq->nohz_flags, 0); 10063 10064 INIT_CSD(&rq->nohz_csd, nohz_csd_func, rq); 10065 #endif 10066 #ifdef CONFIG_HOTPLUG_CPU 10067 rcuwait_init(&rq->hotplug_wait); 10068 #endif 10069 #endif /* CONFIG_SMP */ 10070 hrtick_rq_init(rq); 10071 atomic_set(&rq->nr_iowait, 0); 10072 10073 #ifdef CONFIG_SCHED_CORE 10074 rq->core = rq; 10075 rq->core_pick = NULL; 10076 rq->core_enabled = 0; 10077 rq->core_tree = RB_ROOT; 10078 rq->core_forceidle_count = 0; 10079 rq->core_forceidle_occupation = 0; 10080 rq->core_forceidle_start = 0; 10081 10082 rq->core_cookie = 0UL; 10083 #endif 10084 zalloc_cpumask_var_node(&rq->scratch_mask, GFP_KERNEL, cpu_to_node(i)); 10085 } 10086 10087 set_load_weight(&init_task, false); 10088 10089 /* 10090 * The boot idle thread does lazy MMU switching as well: 10091 */ 10092 mmgrab_lazy_tlb(&init_mm); 10093 enter_lazy_tlb(&init_mm, current); 10094 10095 /* 10096 * The idle task doesn't need the kthread struct to function, but it 10097 * is dressed up as a per-CPU kthread and thus needs to play the part 10098 * if we want to avoid special-casing it in code that deals with per-CPU 10099 * kthreads. 10100 */ 10101 WARN_ON(!set_kthread_struct(current)); 10102 10103 /* 10104 * Make us the idle thread. Technically, schedule() should not be 10105 * called from this thread, however somewhere below it might be, 10106 * but because we are the idle thread, we just pick up running again 10107 * when this runqueue becomes "idle". 10108 */ 10109 init_idle(current, smp_processor_id()); 10110 10111 calc_load_update = jiffies + LOAD_FREQ; 10112 10113 #ifdef CONFIG_SMP 10114 idle_thread_set_boot_cpu(); 10115 balance_push_set(smp_processor_id(), false); 10116 #endif 10117 init_sched_fair_class(); 10118 10119 psi_init(); 10120 10121 init_uclamp(); 10122 10123 preempt_dynamic_init(); 10124 10125 scheduler_running = 1; 10126 } 10127 10128 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP 10129 10130 void __might_sleep(const char *file, int line) 10131 { 10132 unsigned int state = get_current_state(); 10133 /* 10134 * Blocking primitives will set (and therefore destroy) current->state, 10135 * since we will exit with TASK_RUNNING make sure we enter with it, 10136 * otherwise we will destroy state. 10137 */ 10138 WARN_ONCE(state != TASK_RUNNING && current->task_state_change, 10139 "do not call blocking ops when !TASK_RUNNING; " 10140 "state=%x set at [<%p>] %pS\n", state, 10141 (void *)current->task_state_change, 10142 (void *)current->task_state_change); 10143 10144 __might_resched(file, line, 0); 10145 } 10146 EXPORT_SYMBOL(__might_sleep); 10147 10148 static void print_preempt_disable_ip(int preempt_offset, unsigned long ip) 10149 { 10150 if (!IS_ENABLED(CONFIG_DEBUG_PREEMPT)) 10151 return; 10152 10153 if (preempt_count() == preempt_offset) 10154 return; 10155 10156 pr_err("Preemption disabled at:"); 10157 print_ip_sym(KERN_ERR, ip); 10158 } 10159 10160 static inline bool resched_offsets_ok(unsigned int offsets) 10161 { 10162 unsigned int nested = preempt_count(); 10163 10164 nested += rcu_preempt_depth() << MIGHT_RESCHED_RCU_SHIFT; 10165 10166 return nested == offsets; 10167 } 10168 10169 void __might_resched(const char *file, int line, unsigned int offsets) 10170 { 10171 /* Ratelimiting timestamp: */ 10172 static unsigned long prev_jiffy; 10173 10174 unsigned long preempt_disable_ip; 10175 10176 /* WARN_ON_ONCE() by default, no rate limit required: */ 10177 rcu_sleep_check(); 10178 10179 if ((resched_offsets_ok(offsets) && !irqs_disabled() && 10180 !is_idle_task(current) && !current->non_block_count) || 10181 system_state == SYSTEM_BOOTING || system_state > SYSTEM_RUNNING || 10182 oops_in_progress) 10183 return; 10184 10185 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) 10186 return; 10187 prev_jiffy = jiffies; 10188 10189 /* Save this before calling printk(), since that will clobber it: */ 10190 preempt_disable_ip = get_preempt_disable_ip(current); 10191 10192 pr_err("BUG: sleeping function called from invalid context at %s:%d\n", 10193 file, line); 10194 pr_err("in_atomic(): %d, irqs_disabled(): %d, non_block: %d, pid: %d, name: %s\n", 10195 in_atomic(), irqs_disabled(), current->non_block_count, 10196 current->pid, current->comm); 10197 pr_err("preempt_count: %x, expected: %x\n", preempt_count(), 10198 offsets & MIGHT_RESCHED_PREEMPT_MASK); 10199 10200 if (IS_ENABLED(CONFIG_PREEMPT_RCU)) { 10201 pr_err("RCU nest depth: %d, expected: %u\n", 10202 rcu_preempt_depth(), offsets >> MIGHT_RESCHED_RCU_SHIFT); 10203 } 10204 10205 if (task_stack_end_corrupted(current)) 10206 pr_emerg("Thread overran stack, or stack corrupted\n"); 10207 10208 debug_show_held_locks(current); 10209 if (irqs_disabled()) 10210 print_irqtrace_events(current); 10211 10212 print_preempt_disable_ip(offsets & MIGHT_RESCHED_PREEMPT_MASK, 10213 preempt_disable_ip); 10214 10215 dump_stack(); 10216 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 10217 } 10218 EXPORT_SYMBOL(__might_resched); 10219 10220 void __cant_sleep(const char *file, int line, int preempt_offset) 10221 { 10222 static unsigned long prev_jiffy; 10223 10224 if (irqs_disabled()) 10225 return; 10226 10227 if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) 10228 return; 10229 10230 if (preempt_count() > preempt_offset) 10231 return; 10232 10233 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) 10234 return; 10235 prev_jiffy = jiffies; 10236 10237 printk(KERN_ERR "BUG: assuming atomic context at %s:%d\n", file, line); 10238 printk(KERN_ERR "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n", 10239 in_atomic(), irqs_disabled(), 10240 current->pid, current->comm); 10241 10242 debug_show_held_locks(current); 10243 dump_stack(); 10244 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 10245 } 10246 EXPORT_SYMBOL_GPL(__cant_sleep); 10247 10248 #ifdef CONFIG_SMP 10249 void __cant_migrate(const char *file, int line) 10250 { 10251 static unsigned long prev_jiffy; 10252 10253 if (irqs_disabled()) 10254 return; 10255 10256 if (is_migration_disabled(current)) 10257 return; 10258 10259 if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) 10260 return; 10261 10262 if (preempt_count() > 0) 10263 return; 10264 10265 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) 10266 return; 10267 prev_jiffy = jiffies; 10268 10269 pr_err("BUG: assuming non migratable context at %s:%d\n", file, line); 10270 pr_err("in_atomic(): %d, irqs_disabled(): %d, migration_disabled() %u pid: %d, name: %s\n", 10271 in_atomic(), irqs_disabled(), is_migration_disabled(current), 10272 current->pid, current->comm); 10273 10274 debug_show_held_locks(current); 10275 dump_stack(); 10276 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 10277 } 10278 EXPORT_SYMBOL_GPL(__cant_migrate); 10279 #endif 10280 #endif 10281 10282 #ifdef CONFIG_MAGIC_SYSRQ 10283 void normalize_rt_tasks(void) 10284 { 10285 struct task_struct *g, *p; 10286 struct sched_attr attr = { 10287 .sched_policy = SCHED_NORMAL, 10288 }; 10289 10290 read_lock(&tasklist_lock); 10291 for_each_process_thread(g, p) { 10292 /* 10293 * Only normalize user tasks: 10294 */ 10295 if (p->flags & PF_KTHREAD) 10296 continue; 10297 10298 p->se.exec_start = 0; 10299 schedstat_set(p->stats.wait_start, 0); 10300 schedstat_set(p->stats.sleep_start, 0); 10301 schedstat_set(p->stats.block_start, 0); 10302 10303 if (!dl_task(p) && !rt_task(p)) { 10304 /* 10305 * Renice negative nice level userspace 10306 * tasks back to 0: 10307 */ 10308 if (task_nice(p) < 0) 10309 set_user_nice(p, 0); 10310 continue; 10311 } 10312 10313 __sched_setscheduler(p, &attr, false, false); 10314 } 10315 read_unlock(&tasklist_lock); 10316 } 10317 10318 #endif /* CONFIG_MAGIC_SYSRQ */ 10319 10320 #if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) 10321 /* 10322 * These functions are only useful for the IA64 MCA handling, or kdb. 10323 * 10324 * They can only be called when the whole system has been 10325 * stopped - every CPU needs to be quiescent, and no scheduling 10326 * activity can take place. Using them for anything else would 10327 * be a serious bug, and as a result, they aren't even visible 10328 * under any other configuration. 10329 */ 10330 10331 /** 10332 * curr_task - return the current task for a given CPU. 10333 * @cpu: the processor in question. 10334 * 10335 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! 10336 * 10337 * Return: The current task for @cpu. 10338 */ 10339 struct task_struct *curr_task(int cpu) 10340 { 10341 return cpu_curr(cpu); 10342 } 10343 10344 #endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */ 10345 10346 #ifdef CONFIG_IA64 10347 /** 10348 * ia64_set_curr_task - set the current task for a given CPU. 10349 * @cpu: the processor in question. 10350 * @p: the task pointer to set. 10351 * 10352 * Description: This function must only be used when non-maskable interrupts 10353 * are serviced on a separate stack. It allows the architecture to switch the 10354 * notion of the current task on a CPU in a non-blocking manner. This function 10355 * must be called with all CPU's synchronized, and interrupts disabled, the 10356 * and caller must save the original value of the current task (see 10357 * curr_task() above) and restore that value before reenabling interrupts and 10358 * re-starting the system. 10359 * 10360 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! 10361 */ 10362 void ia64_set_curr_task(int cpu, struct task_struct *p) 10363 { 10364 cpu_curr(cpu) = p; 10365 } 10366 10367 #endif 10368 10369 #ifdef CONFIG_CGROUP_SCHED 10370 /* task_group_lock serializes the addition/removal of task groups */ 10371 static DEFINE_SPINLOCK(task_group_lock); 10372 10373 static inline void alloc_uclamp_sched_group(struct task_group *tg, 10374 struct task_group *parent) 10375 { 10376 #ifdef CONFIG_UCLAMP_TASK_GROUP 10377 enum uclamp_id clamp_id; 10378 10379 for_each_clamp_id(clamp_id) { 10380 uclamp_se_set(&tg->uclamp_req[clamp_id], 10381 uclamp_none(clamp_id), false); 10382 tg->uclamp[clamp_id] = parent->uclamp[clamp_id]; 10383 } 10384 #endif 10385 } 10386 10387 static void sched_free_group(struct task_group *tg) 10388 { 10389 free_fair_sched_group(tg); 10390 free_rt_sched_group(tg); 10391 autogroup_free(tg); 10392 kmem_cache_free(task_group_cache, tg); 10393 } 10394 10395 static void sched_free_group_rcu(struct rcu_head *rcu) 10396 { 10397 sched_free_group(container_of(rcu, struct task_group, rcu)); 10398 } 10399 10400 static void sched_unregister_group(struct task_group *tg) 10401 { 10402 unregister_fair_sched_group(tg); 10403 unregister_rt_sched_group(tg); 10404 /* 10405 * We have to wait for yet another RCU grace period to expire, as 10406 * print_cfs_stats() might run concurrently. 10407 */ 10408 call_rcu(&tg->rcu, sched_free_group_rcu); 10409 } 10410 10411 /* allocate runqueue etc for a new task group */ 10412 struct task_group *sched_create_group(struct task_group *parent) 10413 { 10414 struct task_group *tg; 10415 10416 tg = kmem_cache_alloc(task_group_cache, GFP_KERNEL | __GFP_ZERO); 10417 if (!tg) 10418 return ERR_PTR(-ENOMEM); 10419 10420 if (!alloc_fair_sched_group(tg, parent)) 10421 goto err; 10422 10423 if (!alloc_rt_sched_group(tg, parent)) 10424 goto err; 10425 10426 alloc_uclamp_sched_group(tg, parent); 10427 10428 return tg; 10429 10430 err: 10431 sched_free_group(tg); 10432 return ERR_PTR(-ENOMEM); 10433 } 10434 10435 void sched_online_group(struct task_group *tg, struct task_group *parent) 10436 { 10437 unsigned long flags; 10438 10439 spin_lock_irqsave(&task_group_lock, flags); 10440 list_add_rcu(&tg->list, &task_groups); 10441 10442 /* Root should already exist: */ 10443 WARN_ON(!parent); 10444 10445 tg->parent = parent; 10446 INIT_LIST_HEAD(&tg->children); 10447 list_add_rcu(&tg->siblings, &parent->children); 10448 spin_unlock_irqrestore(&task_group_lock, flags); 10449 10450 online_fair_sched_group(tg); 10451 } 10452 10453 /* rcu callback to free various structures associated with a task group */ 10454 static void sched_unregister_group_rcu(struct rcu_head *rhp) 10455 { 10456 /* Now it should be safe to free those cfs_rqs: */ 10457 sched_unregister_group(container_of(rhp, struct task_group, rcu)); 10458 } 10459 10460 void sched_destroy_group(struct task_group *tg) 10461 { 10462 /* Wait for possible concurrent references to cfs_rqs complete: */ 10463 call_rcu(&tg->rcu, sched_unregister_group_rcu); 10464 } 10465 10466 void sched_release_group(struct task_group *tg) 10467 { 10468 unsigned long flags; 10469 10470 /* 10471 * Unlink first, to avoid walk_tg_tree_from() from finding us (via 10472 * sched_cfs_period_timer()). 10473 * 10474 * For this to be effective, we have to wait for all pending users of 10475 * this task group to leave their RCU critical section to ensure no new 10476 * user will see our dying task group any more. Specifically ensure 10477 * that tg_unthrottle_up() won't add decayed cfs_rq's to it. 10478 * 10479 * We therefore defer calling unregister_fair_sched_group() to 10480 * sched_unregister_group() which is guarantied to get called only after the 10481 * current RCU grace period has expired. 10482 */ 10483 spin_lock_irqsave(&task_group_lock, flags); 10484 list_del_rcu(&tg->list); 10485 list_del_rcu(&tg->siblings); 10486 spin_unlock_irqrestore(&task_group_lock, flags); 10487 } 10488 10489 static struct task_group *sched_get_task_group(struct task_struct *tsk) 10490 { 10491 struct task_group *tg; 10492 10493 /* 10494 * All callers are synchronized by task_rq_lock(); we do not use RCU 10495 * which is pointless here. Thus, we pass "true" to task_css_check() 10496 * to prevent lockdep warnings. 10497 */ 10498 tg = container_of(task_css_check(tsk, cpu_cgrp_id, true), 10499 struct task_group, css); 10500 tg = autogroup_task_group(tsk, tg); 10501 10502 return tg; 10503 } 10504 10505 static void sched_change_group(struct task_struct *tsk, struct task_group *group) 10506 { 10507 tsk->sched_task_group = group; 10508 10509 #ifdef CONFIG_FAIR_GROUP_SCHED 10510 if (tsk->sched_class->task_change_group) 10511 tsk->sched_class->task_change_group(tsk); 10512 else 10513 #endif 10514 set_task_rq(tsk, task_cpu(tsk)); 10515 } 10516 10517 /* 10518 * Change task's runqueue when it moves between groups. 10519 * 10520 * The caller of this function should have put the task in its new group by 10521 * now. This function just updates tsk->se.cfs_rq and tsk->se.parent to reflect 10522 * its new group. 10523 */ 10524 void sched_move_task(struct task_struct *tsk) 10525 { 10526 int queued, running, queue_flags = 10527 DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; 10528 struct task_group *group; 10529 struct rq_flags rf; 10530 struct rq *rq; 10531 10532 rq = task_rq_lock(tsk, &rf); 10533 /* 10534 * Esp. with SCHED_AUTOGROUP enabled it is possible to get superfluous 10535 * group changes. 10536 */ 10537 group = sched_get_task_group(tsk); 10538 if (group == tsk->sched_task_group) 10539 goto unlock; 10540 10541 update_rq_clock(rq); 10542 10543 running = task_current(rq, tsk); 10544 queued = task_on_rq_queued(tsk); 10545 10546 if (queued) 10547 dequeue_task(rq, tsk, queue_flags); 10548 if (running) 10549 put_prev_task(rq, tsk); 10550 10551 sched_change_group(tsk, group); 10552 10553 if (queued) 10554 enqueue_task(rq, tsk, queue_flags); 10555 if (running) { 10556 set_next_task(rq, tsk); 10557 /* 10558 * After changing group, the running task may have joined a 10559 * throttled one but it's still the running task. Trigger a 10560 * resched to make sure that task can still run. 10561 */ 10562 resched_curr(rq); 10563 } 10564 10565 unlock: 10566 task_rq_unlock(rq, tsk, &rf); 10567 } 10568 10569 static inline struct task_group *css_tg(struct cgroup_subsys_state *css) 10570 { 10571 return css ? container_of(css, struct task_group, css) : NULL; 10572 } 10573 10574 static struct cgroup_subsys_state * 10575 cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) 10576 { 10577 struct task_group *parent = css_tg(parent_css); 10578 struct task_group *tg; 10579 10580 if (!parent) { 10581 /* This is early initialization for the top cgroup */ 10582 return &root_task_group.css; 10583 } 10584 10585 tg = sched_create_group(parent); 10586 if (IS_ERR(tg)) 10587 return ERR_PTR(-ENOMEM); 10588 10589 return &tg->css; 10590 } 10591 10592 /* Expose task group only after completing cgroup initialization */ 10593 static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) 10594 { 10595 struct task_group *tg = css_tg(css); 10596 struct task_group *parent = css_tg(css->parent); 10597 10598 if (parent) 10599 sched_online_group(tg, parent); 10600 10601 #ifdef CONFIG_UCLAMP_TASK_GROUP 10602 /* Propagate the effective uclamp value for the new group */ 10603 mutex_lock(&uclamp_mutex); 10604 rcu_read_lock(); 10605 cpu_util_update_eff(css); 10606 rcu_read_unlock(); 10607 mutex_unlock(&uclamp_mutex); 10608 #endif 10609 10610 return 0; 10611 } 10612 10613 static void cpu_cgroup_css_released(struct cgroup_subsys_state *css) 10614 { 10615 struct task_group *tg = css_tg(css); 10616 10617 sched_release_group(tg); 10618 } 10619 10620 static void cpu_cgroup_css_free(struct cgroup_subsys_state *css) 10621 { 10622 struct task_group *tg = css_tg(css); 10623 10624 /* 10625 * Relies on the RCU grace period between css_released() and this. 10626 */ 10627 sched_unregister_group(tg); 10628 } 10629 10630 #ifdef CONFIG_RT_GROUP_SCHED 10631 static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) 10632 { 10633 struct task_struct *task; 10634 struct cgroup_subsys_state *css; 10635 10636 cgroup_taskset_for_each(task, css, tset) { 10637 if (!sched_rt_can_attach(css_tg(css), task)) 10638 return -EINVAL; 10639 } 10640 return 0; 10641 } 10642 #endif 10643 10644 static void cpu_cgroup_attach(struct cgroup_taskset *tset) 10645 { 10646 struct task_struct *task; 10647 struct cgroup_subsys_state *css; 10648 10649 cgroup_taskset_for_each(task, css, tset) 10650 sched_move_task(task); 10651 } 10652 10653 #ifdef CONFIG_UCLAMP_TASK_GROUP 10654 static void cpu_util_update_eff(struct cgroup_subsys_state *css) 10655 { 10656 struct cgroup_subsys_state *top_css = css; 10657 struct uclamp_se *uc_parent = NULL; 10658 struct uclamp_se *uc_se = NULL; 10659 unsigned int eff[UCLAMP_CNT]; 10660 enum uclamp_id clamp_id; 10661 unsigned int clamps; 10662 10663 lockdep_assert_held(&uclamp_mutex); 10664 SCHED_WARN_ON(!rcu_read_lock_held()); 10665 10666 css_for_each_descendant_pre(css, top_css) { 10667 uc_parent = css_tg(css)->parent 10668 ? css_tg(css)->parent->uclamp : NULL; 10669 10670 for_each_clamp_id(clamp_id) { 10671 /* Assume effective clamps matches requested clamps */ 10672 eff[clamp_id] = css_tg(css)->uclamp_req[clamp_id].value; 10673 /* Cap effective clamps with parent's effective clamps */ 10674 if (uc_parent && 10675 eff[clamp_id] > uc_parent[clamp_id].value) { 10676 eff[clamp_id] = uc_parent[clamp_id].value; 10677 } 10678 } 10679 /* Ensure protection is always capped by limit */ 10680 eff[UCLAMP_MIN] = min(eff[UCLAMP_MIN], eff[UCLAMP_MAX]); 10681 10682 /* Propagate most restrictive effective clamps */ 10683 clamps = 0x0; 10684 uc_se = css_tg(css)->uclamp; 10685 for_each_clamp_id(clamp_id) { 10686 if (eff[clamp_id] == uc_se[clamp_id].value) 10687 continue; 10688 uc_se[clamp_id].value = eff[clamp_id]; 10689 uc_se[clamp_id].bucket_id = uclamp_bucket_id(eff[clamp_id]); 10690 clamps |= (0x1 << clamp_id); 10691 } 10692 if (!clamps) { 10693 css = css_rightmost_descendant(css); 10694 continue; 10695 } 10696 10697 /* Immediately update descendants RUNNABLE tasks */ 10698 uclamp_update_active_tasks(css); 10699 } 10700 } 10701 10702 /* 10703 * Integer 10^N with a given N exponent by casting to integer the literal "1eN" 10704 * C expression. Since there is no way to convert a macro argument (N) into a 10705 * character constant, use two levels of macros. 10706 */ 10707 #define _POW10(exp) ((unsigned int)1e##exp) 10708 #define POW10(exp) _POW10(exp) 10709 10710 struct uclamp_request { 10711 #define UCLAMP_PERCENT_SHIFT 2 10712 #define UCLAMP_PERCENT_SCALE (100 * POW10(UCLAMP_PERCENT_SHIFT)) 10713 s64 percent; 10714 u64 util; 10715 int ret; 10716 }; 10717 10718 static inline struct uclamp_request 10719 capacity_from_percent(char *buf) 10720 { 10721 struct uclamp_request req = { 10722 .percent = UCLAMP_PERCENT_SCALE, 10723 .util = SCHED_CAPACITY_SCALE, 10724 .ret = 0, 10725 }; 10726 10727 buf = strim(buf); 10728 if (strcmp(buf, "max")) { 10729 req.ret = cgroup_parse_float(buf, UCLAMP_PERCENT_SHIFT, 10730 &req.percent); 10731 if (req.ret) 10732 return req; 10733 if ((u64)req.percent > UCLAMP_PERCENT_SCALE) { 10734 req.ret = -ERANGE; 10735 return req; 10736 } 10737 10738 req.util = req.percent << SCHED_CAPACITY_SHIFT; 10739 req.util = DIV_ROUND_CLOSEST_ULL(req.util, UCLAMP_PERCENT_SCALE); 10740 } 10741 10742 return req; 10743 } 10744 10745 static ssize_t cpu_uclamp_write(struct kernfs_open_file *of, char *buf, 10746 size_t nbytes, loff_t off, 10747 enum uclamp_id clamp_id) 10748 { 10749 struct uclamp_request req; 10750 struct task_group *tg; 10751 10752 req = capacity_from_percent(buf); 10753 if (req.ret) 10754 return req.ret; 10755 10756 static_branch_enable(&sched_uclamp_used); 10757 10758 mutex_lock(&uclamp_mutex); 10759 rcu_read_lock(); 10760 10761 tg = css_tg(of_css(of)); 10762 if (tg->uclamp_req[clamp_id].value != req.util) 10763 uclamp_se_set(&tg->uclamp_req[clamp_id], req.util, false); 10764 10765 /* 10766 * Because of not recoverable conversion rounding we keep track of the 10767 * exact requested value 10768 */ 10769 tg->uclamp_pct[clamp_id] = req.percent; 10770 10771 /* Update effective clamps to track the most restrictive value */ 10772 cpu_util_update_eff(of_css(of)); 10773 10774 rcu_read_unlock(); 10775 mutex_unlock(&uclamp_mutex); 10776 10777 return nbytes; 10778 } 10779 10780 static ssize_t cpu_uclamp_min_write(struct kernfs_open_file *of, 10781 char *buf, size_t nbytes, 10782 loff_t off) 10783 { 10784 return cpu_uclamp_write(of, buf, nbytes, off, UCLAMP_MIN); 10785 } 10786 10787 static ssize_t cpu_uclamp_max_write(struct kernfs_open_file *of, 10788 char *buf, size_t nbytes, 10789 loff_t off) 10790 { 10791 return cpu_uclamp_write(of, buf, nbytes, off, UCLAMP_MAX); 10792 } 10793 10794 static inline void cpu_uclamp_print(struct seq_file *sf, 10795 enum uclamp_id clamp_id) 10796 { 10797 struct task_group *tg; 10798 u64 util_clamp; 10799 u64 percent; 10800 u32 rem; 10801 10802 rcu_read_lock(); 10803 tg = css_tg(seq_css(sf)); 10804 util_clamp = tg->uclamp_req[clamp_id].value; 10805 rcu_read_unlock(); 10806 10807 if (util_clamp == SCHED_CAPACITY_SCALE) { 10808 seq_puts(sf, "max\n"); 10809 return; 10810 } 10811 10812 percent = tg->uclamp_pct[clamp_id]; 10813 percent = div_u64_rem(percent, POW10(UCLAMP_PERCENT_SHIFT), &rem); 10814 seq_printf(sf, "%llu.%0*u\n", percent, UCLAMP_PERCENT_SHIFT, rem); 10815 } 10816 10817 static int cpu_uclamp_min_show(struct seq_file *sf, void *v) 10818 { 10819 cpu_uclamp_print(sf, UCLAMP_MIN); 10820 return 0; 10821 } 10822 10823 static int cpu_uclamp_max_show(struct seq_file *sf, void *v) 10824 { 10825 cpu_uclamp_print(sf, UCLAMP_MAX); 10826 return 0; 10827 } 10828 #endif /* CONFIG_UCLAMP_TASK_GROUP */ 10829 10830 #ifdef CONFIG_FAIR_GROUP_SCHED 10831 static int cpu_shares_write_u64(struct cgroup_subsys_state *css, 10832 struct cftype *cftype, u64 shareval) 10833 { 10834 if (shareval > scale_load_down(ULONG_MAX)) 10835 shareval = MAX_SHARES; 10836 return sched_group_set_shares(css_tg(css), scale_load(shareval)); 10837 } 10838 10839 static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css, 10840 struct cftype *cft) 10841 { 10842 struct task_group *tg = css_tg(css); 10843 10844 return (u64) scale_load_down(tg->shares); 10845 } 10846 10847 #ifdef CONFIG_CFS_BANDWIDTH 10848 static DEFINE_MUTEX(cfs_constraints_mutex); 10849 10850 const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */ 10851 static const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */ 10852 /* More than 203 days if BW_SHIFT equals 20. */ 10853 static const u64 max_cfs_runtime = MAX_BW * NSEC_PER_USEC; 10854 10855 static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime); 10856 10857 static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota, 10858 u64 burst) 10859 { 10860 int i, ret = 0, runtime_enabled, runtime_was_enabled; 10861 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 10862 10863 if (tg == &root_task_group) 10864 return -EINVAL; 10865 10866 /* 10867 * Ensure we have at some amount of bandwidth every period. This is 10868 * to prevent reaching a state of large arrears when throttled via 10869 * entity_tick() resulting in prolonged exit starvation. 10870 */ 10871 if (quota < min_cfs_quota_period || period < min_cfs_quota_period) 10872 return -EINVAL; 10873 10874 /* 10875 * Likewise, bound things on the other side by preventing insane quota 10876 * periods. This also allows us to normalize in computing quota 10877 * feasibility. 10878 */ 10879 if (period > max_cfs_quota_period) 10880 return -EINVAL; 10881 10882 /* 10883 * Bound quota to defend quota against overflow during bandwidth shift. 10884 */ 10885 if (quota != RUNTIME_INF && quota > max_cfs_runtime) 10886 return -EINVAL; 10887 10888 if (quota != RUNTIME_INF && (burst > quota || 10889 burst + quota > max_cfs_runtime)) 10890 return -EINVAL; 10891 10892 /* 10893 * Prevent race between setting of cfs_rq->runtime_enabled and 10894 * unthrottle_offline_cfs_rqs(). 10895 */ 10896 cpus_read_lock(); 10897 mutex_lock(&cfs_constraints_mutex); 10898 ret = __cfs_schedulable(tg, period, quota); 10899 if (ret) 10900 goto out_unlock; 10901 10902 runtime_enabled = quota != RUNTIME_INF; 10903 runtime_was_enabled = cfs_b->quota != RUNTIME_INF; 10904 /* 10905 * If we need to toggle cfs_bandwidth_used, off->on must occur 10906 * before making related changes, and on->off must occur afterwards 10907 */ 10908 if (runtime_enabled && !runtime_was_enabled) 10909 cfs_bandwidth_usage_inc(); 10910 raw_spin_lock_irq(&cfs_b->lock); 10911 cfs_b->period = ns_to_ktime(period); 10912 cfs_b->quota = quota; 10913 cfs_b->burst = burst; 10914 10915 __refill_cfs_bandwidth_runtime(cfs_b); 10916 10917 /* Restart the period timer (if active) to handle new period expiry: */ 10918 if (runtime_enabled) 10919 start_cfs_bandwidth(cfs_b); 10920 10921 raw_spin_unlock_irq(&cfs_b->lock); 10922 10923 for_each_online_cpu(i) { 10924 struct cfs_rq *cfs_rq = tg->cfs_rq[i]; 10925 struct rq *rq = cfs_rq->rq; 10926 struct rq_flags rf; 10927 10928 rq_lock_irq(rq, &rf); 10929 cfs_rq->runtime_enabled = runtime_enabled; 10930 cfs_rq->runtime_remaining = 0; 10931 10932 if (cfs_rq->throttled) 10933 unthrottle_cfs_rq(cfs_rq); 10934 rq_unlock_irq(rq, &rf); 10935 } 10936 if (runtime_was_enabled && !runtime_enabled) 10937 cfs_bandwidth_usage_dec(); 10938 out_unlock: 10939 mutex_unlock(&cfs_constraints_mutex); 10940 cpus_read_unlock(); 10941 10942 return ret; 10943 } 10944 10945 static int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us) 10946 { 10947 u64 quota, period, burst; 10948 10949 period = ktime_to_ns(tg->cfs_bandwidth.period); 10950 burst = tg->cfs_bandwidth.burst; 10951 if (cfs_quota_us < 0) 10952 quota = RUNTIME_INF; 10953 else if ((u64)cfs_quota_us <= U64_MAX / NSEC_PER_USEC) 10954 quota = (u64)cfs_quota_us * NSEC_PER_USEC; 10955 else 10956 return -EINVAL; 10957 10958 return tg_set_cfs_bandwidth(tg, period, quota, burst); 10959 } 10960 10961 static long tg_get_cfs_quota(struct task_group *tg) 10962 { 10963 u64 quota_us; 10964 10965 if (tg->cfs_bandwidth.quota == RUNTIME_INF) 10966 return -1; 10967 10968 quota_us = tg->cfs_bandwidth.quota; 10969 do_div(quota_us, NSEC_PER_USEC); 10970 10971 return quota_us; 10972 } 10973 10974 static int tg_set_cfs_period(struct task_group *tg, long cfs_period_us) 10975 { 10976 u64 quota, period, burst; 10977 10978 if ((u64)cfs_period_us > U64_MAX / NSEC_PER_USEC) 10979 return -EINVAL; 10980 10981 period = (u64)cfs_period_us * NSEC_PER_USEC; 10982 quota = tg->cfs_bandwidth.quota; 10983 burst = tg->cfs_bandwidth.burst; 10984 10985 return tg_set_cfs_bandwidth(tg, period, quota, burst); 10986 } 10987 10988 static long tg_get_cfs_period(struct task_group *tg) 10989 { 10990 u64 cfs_period_us; 10991 10992 cfs_period_us = ktime_to_ns(tg->cfs_bandwidth.period); 10993 do_div(cfs_period_us, NSEC_PER_USEC); 10994 10995 return cfs_period_us; 10996 } 10997 10998 static int tg_set_cfs_burst(struct task_group *tg, long cfs_burst_us) 10999 { 11000 u64 quota, period, burst; 11001 11002 if ((u64)cfs_burst_us > U64_MAX / NSEC_PER_USEC) 11003 return -EINVAL; 11004 11005 burst = (u64)cfs_burst_us * NSEC_PER_USEC; 11006 period = ktime_to_ns(tg->cfs_bandwidth.period); 11007 quota = tg->cfs_bandwidth.quota; 11008 11009 return tg_set_cfs_bandwidth(tg, period, quota, burst); 11010 } 11011 11012 static long tg_get_cfs_burst(struct task_group *tg) 11013 { 11014 u64 burst_us; 11015 11016 burst_us = tg->cfs_bandwidth.burst; 11017 do_div(burst_us, NSEC_PER_USEC); 11018 11019 return burst_us; 11020 } 11021 11022 static s64 cpu_cfs_quota_read_s64(struct cgroup_subsys_state *css, 11023 struct cftype *cft) 11024 { 11025 return tg_get_cfs_quota(css_tg(css)); 11026 } 11027 11028 static int cpu_cfs_quota_write_s64(struct cgroup_subsys_state *css, 11029 struct cftype *cftype, s64 cfs_quota_us) 11030 { 11031 return tg_set_cfs_quota(css_tg(css), cfs_quota_us); 11032 } 11033 11034 static u64 cpu_cfs_period_read_u64(struct cgroup_subsys_state *css, 11035 struct cftype *cft) 11036 { 11037 return tg_get_cfs_period(css_tg(css)); 11038 } 11039 11040 static int cpu_cfs_period_write_u64(struct cgroup_subsys_state *css, 11041 struct cftype *cftype, u64 cfs_period_us) 11042 { 11043 return tg_set_cfs_period(css_tg(css), cfs_period_us); 11044 } 11045 11046 static u64 cpu_cfs_burst_read_u64(struct cgroup_subsys_state *css, 11047 struct cftype *cft) 11048 { 11049 return tg_get_cfs_burst(css_tg(css)); 11050 } 11051 11052 static int cpu_cfs_burst_write_u64(struct cgroup_subsys_state *css, 11053 struct cftype *cftype, u64 cfs_burst_us) 11054 { 11055 return tg_set_cfs_burst(css_tg(css), cfs_burst_us); 11056 } 11057 11058 struct cfs_schedulable_data { 11059 struct task_group *tg; 11060 u64 period, quota; 11061 }; 11062 11063 /* 11064 * normalize group quota/period to be quota/max_period 11065 * note: units are usecs 11066 */ 11067 static u64 normalize_cfs_quota(struct task_group *tg, 11068 struct cfs_schedulable_data *d) 11069 { 11070 u64 quota, period; 11071 11072 if (tg == d->tg) { 11073 period = d->period; 11074 quota = d->quota; 11075 } else { 11076 period = tg_get_cfs_period(tg); 11077 quota = tg_get_cfs_quota(tg); 11078 } 11079 11080 /* note: these should typically be equivalent */ 11081 if (quota == RUNTIME_INF || quota == -1) 11082 return RUNTIME_INF; 11083 11084 return to_ratio(period, quota); 11085 } 11086 11087 static int tg_cfs_schedulable_down(struct task_group *tg, void *data) 11088 { 11089 struct cfs_schedulable_data *d = data; 11090 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 11091 s64 quota = 0, parent_quota = -1; 11092 11093 if (!tg->parent) { 11094 quota = RUNTIME_INF; 11095 } else { 11096 struct cfs_bandwidth *parent_b = &tg->parent->cfs_bandwidth; 11097 11098 quota = normalize_cfs_quota(tg, d); 11099 parent_quota = parent_b->hierarchical_quota; 11100 11101 /* 11102 * Ensure max(child_quota) <= parent_quota. On cgroup2, 11103 * always take the non-RUNTIME_INF min. On cgroup1, only 11104 * inherit when no limit is set. In both cases this is used 11105 * by the scheduler to determine if a given CFS task has a 11106 * bandwidth constraint at some higher level. 11107 */ 11108 if (cgroup_subsys_on_dfl(cpu_cgrp_subsys)) { 11109 if (quota == RUNTIME_INF) 11110 quota = parent_quota; 11111 else if (parent_quota != RUNTIME_INF) 11112 quota = min(quota, parent_quota); 11113 } else { 11114 if (quota == RUNTIME_INF) 11115 quota = parent_quota; 11116 else if (parent_quota != RUNTIME_INF && quota > parent_quota) 11117 return -EINVAL; 11118 } 11119 } 11120 cfs_b->hierarchical_quota = quota; 11121 11122 return 0; 11123 } 11124 11125 static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota) 11126 { 11127 int ret; 11128 struct cfs_schedulable_data data = { 11129 .tg = tg, 11130 .period = period, 11131 .quota = quota, 11132 }; 11133 11134 if (quota != RUNTIME_INF) { 11135 do_div(data.period, NSEC_PER_USEC); 11136 do_div(data.quota, NSEC_PER_USEC); 11137 } 11138 11139 rcu_read_lock(); 11140 ret = walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data); 11141 rcu_read_unlock(); 11142 11143 return ret; 11144 } 11145 11146 static int cpu_cfs_stat_show(struct seq_file *sf, void *v) 11147 { 11148 struct task_group *tg = css_tg(seq_css(sf)); 11149 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 11150 11151 seq_printf(sf, "nr_periods %d\n", cfs_b->nr_periods); 11152 seq_printf(sf, "nr_throttled %d\n", cfs_b->nr_throttled); 11153 seq_printf(sf, "throttled_time %llu\n", cfs_b->throttled_time); 11154 11155 if (schedstat_enabled() && tg != &root_task_group) { 11156 struct sched_statistics *stats; 11157 u64 ws = 0; 11158 int i; 11159 11160 for_each_possible_cpu(i) { 11161 stats = __schedstats_from_se(tg->se[i]); 11162 ws += schedstat_val(stats->wait_sum); 11163 } 11164 11165 seq_printf(sf, "wait_sum %llu\n", ws); 11166 } 11167 11168 seq_printf(sf, "nr_bursts %d\n", cfs_b->nr_burst); 11169 seq_printf(sf, "burst_time %llu\n", cfs_b->burst_time); 11170 11171 return 0; 11172 } 11173 11174 static u64 throttled_time_self(struct task_group *tg) 11175 { 11176 int i; 11177 u64 total = 0; 11178 11179 for_each_possible_cpu(i) { 11180 total += READ_ONCE(tg->cfs_rq[i]->throttled_clock_self_time); 11181 } 11182 11183 return total; 11184 } 11185 11186 static int cpu_cfs_local_stat_show(struct seq_file *sf, void *v) 11187 { 11188 struct task_group *tg = css_tg(seq_css(sf)); 11189 11190 seq_printf(sf, "throttled_time %llu\n", throttled_time_self(tg)); 11191 11192 return 0; 11193 } 11194 #endif /* CONFIG_CFS_BANDWIDTH */ 11195 #endif /* CONFIG_FAIR_GROUP_SCHED */ 11196 11197 #ifdef CONFIG_RT_GROUP_SCHED 11198 static int cpu_rt_runtime_write(struct cgroup_subsys_state *css, 11199 struct cftype *cft, s64 val) 11200 { 11201 return sched_group_set_rt_runtime(css_tg(css), val); 11202 } 11203 11204 static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css, 11205 struct cftype *cft) 11206 { 11207 return sched_group_rt_runtime(css_tg(css)); 11208 } 11209 11210 static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css, 11211 struct cftype *cftype, u64 rt_period_us) 11212 { 11213 return sched_group_set_rt_period(css_tg(css), rt_period_us); 11214 } 11215 11216 static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css, 11217 struct cftype *cft) 11218 { 11219 return sched_group_rt_period(css_tg(css)); 11220 } 11221 #endif /* CONFIG_RT_GROUP_SCHED */ 11222 11223 #ifdef CONFIG_FAIR_GROUP_SCHED 11224 static s64 cpu_idle_read_s64(struct cgroup_subsys_state *css, 11225 struct cftype *cft) 11226 { 11227 return css_tg(css)->idle; 11228 } 11229 11230 static int cpu_idle_write_s64(struct cgroup_subsys_state *css, 11231 struct cftype *cft, s64 idle) 11232 { 11233 return sched_group_set_idle(css_tg(css), idle); 11234 } 11235 #endif 11236 11237 static struct cftype cpu_legacy_files[] = { 11238 #ifdef CONFIG_FAIR_GROUP_SCHED 11239 { 11240 .name = "shares", 11241 .read_u64 = cpu_shares_read_u64, 11242 .write_u64 = cpu_shares_write_u64, 11243 }, 11244 { 11245 .name = "idle", 11246 .read_s64 = cpu_idle_read_s64, 11247 .write_s64 = cpu_idle_write_s64, 11248 }, 11249 #endif 11250 #ifdef CONFIG_CFS_BANDWIDTH 11251 { 11252 .name = "cfs_quota_us", 11253 .read_s64 = cpu_cfs_quota_read_s64, 11254 .write_s64 = cpu_cfs_quota_write_s64, 11255 }, 11256 { 11257 .name = "cfs_period_us", 11258 .read_u64 = cpu_cfs_period_read_u64, 11259 .write_u64 = cpu_cfs_period_write_u64, 11260 }, 11261 { 11262 .name = "cfs_burst_us", 11263 .read_u64 = cpu_cfs_burst_read_u64, 11264 .write_u64 = cpu_cfs_burst_write_u64, 11265 }, 11266 { 11267 .name = "stat", 11268 .seq_show = cpu_cfs_stat_show, 11269 }, 11270 { 11271 .name = "stat.local", 11272 .seq_show = cpu_cfs_local_stat_show, 11273 }, 11274 #endif 11275 #ifdef CONFIG_RT_GROUP_SCHED 11276 { 11277 .name = "rt_runtime_us", 11278 .read_s64 = cpu_rt_runtime_read, 11279 .write_s64 = cpu_rt_runtime_write, 11280 }, 11281 { 11282 .name = "rt_period_us", 11283 .read_u64 = cpu_rt_period_read_uint, 11284 .write_u64 = cpu_rt_period_write_uint, 11285 }, 11286 #endif 11287 #ifdef CONFIG_UCLAMP_TASK_GROUP 11288 { 11289 .name = "uclamp.min", 11290 .flags = CFTYPE_NOT_ON_ROOT, 11291 .seq_show = cpu_uclamp_min_show, 11292 .write = cpu_uclamp_min_write, 11293 }, 11294 { 11295 .name = "uclamp.max", 11296 .flags = CFTYPE_NOT_ON_ROOT, 11297 .seq_show = cpu_uclamp_max_show, 11298 .write = cpu_uclamp_max_write, 11299 }, 11300 #endif 11301 { } /* Terminate */ 11302 }; 11303 11304 static int cpu_extra_stat_show(struct seq_file *sf, 11305 struct cgroup_subsys_state *css) 11306 { 11307 #ifdef CONFIG_CFS_BANDWIDTH 11308 { 11309 struct task_group *tg = css_tg(css); 11310 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 11311 u64 throttled_usec, burst_usec; 11312 11313 throttled_usec = cfs_b->throttled_time; 11314 do_div(throttled_usec, NSEC_PER_USEC); 11315 burst_usec = cfs_b->burst_time; 11316 do_div(burst_usec, NSEC_PER_USEC); 11317 11318 seq_printf(sf, "nr_periods %d\n" 11319 "nr_throttled %d\n" 11320 "throttled_usec %llu\n" 11321 "nr_bursts %d\n" 11322 "burst_usec %llu\n", 11323 cfs_b->nr_periods, cfs_b->nr_throttled, 11324 throttled_usec, cfs_b->nr_burst, burst_usec); 11325 } 11326 #endif 11327 return 0; 11328 } 11329 11330 static int cpu_local_stat_show(struct seq_file *sf, 11331 struct cgroup_subsys_state *css) 11332 { 11333 #ifdef CONFIG_CFS_BANDWIDTH 11334 { 11335 struct task_group *tg = css_tg(css); 11336 u64 throttled_self_usec; 11337 11338 throttled_self_usec = throttled_time_self(tg); 11339 do_div(throttled_self_usec, NSEC_PER_USEC); 11340 11341 seq_printf(sf, "throttled_usec %llu\n", 11342 throttled_self_usec); 11343 } 11344 #endif 11345 return 0; 11346 } 11347 11348 #ifdef CONFIG_FAIR_GROUP_SCHED 11349 static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css, 11350 struct cftype *cft) 11351 { 11352 struct task_group *tg = css_tg(css); 11353 u64 weight = scale_load_down(tg->shares); 11354 11355 return DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024); 11356 } 11357 11358 static int cpu_weight_write_u64(struct cgroup_subsys_state *css, 11359 struct cftype *cft, u64 weight) 11360 { 11361 /* 11362 * cgroup weight knobs should use the common MIN, DFL and MAX 11363 * values which are 1, 100 and 10000 respectively. While it loses 11364 * a bit of range on both ends, it maps pretty well onto the shares 11365 * value used by scheduler and the round-trip conversions preserve 11366 * the original value over the entire range. 11367 */ 11368 if (weight < CGROUP_WEIGHT_MIN || weight > CGROUP_WEIGHT_MAX) 11369 return -ERANGE; 11370 11371 weight = DIV_ROUND_CLOSEST_ULL(weight * 1024, CGROUP_WEIGHT_DFL); 11372 11373 return sched_group_set_shares(css_tg(css), scale_load(weight)); 11374 } 11375 11376 static s64 cpu_weight_nice_read_s64(struct cgroup_subsys_state *css, 11377 struct cftype *cft) 11378 { 11379 unsigned long weight = scale_load_down(css_tg(css)->shares); 11380 int last_delta = INT_MAX; 11381 int prio, delta; 11382 11383 /* find the closest nice value to the current weight */ 11384 for (prio = 0; prio < ARRAY_SIZE(sched_prio_to_weight); prio++) { 11385 delta = abs(sched_prio_to_weight[prio] - weight); 11386 if (delta >= last_delta) 11387 break; 11388 last_delta = delta; 11389 } 11390 11391 return PRIO_TO_NICE(prio - 1 + MAX_RT_PRIO); 11392 } 11393 11394 static int cpu_weight_nice_write_s64(struct cgroup_subsys_state *css, 11395 struct cftype *cft, s64 nice) 11396 { 11397 unsigned long weight; 11398 int idx; 11399 11400 if (nice < MIN_NICE || nice > MAX_NICE) 11401 return -ERANGE; 11402 11403 idx = NICE_TO_PRIO(nice) - MAX_RT_PRIO; 11404 idx = array_index_nospec(idx, 40); 11405 weight = sched_prio_to_weight[idx]; 11406 11407 return sched_group_set_shares(css_tg(css), scale_load(weight)); 11408 } 11409 #endif 11410 11411 static void __maybe_unused cpu_period_quota_print(struct seq_file *sf, 11412 long period, long quota) 11413 { 11414 if (quota < 0) 11415 seq_puts(sf, "max"); 11416 else 11417 seq_printf(sf, "%ld", quota); 11418 11419 seq_printf(sf, " %ld\n", period); 11420 } 11421 11422 /* caller should put the current value in *@periodp before calling */ 11423 static int __maybe_unused cpu_period_quota_parse(char *buf, 11424 u64 *periodp, u64 *quotap) 11425 { 11426 char tok[21]; /* U64_MAX */ 11427 11428 if (sscanf(buf, "%20s %llu", tok, periodp) < 1) 11429 return -EINVAL; 11430 11431 *periodp *= NSEC_PER_USEC; 11432 11433 if (sscanf(tok, "%llu", quotap)) 11434 *quotap *= NSEC_PER_USEC; 11435 else if (!strcmp(tok, "max")) 11436 *quotap = RUNTIME_INF; 11437 else 11438 return -EINVAL; 11439 11440 return 0; 11441 } 11442 11443 #ifdef CONFIG_CFS_BANDWIDTH 11444 static int cpu_max_show(struct seq_file *sf, void *v) 11445 { 11446 struct task_group *tg = css_tg(seq_css(sf)); 11447 11448 cpu_period_quota_print(sf, tg_get_cfs_period(tg), tg_get_cfs_quota(tg)); 11449 return 0; 11450 } 11451 11452 static ssize_t cpu_max_write(struct kernfs_open_file *of, 11453 char *buf, size_t nbytes, loff_t off) 11454 { 11455 struct task_group *tg = css_tg(of_css(of)); 11456 u64 period = tg_get_cfs_period(tg); 11457 u64 burst = tg_get_cfs_burst(tg); 11458 u64 quota; 11459 int ret; 11460 11461 ret = cpu_period_quota_parse(buf, &period, "a); 11462 if (!ret) 11463 ret = tg_set_cfs_bandwidth(tg, period, quota, burst); 11464 return ret ?: nbytes; 11465 } 11466 #endif 11467 11468 static struct cftype cpu_files[] = { 11469 #ifdef CONFIG_FAIR_GROUP_SCHED 11470 { 11471 .name = "weight", 11472 .flags = CFTYPE_NOT_ON_ROOT, 11473 .read_u64 = cpu_weight_read_u64, 11474 .write_u64 = cpu_weight_write_u64, 11475 }, 11476 { 11477 .name = "weight.nice", 11478 .flags = CFTYPE_NOT_ON_ROOT, 11479 .read_s64 = cpu_weight_nice_read_s64, 11480 .write_s64 = cpu_weight_nice_write_s64, 11481 }, 11482 { 11483 .name = "idle", 11484 .flags = CFTYPE_NOT_ON_ROOT, 11485 .read_s64 = cpu_idle_read_s64, 11486 .write_s64 = cpu_idle_write_s64, 11487 }, 11488 #endif 11489 #ifdef CONFIG_CFS_BANDWIDTH 11490 { 11491 .name = "max", 11492 .flags = CFTYPE_NOT_ON_ROOT, 11493 .seq_show = cpu_max_show, 11494 .write = cpu_max_write, 11495 }, 11496 { 11497 .name = "max.burst", 11498 .flags = CFTYPE_NOT_ON_ROOT, 11499 .read_u64 = cpu_cfs_burst_read_u64, 11500 .write_u64 = cpu_cfs_burst_write_u64, 11501 }, 11502 #endif 11503 #ifdef CONFIG_UCLAMP_TASK_GROUP 11504 { 11505 .name = "uclamp.min", 11506 .flags = CFTYPE_NOT_ON_ROOT, 11507 .seq_show = cpu_uclamp_min_show, 11508 .write = cpu_uclamp_min_write, 11509 }, 11510 { 11511 .name = "uclamp.max", 11512 .flags = CFTYPE_NOT_ON_ROOT, 11513 .seq_show = cpu_uclamp_max_show, 11514 .write = cpu_uclamp_max_write, 11515 }, 11516 #endif 11517 { } /* terminate */ 11518 }; 11519 11520 struct cgroup_subsys cpu_cgrp_subsys = { 11521 .css_alloc = cpu_cgroup_css_alloc, 11522 .css_online = cpu_cgroup_css_online, 11523 .css_released = cpu_cgroup_css_released, 11524 .css_free = cpu_cgroup_css_free, 11525 .css_extra_stat_show = cpu_extra_stat_show, 11526 .css_local_stat_show = cpu_local_stat_show, 11527 #ifdef CONFIG_RT_GROUP_SCHED 11528 .can_attach = cpu_cgroup_can_attach, 11529 #endif 11530 .attach = cpu_cgroup_attach, 11531 .legacy_cftypes = cpu_legacy_files, 11532 .dfl_cftypes = cpu_files, 11533 .early_init = true, 11534 .threaded = true, 11535 }; 11536 11537 #endif /* CONFIG_CGROUP_SCHED */ 11538 11539 void dump_cpu_task(int cpu) 11540 { 11541 if (cpu == smp_processor_id() && in_hardirq()) { 11542 struct pt_regs *regs; 11543 11544 regs = get_irq_regs(); 11545 if (regs) { 11546 show_regs(regs); 11547 return; 11548 } 11549 } 11550 11551 if (trigger_single_cpu_backtrace(cpu)) 11552 return; 11553 11554 pr_info("Task dump for CPU %d:\n", cpu); 11555 sched_show_task(cpu_curr(cpu)); 11556 } 11557 11558 /* 11559 * Nice levels are multiplicative, with a gentle 10% change for every 11560 * nice level changed. I.e. when a CPU-bound task goes from nice 0 to 11561 * nice 1, it will get ~10% less CPU time than another CPU-bound task 11562 * that remained on nice 0. 11563 * 11564 * The "10% effect" is relative and cumulative: from _any_ nice level, 11565 * if you go up 1 level, it's -10% CPU usage, if you go down 1 level 11566 * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25. 11567 * If a task goes up by ~10% and another task goes down by ~10% then 11568 * the relative distance between them is ~25%.) 11569 */ 11570 const int sched_prio_to_weight[40] = { 11571 /* -20 */ 88761, 71755, 56483, 46273, 36291, 11572 /* -15 */ 29154, 23254, 18705, 14949, 11916, 11573 /* -10 */ 9548, 7620, 6100, 4904, 3906, 11574 /* -5 */ 3121, 2501, 1991, 1586, 1277, 11575 /* 0 */ 1024, 820, 655, 526, 423, 11576 /* 5 */ 335, 272, 215, 172, 137, 11577 /* 10 */ 110, 87, 70, 56, 45, 11578 /* 15 */ 36, 29, 23, 18, 15, 11579 }; 11580 11581 /* 11582 * Inverse (2^32/x) values of the sched_prio_to_weight[] array, precalculated. 11583 * 11584 * In cases where the weight does not change often, we can use the 11585 * precalculated inverse to speed up arithmetics by turning divisions 11586 * into multiplications: 11587 */ 11588 const u32 sched_prio_to_wmult[40] = { 11589 /* -20 */ 48388, 59856, 76040, 92818, 118348, 11590 /* -15 */ 147320, 184698, 229616, 287308, 360437, 11591 /* -10 */ 449829, 563644, 704093, 875809, 1099582, 11592 /* -5 */ 1376151, 1717300, 2157191, 2708050, 3363326, 11593 /* 0 */ 4194304, 5237765, 6557202, 8165337, 10153587, 11594 /* 5 */ 12820798, 15790321, 19976592, 24970740, 31350126, 11595 /* 10 */ 39045157, 49367440, 61356676, 76695844, 95443717, 11596 /* 15 */ 119304647, 148102320, 186737708, 238609294, 286331153, 11597 }; 11598 11599 void call_trace_sched_update_nr_running(struct rq *rq, int count) 11600 { 11601 trace_sched_update_nr_running_tp(rq, count); 11602 } 11603 11604 #ifdef CONFIG_SCHED_MM_CID 11605 11606 /* 11607 * @cid_lock: Guarantee forward-progress of cid allocation. 11608 * 11609 * Concurrency ID allocation within a bitmap is mostly lock-free. The cid_lock 11610 * is only used when contention is detected by the lock-free allocation so 11611 * forward progress can be guaranteed. 11612 */ 11613 DEFINE_RAW_SPINLOCK(cid_lock); 11614 11615 /* 11616 * @use_cid_lock: Select cid allocation behavior: lock-free vs spinlock. 11617 * 11618 * When @use_cid_lock is 0, the cid allocation is lock-free. When contention is 11619 * detected, it is set to 1 to ensure that all newly coming allocations are 11620 * serialized by @cid_lock until the allocation which detected contention 11621 * completes and sets @use_cid_lock back to 0. This guarantees forward progress 11622 * of a cid allocation. 11623 */ 11624 int use_cid_lock; 11625 11626 /* 11627 * mm_cid remote-clear implements a lock-free algorithm to clear per-mm/cpu cid 11628 * concurrently with respect to the execution of the source runqueue context 11629 * switch. 11630 * 11631 * There is one basic properties we want to guarantee here: 11632 * 11633 * (1) Remote-clear should _never_ mark a per-cpu cid UNSET when it is actively 11634 * used by a task. That would lead to concurrent allocation of the cid and 11635 * userspace corruption. 11636 * 11637 * Provide this guarantee by introducing a Dekker memory ordering to guarantee 11638 * that a pair of loads observe at least one of a pair of stores, which can be 11639 * shown as: 11640 * 11641 * X = Y = 0 11642 * 11643 * w[X]=1 w[Y]=1 11644 * MB MB 11645 * r[Y]=y r[X]=x 11646 * 11647 * Which guarantees that x==0 && y==0 is impossible. But rather than using 11648 * values 0 and 1, this algorithm cares about specific state transitions of the 11649 * runqueue current task (as updated by the scheduler context switch), and the 11650 * per-mm/cpu cid value. 11651 * 11652 * Let's introduce task (Y) which has task->mm == mm and task (N) which has 11653 * task->mm != mm for the rest of the discussion. There are two scheduler state 11654 * transitions on context switch we care about: 11655 * 11656 * (TSA) Store to rq->curr with transition from (N) to (Y) 11657 * 11658 * (TSB) Store to rq->curr with transition from (Y) to (N) 11659 * 11660 * On the remote-clear side, there is one transition we care about: 11661 * 11662 * (TMA) cmpxchg to *pcpu_cid to set the LAZY flag 11663 * 11664 * There is also a transition to UNSET state which can be performed from all 11665 * sides (scheduler, remote-clear). It is always performed with a cmpxchg which 11666 * guarantees that only a single thread will succeed: 11667 * 11668 * (TMB) cmpxchg to *pcpu_cid to mark UNSET 11669 * 11670 * Just to be clear, what we do _not_ want to happen is a transition to UNSET 11671 * when a thread is actively using the cid (property (1)). 11672 * 11673 * Let's looks at the relevant combinations of TSA/TSB, and TMA transitions. 11674 * 11675 * Scenario A) (TSA)+(TMA) (from next task perspective) 11676 * 11677 * CPU0 CPU1 11678 * 11679 * Context switch CS-1 Remote-clear 11680 * - store to rq->curr: (N)->(Y) (TSA) - cmpxchg to *pcpu_id to LAZY (TMA) 11681 * (implied barrier after cmpxchg) 11682 * - switch_mm_cid() 11683 * - memory barrier (see switch_mm_cid() 11684 * comment explaining how this barrier 11685 * is combined with other scheduler 11686 * barriers) 11687 * - mm_cid_get (next) 11688 * - READ_ONCE(*pcpu_cid) - rcu_dereference(src_rq->curr) 11689 * 11690 * This Dekker ensures that either task (Y) is observed by the 11691 * rcu_dereference() or the LAZY flag is observed by READ_ONCE(), or both are 11692 * observed. 11693 * 11694 * If task (Y) store is observed by rcu_dereference(), it means that there is 11695 * still an active task on the cpu. Remote-clear will therefore not transition 11696 * to UNSET, which fulfills property (1). 11697 * 11698 * If task (Y) is not observed, but the lazy flag is observed by READ_ONCE(), 11699 * it will move its state to UNSET, which clears the percpu cid perhaps 11700 * uselessly (which is not an issue for correctness). Because task (Y) is not 11701 * observed, CPU1 can move ahead to set the state to UNSET. Because moving 11702 * state to UNSET is done with a cmpxchg expecting that the old state has the 11703 * LAZY flag set, only one thread will successfully UNSET. 11704 * 11705 * If both states (LAZY flag and task (Y)) are observed, the thread on CPU0 11706 * will observe the LAZY flag and transition to UNSET (perhaps uselessly), and 11707 * CPU1 will observe task (Y) and do nothing more, which is fine. 11708 * 11709 * What we are effectively preventing with this Dekker is a scenario where 11710 * neither LAZY flag nor store (Y) are observed, which would fail property (1) 11711 * because this would UNSET a cid which is actively used. 11712 */ 11713 11714 void sched_mm_cid_migrate_from(struct task_struct *t) 11715 { 11716 t->migrate_from_cpu = task_cpu(t); 11717 } 11718 11719 static 11720 int __sched_mm_cid_migrate_from_fetch_cid(struct rq *src_rq, 11721 struct task_struct *t, 11722 struct mm_cid *src_pcpu_cid) 11723 { 11724 struct mm_struct *mm = t->mm; 11725 struct task_struct *src_task; 11726 int src_cid, last_mm_cid; 11727 11728 if (!mm) 11729 return -1; 11730 11731 last_mm_cid = t->last_mm_cid; 11732 /* 11733 * If the migrated task has no last cid, or if the current 11734 * task on src rq uses the cid, it means the source cid does not need 11735 * to be moved to the destination cpu. 11736 */ 11737 if (last_mm_cid == -1) 11738 return -1; 11739 src_cid = READ_ONCE(src_pcpu_cid->cid); 11740 if (!mm_cid_is_valid(src_cid) || last_mm_cid != src_cid) 11741 return -1; 11742 11743 /* 11744 * If we observe an active task using the mm on this rq, it means we 11745 * are not the last task to be migrated from this cpu for this mm, so 11746 * there is no need to move src_cid to the destination cpu. 11747 */ 11748 rcu_read_lock(); 11749 src_task = rcu_dereference(src_rq->curr); 11750 if (READ_ONCE(src_task->mm_cid_active) && src_task->mm == mm) { 11751 rcu_read_unlock(); 11752 t->last_mm_cid = -1; 11753 return -1; 11754 } 11755 rcu_read_unlock(); 11756 11757 return src_cid; 11758 } 11759 11760 static 11761 int __sched_mm_cid_migrate_from_try_steal_cid(struct rq *src_rq, 11762 struct task_struct *t, 11763 struct mm_cid *src_pcpu_cid, 11764 int src_cid) 11765 { 11766 struct task_struct *src_task; 11767 struct mm_struct *mm = t->mm; 11768 int lazy_cid; 11769 11770 if (src_cid == -1) 11771 return -1; 11772 11773 /* 11774 * Attempt to clear the source cpu cid to move it to the destination 11775 * cpu. 11776 */ 11777 lazy_cid = mm_cid_set_lazy_put(src_cid); 11778 if (!try_cmpxchg(&src_pcpu_cid->cid, &src_cid, lazy_cid)) 11779 return -1; 11780 11781 /* 11782 * The implicit barrier after cmpxchg per-mm/cpu cid before loading 11783 * rq->curr->mm matches the scheduler barrier in context_switch() 11784 * between store to rq->curr and load of prev and next task's 11785 * per-mm/cpu cid. 11786 * 11787 * The implicit barrier after cmpxchg per-mm/cpu cid before loading 11788 * rq->curr->mm_cid_active matches the barrier in 11789 * sched_mm_cid_exit_signals(), sched_mm_cid_before_execve(), and 11790 * sched_mm_cid_after_execve() between store to t->mm_cid_active and 11791 * load of per-mm/cpu cid. 11792 */ 11793 11794 /* 11795 * If we observe an active task using the mm on this rq after setting 11796 * the lazy-put flag, this task will be responsible for transitioning 11797 * from lazy-put flag set to MM_CID_UNSET. 11798 */ 11799 rcu_read_lock(); 11800 src_task = rcu_dereference(src_rq->curr); 11801 if (READ_ONCE(src_task->mm_cid_active) && src_task->mm == mm) { 11802 rcu_read_unlock(); 11803 /* 11804 * We observed an active task for this mm, there is therefore 11805 * no point in moving this cid to the destination cpu. 11806 */ 11807 t->last_mm_cid = -1; 11808 return -1; 11809 } 11810 rcu_read_unlock(); 11811 11812 /* 11813 * The src_cid is unused, so it can be unset. 11814 */ 11815 if (!try_cmpxchg(&src_pcpu_cid->cid, &lazy_cid, MM_CID_UNSET)) 11816 return -1; 11817 return src_cid; 11818 } 11819 11820 /* 11821 * Migration to dst cpu. Called with dst_rq lock held. 11822 * Interrupts are disabled, which keeps the window of cid ownership without the 11823 * source rq lock held small. 11824 */ 11825 void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t) 11826 { 11827 struct mm_cid *src_pcpu_cid, *dst_pcpu_cid; 11828 struct mm_struct *mm = t->mm; 11829 int src_cid, dst_cid, src_cpu; 11830 struct rq *src_rq; 11831 11832 lockdep_assert_rq_held(dst_rq); 11833 11834 if (!mm) 11835 return; 11836 src_cpu = t->migrate_from_cpu; 11837 if (src_cpu == -1) { 11838 t->last_mm_cid = -1; 11839 return; 11840 } 11841 /* 11842 * Move the src cid if the dst cid is unset. This keeps id 11843 * allocation closest to 0 in cases where few threads migrate around 11844 * many cpus. 11845 * 11846 * If destination cid is already set, we may have to just clear 11847 * the src cid to ensure compactness in frequent migrations 11848 * scenarios. 11849 * 11850 * It is not useful to clear the src cid when the number of threads is 11851 * greater or equal to the number of allowed cpus, because user-space 11852 * can expect that the number of allowed cids can reach the number of 11853 * allowed cpus. 11854 */ 11855 dst_pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu_of(dst_rq)); 11856 dst_cid = READ_ONCE(dst_pcpu_cid->cid); 11857 if (!mm_cid_is_unset(dst_cid) && 11858 atomic_read(&mm->mm_users) >= t->nr_cpus_allowed) 11859 return; 11860 src_pcpu_cid = per_cpu_ptr(mm->pcpu_cid, src_cpu); 11861 src_rq = cpu_rq(src_cpu); 11862 src_cid = __sched_mm_cid_migrate_from_fetch_cid(src_rq, t, src_pcpu_cid); 11863 if (src_cid == -1) 11864 return; 11865 src_cid = __sched_mm_cid_migrate_from_try_steal_cid(src_rq, t, src_pcpu_cid, 11866 src_cid); 11867 if (src_cid == -1) 11868 return; 11869 if (!mm_cid_is_unset(dst_cid)) { 11870 __mm_cid_put(mm, src_cid); 11871 return; 11872 } 11873 /* Move src_cid to dst cpu. */ 11874 mm_cid_snapshot_time(dst_rq, mm); 11875 WRITE_ONCE(dst_pcpu_cid->cid, src_cid); 11876 } 11877 11878 static void sched_mm_cid_remote_clear(struct mm_struct *mm, struct mm_cid *pcpu_cid, 11879 int cpu) 11880 { 11881 struct rq *rq = cpu_rq(cpu); 11882 struct task_struct *t; 11883 unsigned long flags; 11884 int cid, lazy_cid; 11885 11886 cid = READ_ONCE(pcpu_cid->cid); 11887 if (!mm_cid_is_valid(cid)) 11888 return; 11889 11890 /* 11891 * Clear the cpu cid if it is set to keep cid allocation compact. If 11892 * there happens to be other tasks left on the source cpu using this 11893 * mm, the next task using this mm will reallocate its cid on context 11894 * switch. 11895 */ 11896 lazy_cid = mm_cid_set_lazy_put(cid); 11897 if (!try_cmpxchg(&pcpu_cid->cid, &cid, lazy_cid)) 11898 return; 11899 11900 /* 11901 * The implicit barrier after cmpxchg per-mm/cpu cid before loading 11902 * rq->curr->mm matches the scheduler barrier in context_switch() 11903 * between store to rq->curr and load of prev and next task's 11904 * per-mm/cpu cid. 11905 * 11906 * The implicit barrier after cmpxchg per-mm/cpu cid before loading 11907 * rq->curr->mm_cid_active matches the barrier in 11908 * sched_mm_cid_exit_signals(), sched_mm_cid_before_execve(), and 11909 * sched_mm_cid_after_execve() between store to t->mm_cid_active and 11910 * load of per-mm/cpu cid. 11911 */ 11912 11913 /* 11914 * If we observe an active task using the mm on this rq after setting 11915 * the lazy-put flag, that task will be responsible for transitioning 11916 * from lazy-put flag set to MM_CID_UNSET. 11917 */ 11918 rcu_read_lock(); 11919 t = rcu_dereference(rq->curr); 11920 if (READ_ONCE(t->mm_cid_active) && t->mm == mm) { 11921 rcu_read_unlock(); 11922 return; 11923 } 11924 rcu_read_unlock(); 11925 11926 /* 11927 * The cid is unused, so it can be unset. 11928 * Disable interrupts to keep the window of cid ownership without rq 11929 * lock small. 11930 */ 11931 local_irq_save(flags); 11932 if (try_cmpxchg(&pcpu_cid->cid, &lazy_cid, MM_CID_UNSET)) 11933 __mm_cid_put(mm, cid); 11934 local_irq_restore(flags); 11935 } 11936 11937 static void sched_mm_cid_remote_clear_old(struct mm_struct *mm, int cpu) 11938 { 11939 struct rq *rq = cpu_rq(cpu); 11940 struct mm_cid *pcpu_cid; 11941 struct task_struct *curr; 11942 u64 rq_clock; 11943 11944 /* 11945 * rq->clock load is racy on 32-bit but one spurious clear once in a 11946 * while is irrelevant. 11947 */ 11948 rq_clock = READ_ONCE(rq->clock); 11949 pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu); 11950 11951 /* 11952 * In order to take care of infrequently scheduled tasks, bump the time 11953 * snapshot associated with this cid if an active task using the mm is 11954 * observed on this rq. 11955 */ 11956 rcu_read_lock(); 11957 curr = rcu_dereference(rq->curr); 11958 if (READ_ONCE(curr->mm_cid_active) && curr->mm == mm) { 11959 WRITE_ONCE(pcpu_cid->time, rq_clock); 11960 rcu_read_unlock(); 11961 return; 11962 } 11963 rcu_read_unlock(); 11964 11965 if (rq_clock < pcpu_cid->time + SCHED_MM_CID_PERIOD_NS) 11966 return; 11967 sched_mm_cid_remote_clear(mm, pcpu_cid, cpu); 11968 } 11969 11970 static void sched_mm_cid_remote_clear_weight(struct mm_struct *mm, int cpu, 11971 int weight) 11972 { 11973 struct mm_cid *pcpu_cid; 11974 int cid; 11975 11976 pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu); 11977 cid = READ_ONCE(pcpu_cid->cid); 11978 if (!mm_cid_is_valid(cid) || cid < weight) 11979 return; 11980 sched_mm_cid_remote_clear(mm, pcpu_cid, cpu); 11981 } 11982 11983 static void task_mm_cid_work(struct callback_head *work) 11984 { 11985 unsigned long now = jiffies, old_scan, next_scan; 11986 struct task_struct *t = current; 11987 struct cpumask *cidmask; 11988 struct mm_struct *mm; 11989 int weight, cpu; 11990 11991 SCHED_WARN_ON(t != container_of(work, struct task_struct, cid_work)); 11992 11993 work->next = work; /* Prevent double-add */ 11994 if (t->flags & PF_EXITING) 11995 return; 11996 mm = t->mm; 11997 if (!mm) 11998 return; 11999 old_scan = READ_ONCE(mm->mm_cid_next_scan); 12000 next_scan = now + msecs_to_jiffies(MM_CID_SCAN_DELAY); 12001 if (!old_scan) { 12002 unsigned long res; 12003 12004 res = cmpxchg(&mm->mm_cid_next_scan, old_scan, next_scan); 12005 if (res != old_scan) 12006 old_scan = res; 12007 else 12008 old_scan = next_scan; 12009 } 12010 if (time_before(now, old_scan)) 12011 return; 12012 if (!try_cmpxchg(&mm->mm_cid_next_scan, &old_scan, next_scan)) 12013 return; 12014 cidmask = mm_cidmask(mm); 12015 /* Clear cids that were not recently used. */ 12016 for_each_possible_cpu(cpu) 12017 sched_mm_cid_remote_clear_old(mm, cpu); 12018 weight = cpumask_weight(cidmask); 12019 /* 12020 * Clear cids that are greater or equal to the cidmask weight to 12021 * recompact it. 12022 */ 12023 for_each_possible_cpu(cpu) 12024 sched_mm_cid_remote_clear_weight(mm, cpu, weight); 12025 } 12026 12027 void init_sched_mm_cid(struct task_struct *t) 12028 { 12029 struct mm_struct *mm = t->mm; 12030 int mm_users = 0; 12031 12032 if (mm) { 12033 mm_users = atomic_read(&mm->mm_users); 12034 if (mm_users == 1) 12035 mm->mm_cid_next_scan = jiffies + msecs_to_jiffies(MM_CID_SCAN_DELAY); 12036 } 12037 t->cid_work.next = &t->cid_work; /* Protect against double add */ 12038 init_task_work(&t->cid_work, task_mm_cid_work); 12039 } 12040 12041 void task_tick_mm_cid(struct rq *rq, struct task_struct *curr) 12042 { 12043 struct callback_head *work = &curr->cid_work; 12044 unsigned long now = jiffies; 12045 12046 if (!curr->mm || (curr->flags & (PF_EXITING | PF_KTHREAD)) || 12047 work->next != work) 12048 return; 12049 if (time_before(now, READ_ONCE(curr->mm->mm_cid_next_scan))) 12050 return; 12051 task_work_add(curr, work, TWA_RESUME); 12052 } 12053 12054 void sched_mm_cid_exit_signals(struct task_struct *t) 12055 { 12056 struct mm_struct *mm = t->mm; 12057 struct rq_flags rf; 12058 struct rq *rq; 12059 12060 if (!mm) 12061 return; 12062 12063 preempt_disable(); 12064 rq = this_rq(); 12065 rq_lock_irqsave(rq, &rf); 12066 preempt_enable_no_resched(); /* holding spinlock */ 12067 WRITE_ONCE(t->mm_cid_active, 0); 12068 /* 12069 * Store t->mm_cid_active before loading per-mm/cpu cid. 12070 * Matches barrier in sched_mm_cid_remote_clear_old(). 12071 */ 12072 smp_mb(); 12073 mm_cid_put(mm); 12074 t->last_mm_cid = t->mm_cid = -1; 12075 rq_unlock_irqrestore(rq, &rf); 12076 } 12077 12078 void sched_mm_cid_before_execve(struct task_struct *t) 12079 { 12080 struct mm_struct *mm = t->mm; 12081 struct rq_flags rf; 12082 struct rq *rq; 12083 12084 if (!mm) 12085 return; 12086 12087 preempt_disable(); 12088 rq = this_rq(); 12089 rq_lock_irqsave(rq, &rf); 12090 preempt_enable_no_resched(); /* holding spinlock */ 12091 WRITE_ONCE(t->mm_cid_active, 0); 12092 /* 12093 * Store t->mm_cid_active before loading per-mm/cpu cid. 12094 * Matches barrier in sched_mm_cid_remote_clear_old(). 12095 */ 12096 smp_mb(); 12097 mm_cid_put(mm); 12098 t->last_mm_cid = t->mm_cid = -1; 12099 rq_unlock_irqrestore(rq, &rf); 12100 } 12101 12102 void sched_mm_cid_after_execve(struct task_struct *t) 12103 { 12104 struct mm_struct *mm = t->mm; 12105 struct rq_flags rf; 12106 struct rq *rq; 12107 12108 if (!mm) 12109 return; 12110 12111 preempt_disable(); 12112 rq = this_rq(); 12113 rq_lock_irqsave(rq, &rf); 12114 preempt_enable_no_resched(); /* holding spinlock */ 12115 WRITE_ONCE(t->mm_cid_active, 1); 12116 /* 12117 * Store t->mm_cid_active before loading per-mm/cpu cid. 12118 * Matches barrier in sched_mm_cid_remote_clear_old(). 12119 */ 12120 smp_mb(); 12121 t->last_mm_cid = t->mm_cid = mm_cid_get(rq, mm); 12122 rq_unlock_irqrestore(rq, &rf); 12123 rseq_set_notify_resume(t); 12124 } 12125 12126 void sched_mm_cid_fork(struct task_struct *t) 12127 { 12128 WARN_ON_ONCE(!t->mm || t->mm_cid != -1); 12129 t->mm_cid_active = 1; 12130 } 12131 #endif 12132