1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * kernel/sched/core.c 4 * 5 * Core kernel scheduler code and related syscalls 6 * 7 * Copyright (C) 1991-2002 Linus Torvalds 8 */ 9 #include <linux/highmem.h> 10 #include <linux/hrtimer_api.h> 11 #include <linux/ktime_api.h> 12 #include <linux/sched/signal.h> 13 #include <linux/syscalls_api.h> 14 #include <linux/debug_locks.h> 15 #include <linux/prefetch.h> 16 #include <linux/capability.h> 17 #include <linux/pgtable_api.h> 18 #include <linux/wait_bit.h> 19 #include <linux/jiffies.h> 20 #include <linux/spinlock_api.h> 21 #include <linux/cpumask_api.h> 22 #include <linux/lockdep_api.h> 23 #include <linux/hardirq.h> 24 #include <linux/softirq.h> 25 #include <linux/refcount_api.h> 26 #include <linux/topology.h> 27 #include <linux/sched/clock.h> 28 #include <linux/sched/cond_resched.h> 29 #include <linux/sched/cputime.h> 30 #include <linux/sched/debug.h> 31 #include <linux/sched/hotplug.h> 32 #include <linux/sched/init.h> 33 #include <linux/sched/isolation.h> 34 #include <linux/sched/loadavg.h> 35 #include <linux/sched/mm.h> 36 #include <linux/sched/nohz.h> 37 #include <linux/sched/rseq_api.h> 38 #include <linux/sched/rt.h> 39 40 #include <linux/blkdev.h> 41 #include <linux/context_tracking.h> 42 #include <linux/cpuset.h> 43 #include <linux/delayacct.h> 44 #include <linux/init_task.h> 45 #include <linux/interrupt.h> 46 #include <linux/ioprio.h> 47 #include <linux/kallsyms.h> 48 #include <linux/kcov.h> 49 #include <linux/kprobes.h> 50 #include <linux/llist_api.h> 51 #include <linux/mmu_context.h> 52 #include <linux/mmzone.h> 53 #include <linux/mutex_api.h> 54 #include <linux/nmi.h> 55 #include <linux/nospec.h> 56 #include <linux/perf_event_api.h> 57 #include <linux/profile.h> 58 #include <linux/psi.h> 59 #include <linux/rcuwait_api.h> 60 #include <linux/sched/wake_q.h> 61 #include <linux/scs.h> 62 #include <linux/slab.h> 63 #include <linux/syscalls.h> 64 #include <linux/vtime.h> 65 #include <linux/wait_api.h> 66 #include <linux/workqueue_api.h> 67 68 #ifdef CONFIG_PREEMPT_DYNAMIC 69 # ifdef CONFIG_GENERIC_ENTRY 70 # include <linux/entry-common.h> 71 # endif 72 #endif 73 74 #include <uapi/linux/sched/types.h> 75 76 #include <asm/irq_regs.h> 77 #include <asm/switch_to.h> 78 #include <asm/tlb.h> 79 80 #define CREATE_TRACE_POINTS 81 #include <linux/sched/rseq_api.h> 82 #include <trace/events/sched.h> 83 #include <trace/events/ipi.h> 84 #undef CREATE_TRACE_POINTS 85 86 #include "sched.h" 87 #include "stats.h" 88 #include "autogroup.h" 89 90 #include "autogroup.h" 91 #include "pelt.h" 92 #include "smp.h" 93 #include "stats.h" 94 95 #include "../workqueue_internal.h" 96 #include "../../io_uring/io-wq.h" 97 #include "../smpboot.h" 98 99 EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpu); 100 EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpumask); 101 102 /* 103 * Export tracepoints that act as a bare tracehook (ie: have no trace event 104 * associated with them) to allow external modules to probe them. 105 */ 106 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_cfs_tp); 107 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_rt_tp); 108 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_dl_tp); 109 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_irq_tp); 110 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_se_tp); 111 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_thermal_tp); 112 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_cpu_capacity_tp); 113 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_overutilized_tp); 114 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_cfs_tp); 115 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_se_tp); 116 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_update_nr_running_tp); 117 118 DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); 119 120 #ifdef CONFIG_SCHED_DEBUG 121 /* 122 * Debugging: various feature bits 123 * 124 * If SCHED_DEBUG is disabled, each compilation unit has its own copy of 125 * sysctl_sched_features, defined in sched.h, to allow constants propagation 126 * at compile time and compiler optimization based on features default. 127 */ 128 #define SCHED_FEAT(name, enabled) \ 129 (1UL << __SCHED_FEAT_##name) * enabled | 130 const_debug unsigned int sysctl_sched_features = 131 #include "features.h" 132 0; 133 #undef SCHED_FEAT 134 135 /* 136 * Print a warning if need_resched is set for the given duration (if 137 * LATENCY_WARN is enabled). 138 * 139 * If sysctl_resched_latency_warn_once is set, only one warning will be shown 140 * per boot. 141 */ 142 __read_mostly int sysctl_resched_latency_warn_ms = 100; 143 __read_mostly int sysctl_resched_latency_warn_once = 1; 144 #endif /* CONFIG_SCHED_DEBUG */ 145 146 /* 147 * Number of tasks to iterate in a single balance run. 148 * Limited because this is done with IRQs disabled. 149 */ 150 const_debug unsigned int sysctl_sched_nr_migrate = SCHED_NR_MIGRATE_BREAK; 151 152 __read_mostly int scheduler_running; 153 154 #ifdef CONFIG_SCHED_CORE 155 156 DEFINE_STATIC_KEY_FALSE(__sched_core_enabled); 157 158 /* kernel prio, less is more */ 159 static inline int __task_prio(const struct task_struct *p) 160 { 161 if (p->sched_class == &stop_sched_class) /* trumps deadline */ 162 return -2; 163 164 if (rt_prio(p->prio)) /* includes deadline */ 165 return p->prio; /* [-1, 99] */ 166 167 if (p->sched_class == &idle_sched_class) 168 return MAX_RT_PRIO + NICE_WIDTH; /* 140 */ 169 170 return MAX_RT_PRIO + MAX_NICE; /* 120, squash fair */ 171 } 172 173 /* 174 * l(a,b) 175 * le(a,b) := !l(b,a) 176 * g(a,b) := l(b,a) 177 * ge(a,b) := !l(a,b) 178 */ 179 180 /* real prio, less is less */ 181 static inline bool prio_less(const struct task_struct *a, 182 const struct task_struct *b, bool in_fi) 183 { 184 185 int pa = __task_prio(a), pb = __task_prio(b); 186 187 if (-pa < -pb) 188 return true; 189 190 if (-pb < -pa) 191 return false; 192 193 if (pa == -1) /* dl_prio() doesn't work because of stop_class above */ 194 return !dl_time_before(a->dl.deadline, b->dl.deadline); 195 196 if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */ 197 return cfs_prio_less(a, b, in_fi); 198 199 return false; 200 } 201 202 static inline bool __sched_core_less(const struct task_struct *a, 203 const struct task_struct *b) 204 { 205 if (a->core_cookie < b->core_cookie) 206 return true; 207 208 if (a->core_cookie > b->core_cookie) 209 return false; 210 211 /* flip prio, so high prio is leftmost */ 212 if (prio_less(b, a, !!task_rq(a)->core->core_forceidle_count)) 213 return true; 214 215 return false; 216 } 217 218 #define __node_2_sc(node) rb_entry((node), struct task_struct, core_node) 219 220 static inline bool rb_sched_core_less(struct rb_node *a, const struct rb_node *b) 221 { 222 return __sched_core_less(__node_2_sc(a), __node_2_sc(b)); 223 } 224 225 static inline int rb_sched_core_cmp(const void *key, const struct rb_node *node) 226 { 227 const struct task_struct *p = __node_2_sc(node); 228 unsigned long cookie = (unsigned long)key; 229 230 if (cookie < p->core_cookie) 231 return -1; 232 233 if (cookie > p->core_cookie) 234 return 1; 235 236 return 0; 237 } 238 239 void sched_core_enqueue(struct rq *rq, struct task_struct *p) 240 { 241 rq->core->core_task_seq++; 242 243 if (!p->core_cookie) 244 return; 245 246 rb_add(&p->core_node, &rq->core_tree, rb_sched_core_less); 247 } 248 249 void sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) 250 { 251 rq->core->core_task_seq++; 252 253 if (sched_core_enqueued(p)) { 254 rb_erase(&p->core_node, &rq->core_tree); 255 RB_CLEAR_NODE(&p->core_node); 256 } 257 258 /* 259 * Migrating the last task off the cpu, with the cpu in forced idle 260 * state. Reschedule to create an accounting edge for forced idle, 261 * and re-examine whether the core is still in forced idle state. 262 */ 263 if (!(flags & DEQUEUE_SAVE) && rq->nr_running == 1 && 264 rq->core->core_forceidle_count && rq->curr == rq->idle) 265 resched_curr(rq); 266 } 267 268 static int sched_task_is_throttled(struct task_struct *p, int cpu) 269 { 270 if (p->sched_class->task_is_throttled) 271 return p->sched_class->task_is_throttled(p, cpu); 272 273 return 0; 274 } 275 276 static struct task_struct *sched_core_next(struct task_struct *p, unsigned long cookie) 277 { 278 struct rb_node *node = &p->core_node; 279 int cpu = task_cpu(p); 280 281 do { 282 node = rb_next(node); 283 if (!node) 284 return NULL; 285 286 p = __node_2_sc(node); 287 if (p->core_cookie != cookie) 288 return NULL; 289 290 } while (sched_task_is_throttled(p, cpu)); 291 292 return p; 293 } 294 295 /* 296 * Find left-most (aka, highest priority) and unthrottled task matching @cookie. 297 * If no suitable task is found, NULL will be returned. 298 */ 299 static struct task_struct *sched_core_find(struct rq *rq, unsigned long cookie) 300 { 301 struct task_struct *p; 302 struct rb_node *node; 303 304 node = rb_find_first((void *)cookie, &rq->core_tree, rb_sched_core_cmp); 305 if (!node) 306 return NULL; 307 308 p = __node_2_sc(node); 309 if (!sched_task_is_throttled(p, rq->cpu)) 310 return p; 311 312 return sched_core_next(p, cookie); 313 } 314 315 /* 316 * Magic required such that: 317 * 318 * raw_spin_rq_lock(rq); 319 * ... 320 * raw_spin_rq_unlock(rq); 321 * 322 * ends up locking and unlocking the _same_ lock, and all CPUs 323 * always agree on what rq has what lock. 324 * 325 * XXX entirely possible to selectively enable cores, don't bother for now. 326 */ 327 328 static DEFINE_MUTEX(sched_core_mutex); 329 static atomic_t sched_core_count; 330 static struct cpumask sched_core_mask; 331 332 static void sched_core_lock(int cpu, unsigned long *flags) 333 { 334 const struct cpumask *smt_mask = cpu_smt_mask(cpu); 335 int t, i = 0; 336 337 local_irq_save(*flags); 338 for_each_cpu(t, smt_mask) 339 raw_spin_lock_nested(&cpu_rq(t)->__lock, i++); 340 } 341 342 static void sched_core_unlock(int cpu, unsigned long *flags) 343 { 344 const struct cpumask *smt_mask = cpu_smt_mask(cpu); 345 int t; 346 347 for_each_cpu(t, smt_mask) 348 raw_spin_unlock(&cpu_rq(t)->__lock); 349 local_irq_restore(*flags); 350 } 351 352 static void __sched_core_flip(bool enabled) 353 { 354 unsigned long flags; 355 int cpu, t; 356 357 cpus_read_lock(); 358 359 /* 360 * Toggle the online cores, one by one. 361 */ 362 cpumask_copy(&sched_core_mask, cpu_online_mask); 363 for_each_cpu(cpu, &sched_core_mask) { 364 const struct cpumask *smt_mask = cpu_smt_mask(cpu); 365 366 sched_core_lock(cpu, &flags); 367 368 for_each_cpu(t, smt_mask) 369 cpu_rq(t)->core_enabled = enabled; 370 371 cpu_rq(cpu)->core->core_forceidle_start = 0; 372 373 sched_core_unlock(cpu, &flags); 374 375 cpumask_andnot(&sched_core_mask, &sched_core_mask, smt_mask); 376 } 377 378 /* 379 * Toggle the offline CPUs. 380 */ 381 for_each_cpu_andnot(cpu, cpu_possible_mask, cpu_online_mask) 382 cpu_rq(cpu)->core_enabled = enabled; 383 384 cpus_read_unlock(); 385 } 386 387 static void sched_core_assert_empty(void) 388 { 389 int cpu; 390 391 for_each_possible_cpu(cpu) 392 WARN_ON_ONCE(!RB_EMPTY_ROOT(&cpu_rq(cpu)->core_tree)); 393 } 394 395 static void __sched_core_enable(void) 396 { 397 static_branch_enable(&__sched_core_enabled); 398 /* 399 * Ensure all previous instances of raw_spin_rq_*lock() have finished 400 * and future ones will observe !sched_core_disabled(). 401 */ 402 synchronize_rcu(); 403 __sched_core_flip(true); 404 sched_core_assert_empty(); 405 } 406 407 static void __sched_core_disable(void) 408 { 409 sched_core_assert_empty(); 410 __sched_core_flip(false); 411 static_branch_disable(&__sched_core_enabled); 412 } 413 414 void sched_core_get(void) 415 { 416 if (atomic_inc_not_zero(&sched_core_count)) 417 return; 418 419 mutex_lock(&sched_core_mutex); 420 if (!atomic_read(&sched_core_count)) 421 __sched_core_enable(); 422 423 smp_mb__before_atomic(); 424 atomic_inc(&sched_core_count); 425 mutex_unlock(&sched_core_mutex); 426 } 427 428 static void __sched_core_put(struct work_struct *work) 429 { 430 if (atomic_dec_and_mutex_lock(&sched_core_count, &sched_core_mutex)) { 431 __sched_core_disable(); 432 mutex_unlock(&sched_core_mutex); 433 } 434 } 435 436 void sched_core_put(void) 437 { 438 static DECLARE_WORK(_work, __sched_core_put); 439 440 /* 441 * "There can be only one" 442 * 443 * Either this is the last one, or we don't actually need to do any 444 * 'work'. If it is the last *again*, we rely on 445 * WORK_STRUCT_PENDING_BIT. 446 */ 447 if (!atomic_add_unless(&sched_core_count, -1, 1)) 448 schedule_work(&_work); 449 } 450 451 #else /* !CONFIG_SCHED_CORE */ 452 453 static inline void sched_core_enqueue(struct rq *rq, struct task_struct *p) { } 454 static inline void 455 sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) { } 456 457 #endif /* CONFIG_SCHED_CORE */ 458 459 /* 460 * Serialization rules: 461 * 462 * Lock order: 463 * 464 * p->pi_lock 465 * rq->lock 466 * hrtimer_cpu_base->lock (hrtimer_start() for bandwidth controls) 467 * 468 * rq1->lock 469 * rq2->lock where: rq1 < rq2 470 * 471 * Regular state: 472 * 473 * Normal scheduling state is serialized by rq->lock. __schedule() takes the 474 * local CPU's rq->lock, it optionally removes the task from the runqueue and 475 * always looks at the local rq data structures to find the most eligible task 476 * to run next. 477 * 478 * Task enqueue is also under rq->lock, possibly taken from another CPU. 479 * Wakeups from another LLC domain might use an IPI to transfer the enqueue to 480 * the local CPU to avoid bouncing the runqueue state around [ see 481 * ttwu_queue_wakelist() ] 482 * 483 * Task wakeup, specifically wakeups that involve migration, are horribly 484 * complicated to avoid having to take two rq->locks. 485 * 486 * Special state: 487 * 488 * System-calls and anything external will use task_rq_lock() which acquires 489 * both p->pi_lock and rq->lock. As a consequence the state they change is 490 * stable while holding either lock: 491 * 492 * - sched_setaffinity()/ 493 * set_cpus_allowed_ptr(): p->cpus_ptr, p->nr_cpus_allowed 494 * - set_user_nice(): p->se.load, p->*prio 495 * - __sched_setscheduler(): p->sched_class, p->policy, p->*prio, 496 * p->se.load, p->rt_priority, 497 * p->dl.dl_{runtime, deadline, period, flags, bw, density} 498 * - sched_setnuma(): p->numa_preferred_nid 499 * - sched_move_task(): p->sched_task_group 500 * - uclamp_update_active() p->uclamp* 501 * 502 * p->state <- TASK_*: 503 * 504 * is changed locklessly using set_current_state(), __set_current_state() or 505 * set_special_state(), see their respective comments, or by 506 * try_to_wake_up(). This latter uses p->pi_lock to serialize against 507 * concurrent self. 508 * 509 * p->on_rq <- { 0, 1 = TASK_ON_RQ_QUEUED, 2 = TASK_ON_RQ_MIGRATING }: 510 * 511 * is set by activate_task() and cleared by deactivate_task(), under 512 * rq->lock. Non-zero indicates the task is runnable, the special 513 * ON_RQ_MIGRATING state is used for migration without holding both 514 * rq->locks. It indicates task_cpu() is not stable, see task_rq_lock(). 515 * 516 * p->on_cpu <- { 0, 1 }: 517 * 518 * is set by prepare_task() and cleared by finish_task() such that it will be 519 * set before p is scheduled-in and cleared after p is scheduled-out, both 520 * under rq->lock. Non-zero indicates the task is running on its CPU. 521 * 522 * [ The astute reader will observe that it is possible for two tasks on one 523 * CPU to have ->on_cpu = 1 at the same time. ] 524 * 525 * task_cpu(p): is changed by set_task_cpu(), the rules are: 526 * 527 * - Don't call set_task_cpu() on a blocked task: 528 * 529 * We don't care what CPU we're not running on, this simplifies hotplug, 530 * the CPU assignment of blocked tasks isn't required to be valid. 531 * 532 * - for try_to_wake_up(), called under p->pi_lock: 533 * 534 * This allows try_to_wake_up() to only take one rq->lock, see its comment. 535 * 536 * - for migration called under rq->lock: 537 * [ see task_on_rq_migrating() in task_rq_lock() ] 538 * 539 * o move_queued_task() 540 * o detach_task() 541 * 542 * - for migration called under double_rq_lock(): 543 * 544 * o __migrate_swap_task() 545 * o push_rt_task() / pull_rt_task() 546 * o push_dl_task() / pull_dl_task() 547 * o dl_task_offline_migration() 548 * 549 */ 550 551 void raw_spin_rq_lock_nested(struct rq *rq, int subclass) 552 { 553 raw_spinlock_t *lock; 554 555 /* Matches synchronize_rcu() in __sched_core_enable() */ 556 preempt_disable(); 557 if (sched_core_disabled()) { 558 raw_spin_lock_nested(&rq->__lock, subclass); 559 /* preempt_count *MUST* be > 1 */ 560 preempt_enable_no_resched(); 561 return; 562 } 563 564 for (;;) { 565 lock = __rq_lockp(rq); 566 raw_spin_lock_nested(lock, subclass); 567 if (likely(lock == __rq_lockp(rq))) { 568 /* preempt_count *MUST* be > 1 */ 569 preempt_enable_no_resched(); 570 return; 571 } 572 raw_spin_unlock(lock); 573 } 574 } 575 576 bool raw_spin_rq_trylock(struct rq *rq) 577 { 578 raw_spinlock_t *lock; 579 bool ret; 580 581 /* Matches synchronize_rcu() in __sched_core_enable() */ 582 preempt_disable(); 583 if (sched_core_disabled()) { 584 ret = raw_spin_trylock(&rq->__lock); 585 preempt_enable(); 586 return ret; 587 } 588 589 for (;;) { 590 lock = __rq_lockp(rq); 591 ret = raw_spin_trylock(lock); 592 if (!ret || (likely(lock == __rq_lockp(rq)))) { 593 preempt_enable(); 594 return ret; 595 } 596 raw_spin_unlock(lock); 597 } 598 } 599 600 void raw_spin_rq_unlock(struct rq *rq) 601 { 602 raw_spin_unlock(rq_lockp(rq)); 603 } 604 605 #ifdef CONFIG_SMP 606 /* 607 * double_rq_lock - safely lock two runqueues 608 */ 609 void double_rq_lock(struct rq *rq1, struct rq *rq2) 610 { 611 lockdep_assert_irqs_disabled(); 612 613 if (rq_order_less(rq2, rq1)) 614 swap(rq1, rq2); 615 616 raw_spin_rq_lock(rq1); 617 if (__rq_lockp(rq1) != __rq_lockp(rq2)) 618 raw_spin_rq_lock_nested(rq2, SINGLE_DEPTH_NESTING); 619 620 double_rq_clock_clear_update(rq1, rq2); 621 } 622 #endif 623 624 /* 625 * __task_rq_lock - lock the rq @p resides on. 626 */ 627 struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf) 628 __acquires(rq->lock) 629 { 630 struct rq *rq; 631 632 lockdep_assert_held(&p->pi_lock); 633 634 for (;;) { 635 rq = task_rq(p); 636 raw_spin_rq_lock(rq); 637 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) { 638 rq_pin_lock(rq, rf); 639 return rq; 640 } 641 raw_spin_rq_unlock(rq); 642 643 while (unlikely(task_on_rq_migrating(p))) 644 cpu_relax(); 645 } 646 } 647 648 /* 649 * task_rq_lock - lock p->pi_lock and lock the rq @p resides on. 650 */ 651 struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf) 652 __acquires(p->pi_lock) 653 __acquires(rq->lock) 654 { 655 struct rq *rq; 656 657 for (;;) { 658 raw_spin_lock_irqsave(&p->pi_lock, rf->flags); 659 rq = task_rq(p); 660 raw_spin_rq_lock(rq); 661 /* 662 * move_queued_task() task_rq_lock() 663 * 664 * ACQUIRE (rq->lock) 665 * [S] ->on_rq = MIGRATING [L] rq = task_rq() 666 * WMB (__set_task_cpu()) ACQUIRE (rq->lock); 667 * [S] ->cpu = new_cpu [L] task_rq() 668 * [L] ->on_rq 669 * RELEASE (rq->lock) 670 * 671 * If we observe the old CPU in task_rq_lock(), the acquire of 672 * the old rq->lock will fully serialize against the stores. 673 * 674 * If we observe the new CPU in task_rq_lock(), the address 675 * dependency headed by '[L] rq = task_rq()' and the acquire 676 * will pair with the WMB to ensure we then also see migrating. 677 */ 678 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) { 679 rq_pin_lock(rq, rf); 680 return rq; 681 } 682 raw_spin_rq_unlock(rq); 683 raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags); 684 685 while (unlikely(task_on_rq_migrating(p))) 686 cpu_relax(); 687 } 688 } 689 690 /* 691 * RQ-clock updating methods: 692 */ 693 694 static void update_rq_clock_task(struct rq *rq, s64 delta) 695 { 696 /* 697 * In theory, the compile should just see 0 here, and optimize out the call 698 * to sched_rt_avg_update. But I don't trust it... 699 */ 700 s64 __maybe_unused steal = 0, irq_delta = 0; 701 702 #ifdef CONFIG_IRQ_TIME_ACCOUNTING 703 irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time; 704 705 /* 706 * Since irq_time is only updated on {soft,}irq_exit, we might run into 707 * this case when a previous update_rq_clock() happened inside a 708 * {soft,}irq region. 709 * 710 * When this happens, we stop ->clock_task and only update the 711 * prev_irq_time stamp to account for the part that fit, so that a next 712 * update will consume the rest. This ensures ->clock_task is 713 * monotonic. 714 * 715 * It does however cause some slight miss-attribution of {soft,}irq 716 * time, a more accurate solution would be to update the irq_time using 717 * the current rq->clock timestamp, except that would require using 718 * atomic ops. 719 */ 720 if (irq_delta > delta) 721 irq_delta = delta; 722 723 rq->prev_irq_time += irq_delta; 724 delta -= irq_delta; 725 psi_account_irqtime(rq->curr, irq_delta); 726 delayacct_irq(rq->curr, irq_delta); 727 #endif 728 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING 729 if (static_key_false((¶virt_steal_rq_enabled))) { 730 steal = paravirt_steal_clock(cpu_of(rq)); 731 steal -= rq->prev_steal_time_rq; 732 733 if (unlikely(steal > delta)) 734 steal = delta; 735 736 rq->prev_steal_time_rq += steal; 737 delta -= steal; 738 } 739 #endif 740 741 rq->clock_task += delta; 742 743 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ 744 if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY)) 745 update_irq_load_avg(rq, irq_delta + steal); 746 #endif 747 update_rq_clock_pelt(rq, delta); 748 } 749 750 void update_rq_clock(struct rq *rq) 751 { 752 s64 delta; 753 754 lockdep_assert_rq_held(rq); 755 756 if (rq->clock_update_flags & RQCF_ACT_SKIP) 757 return; 758 759 #ifdef CONFIG_SCHED_DEBUG 760 if (sched_feat(WARN_DOUBLE_CLOCK)) 761 SCHED_WARN_ON(rq->clock_update_flags & RQCF_UPDATED); 762 rq->clock_update_flags |= RQCF_UPDATED; 763 #endif 764 765 delta = sched_clock_cpu(cpu_of(rq)) - rq->clock; 766 if (delta < 0) 767 return; 768 rq->clock += delta; 769 update_rq_clock_task(rq, delta); 770 } 771 772 #ifdef CONFIG_SCHED_HRTICK 773 /* 774 * Use HR-timers to deliver accurate preemption points. 775 */ 776 777 static void hrtick_clear(struct rq *rq) 778 { 779 if (hrtimer_active(&rq->hrtick_timer)) 780 hrtimer_cancel(&rq->hrtick_timer); 781 } 782 783 /* 784 * High-resolution timer tick. 785 * Runs from hardirq context with interrupts disabled. 786 */ 787 static enum hrtimer_restart hrtick(struct hrtimer *timer) 788 { 789 struct rq *rq = container_of(timer, struct rq, hrtick_timer); 790 struct rq_flags rf; 791 792 WARN_ON_ONCE(cpu_of(rq) != smp_processor_id()); 793 794 rq_lock(rq, &rf); 795 update_rq_clock(rq); 796 rq->curr->sched_class->task_tick(rq, rq->curr, 1); 797 rq_unlock(rq, &rf); 798 799 return HRTIMER_NORESTART; 800 } 801 802 #ifdef CONFIG_SMP 803 804 static void __hrtick_restart(struct rq *rq) 805 { 806 struct hrtimer *timer = &rq->hrtick_timer; 807 ktime_t time = rq->hrtick_time; 808 809 hrtimer_start(timer, time, HRTIMER_MODE_ABS_PINNED_HARD); 810 } 811 812 /* 813 * called from hardirq (IPI) context 814 */ 815 static void __hrtick_start(void *arg) 816 { 817 struct rq *rq = arg; 818 struct rq_flags rf; 819 820 rq_lock(rq, &rf); 821 __hrtick_restart(rq); 822 rq_unlock(rq, &rf); 823 } 824 825 /* 826 * Called to set the hrtick timer state. 827 * 828 * called with rq->lock held and irqs disabled 829 */ 830 void hrtick_start(struct rq *rq, u64 delay) 831 { 832 struct hrtimer *timer = &rq->hrtick_timer; 833 s64 delta; 834 835 /* 836 * Don't schedule slices shorter than 10000ns, that just 837 * doesn't make sense and can cause timer DoS. 838 */ 839 delta = max_t(s64, delay, 10000LL); 840 rq->hrtick_time = ktime_add_ns(timer->base->get_time(), delta); 841 842 if (rq == this_rq()) 843 __hrtick_restart(rq); 844 else 845 smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd); 846 } 847 848 #else 849 /* 850 * Called to set the hrtick timer state. 851 * 852 * called with rq->lock held and irqs disabled 853 */ 854 void hrtick_start(struct rq *rq, u64 delay) 855 { 856 /* 857 * Don't schedule slices shorter than 10000ns, that just 858 * doesn't make sense. Rely on vruntime for fairness. 859 */ 860 delay = max_t(u64, delay, 10000LL); 861 hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay), 862 HRTIMER_MODE_REL_PINNED_HARD); 863 } 864 865 #endif /* CONFIG_SMP */ 866 867 static void hrtick_rq_init(struct rq *rq) 868 { 869 #ifdef CONFIG_SMP 870 INIT_CSD(&rq->hrtick_csd, __hrtick_start, rq); 871 #endif 872 hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); 873 rq->hrtick_timer.function = hrtick; 874 } 875 #else /* CONFIG_SCHED_HRTICK */ 876 static inline void hrtick_clear(struct rq *rq) 877 { 878 } 879 880 static inline void hrtick_rq_init(struct rq *rq) 881 { 882 } 883 #endif /* CONFIG_SCHED_HRTICK */ 884 885 /* 886 * cmpxchg based fetch_or, macro so it works for different integer types 887 */ 888 #define fetch_or(ptr, mask) \ 889 ({ \ 890 typeof(ptr) _ptr = (ptr); \ 891 typeof(mask) _mask = (mask); \ 892 typeof(*_ptr) _val = *_ptr; \ 893 \ 894 do { \ 895 } while (!try_cmpxchg(_ptr, &_val, _val | _mask)); \ 896 _val; \ 897 }) 898 899 #if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG) 900 /* 901 * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG, 902 * this avoids any races wrt polling state changes and thereby avoids 903 * spurious IPIs. 904 */ 905 static inline bool set_nr_and_not_polling(struct task_struct *p) 906 { 907 struct thread_info *ti = task_thread_info(p); 908 return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG); 909 } 910 911 /* 912 * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set. 913 * 914 * If this returns true, then the idle task promises to call 915 * sched_ttwu_pending() and reschedule soon. 916 */ 917 static bool set_nr_if_polling(struct task_struct *p) 918 { 919 struct thread_info *ti = task_thread_info(p); 920 typeof(ti->flags) val = READ_ONCE(ti->flags); 921 922 for (;;) { 923 if (!(val & _TIF_POLLING_NRFLAG)) 924 return false; 925 if (val & _TIF_NEED_RESCHED) 926 return true; 927 if (try_cmpxchg(&ti->flags, &val, val | _TIF_NEED_RESCHED)) 928 break; 929 } 930 return true; 931 } 932 933 #else 934 static inline bool set_nr_and_not_polling(struct task_struct *p) 935 { 936 set_tsk_need_resched(p); 937 return true; 938 } 939 940 #ifdef CONFIG_SMP 941 static inline bool set_nr_if_polling(struct task_struct *p) 942 { 943 return false; 944 } 945 #endif 946 #endif 947 948 static bool __wake_q_add(struct wake_q_head *head, struct task_struct *task) 949 { 950 struct wake_q_node *node = &task->wake_q; 951 952 /* 953 * Atomically grab the task, if ->wake_q is !nil already it means 954 * it's already queued (either by us or someone else) and will get the 955 * wakeup due to that. 956 * 957 * In order to ensure that a pending wakeup will observe our pending 958 * state, even in the failed case, an explicit smp_mb() must be used. 959 */ 960 smp_mb__before_atomic(); 961 if (unlikely(cmpxchg_relaxed(&node->next, NULL, WAKE_Q_TAIL))) 962 return false; 963 964 /* 965 * The head is context local, there can be no concurrency. 966 */ 967 *head->lastp = node; 968 head->lastp = &node->next; 969 return true; 970 } 971 972 /** 973 * wake_q_add() - queue a wakeup for 'later' waking. 974 * @head: the wake_q_head to add @task to 975 * @task: the task to queue for 'later' wakeup 976 * 977 * Queue a task for later wakeup, most likely by the wake_up_q() call in the 978 * same context, _HOWEVER_ this is not guaranteed, the wakeup can come 979 * instantly. 980 * 981 * This function must be used as-if it were wake_up_process(); IOW the task 982 * must be ready to be woken at this location. 983 */ 984 void wake_q_add(struct wake_q_head *head, struct task_struct *task) 985 { 986 if (__wake_q_add(head, task)) 987 get_task_struct(task); 988 } 989 990 /** 991 * wake_q_add_safe() - safely queue a wakeup for 'later' waking. 992 * @head: the wake_q_head to add @task to 993 * @task: the task to queue for 'later' wakeup 994 * 995 * Queue a task for later wakeup, most likely by the wake_up_q() call in the 996 * same context, _HOWEVER_ this is not guaranteed, the wakeup can come 997 * instantly. 998 * 999 * This function must be used as-if it were wake_up_process(); IOW the task 1000 * must be ready to be woken at this location. 1001 * 1002 * This function is essentially a task-safe equivalent to wake_q_add(). Callers 1003 * that already hold reference to @task can call the 'safe' version and trust 1004 * wake_q to do the right thing depending whether or not the @task is already 1005 * queued for wakeup. 1006 */ 1007 void wake_q_add_safe(struct wake_q_head *head, struct task_struct *task) 1008 { 1009 if (!__wake_q_add(head, task)) 1010 put_task_struct(task); 1011 } 1012 1013 void wake_up_q(struct wake_q_head *head) 1014 { 1015 struct wake_q_node *node = head->first; 1016 1017 while (node != WAKE_Q_TAIL) { 1018 struct task_struct *task; 1019 1020 task = container_of(node, struct task_struct, wake_q); 1021 /* Task can safely be re-inserted now: */ 1022 node = node->next; 1023 task->wake_q.next = NULL; 1024 1025 /* 1026 * wake_up_process() executes a full barrier, which pairs with 1027 * the queueing in wake_q_add() so as not to miss wakeups. 1028 */ 1029 wake_up_process(task); 1030 put_task_struct(task); 1031 } 1032 } 1033 1034 /* 1035 * resched_curr - mark rq's current task 'to be rescheduled now'. 1036 * 1037 * On UP this means the setting of the need_resched flag, on SMP it 1038 * might also involve a cross-CPU call to trigger the scheduler on 1039 * the target CPU. 1040 */ 1041 void resched_curr(struct rq *rq) 1042 { 1043 struct task_struct *curr = rq->curr; 1044 int cpu; 1045 1046 lockdep_assert_rq_held(rq); 1047 1048 if (test_tsk_need_resched(curr)) 1049 return; 1050 1051 cpu = cpu_of(rq); 1052 1053 if (cpu == smp_processor_id()) { 1054 set_tsk_need_resched(curr); 1055 set_preempt_need_resched(); 1056 return; 1057 } 1058 1059 if (set_nr_and_not_polling(curr)) 1060 smp_send_reschedule(cpu); 1061 else 1062 trace_sched_wake_idle_without_ipi(cpu); 1063 } 1064 1065 void resched_cpu(int cpu) 1066 { 1067 struct rq *rq = cpu_rq(cpu); 1068 unsigned long flags; 1069 1070 raw_spin_rq_lock_irqsave(rq, flags); 1071 if (cpu_online(cpu) || cpu == smp_processor_id()) 1072 resched_curr(rq); 1073 raw_spin_rq_unlock_irqrestore(rq, flags); 1074 } 1075 1076 #ifdef CONFIG_SMP 1077 #ifdef CONFIG_NO_HZ_COMMON 1078 /* 1079 * In the semi idle case, use the nearest busy CPU for migrating timers 1080 * from an idle CPU. This is good for power-savings. 1081 * 1082 * We don't do similar optimization for completely idle system, as 1083 * selecting an idle CPU will add more delays to the timers than intended 1084 * (as that CPU's timer base may not be uptodate wrt jiffies etc). 1085 */ 1086 int get_nohz_timer_target(void) 1087 { 1088 int i, cpu = smp_processor_id(), default_cpu = -1; 1089 struct sched_domain *sd; 1090 const struct cpumask *hk_mask; 1091 1092 if (housekeeping_cpu(cpu, HK_TYPE_TIMER)) { 1093 if (!idle_cpu(cpu)) 1094 return cpu; 1095 default_cpu = cpu; 1096 } 1097 1098 hk_mask = housekeeping_cpumask(HK_TYPE_TIMER); 1099 1100 rcu_read_lock(); 1101 for_each_domain(cpu, sd) { 1102 for_each_cpu_and(i, sched_domain_span(sd), hk_mask) { 1103 if (cpu == i) 1104 continue; 1105 1106 if (!idle_cpu(i)) { 1107 cpu = i; 1108 goto unlock; 1109 } 1110 } 1111 } 1112 1113 if (default_cpu == -1) 1114 default_cpu = housekeeping_any_cpu(HK_TYPE_TIMER); 1115 cpu = default_cpu; 1116 unlock: 1117 rcu_read_unlock(); 1118 return cpu; 1119 } 1120 1121 /* 1122 * When add_timer_on() enqueues a timer into the timer wheel of an 1123 * idle CPU then this timer might expire before the next timer event 1124 * which is scheduled to wake up that CPU. In case of a completely 1125 * idle system the next event might even be infinite time into the 1126 * future. wake_up_idle_cpu() ensures that the CPU is woken up and 1127 * leaves the inner idle loop so the newly added timer is taken into 1128 * account when the CPU goes back to idle and evaluates the timer 1129 * wheel for the next timer event. 1130 */ 1131 static void wake_up_idle_cpu(int cpu) 1132 { 1133 struct rq *rq = cpu_rq(cpu); 1134 1135 if (cpu == smp_processor_id()) 1136 return; 1137 1138 if (set_nr_and_not_polling(rq->idle)) 1139 smp_send_reschedule(cpu); 1140 else 1141 trace_sched_wake_idle_without_ipi(cpu); 1142 } 1143 1144 static bool wake_up_full_nohz_cpu(int cpu) 1145 { 1146 /* 1147 * We just need the target to call irq_exit() and re-evaluate 1148 * the next tick. The nohz full kick at least implies that. 1149 * If needed we can still optimize that later with an 1150 * empty IRQ. 1151 */ 1152 if (cpu_is_offline(cpu)) 1153 return true; /* Don't try to wake offline CPUs. */ 1154 if (tick_nohz_full_cpu(cpu)) { 1155 if (cpu != smp_processor_id() || 1156 tick_nohz_tick_stopped()) 1157 tick_nohz_full_kick_cpu(cpu); 1158 return true; 1159 } 1160 1161 return false; 1162 } 1163 1164 /* 1165 * Wake up the specified CPU. If the CPU is going offline, it is the 1166 * caller's responsibility to deal with the lost wakeup, for example, 1167 * by hooking into the CPU_DEAD notifier like timers and hrtimers do. 1168 */ 1169 void wake_up_nohz_cpu(int cpu) 1170 { 1171 if (!wake_up_full_nohz_cpu(cpu)) 1172 wake_up_idle_cpu(cpu); 1173 } 1174 1175 static void nohz_csd_func(void *info) 1176 { 1177 struct rq *rq = info; 1178 int cpu = cpu_of(rq); 1179 unsigned int flags; 1180 1181 /* 1182 * Release the rq::nohz_csd. 1183 */ 1184 flags = atomic_fetch_andnot(NOHZ_KICK_MASK | NOHZ_NEWILB_KICK, nohz_flags(cpu)); 1185 WARN_ON(!(flags & NOHZ_KICK_MASK)); 1186 1187 rq->idle_balance = idle_cpu(cpu); 1188 if (rq->idle_balance && !need_resched()) { 1189 rq->nohz_idle_balance = flags; 1190 raise_softirq_irqoff(SCHED_SOFTIRQ); 1191 } 1192 } 1193 1194 #endif /* CONFIG_NO_HZ_COMMON */ 1195 1196 #ifdef CONFIG_NO_HZ_FULL 1197 bool sched_can_stop_tick(struct rq *rq) 1198 { 1199 int fifo_nr_running; 1200 1201 /* Deadline tasks, even if single, need the tick */ 1202 if (rq->dl.dl_nr_running) 1203 return false; 1204 1205 /* 1206 * If there are more than one RR tasks, we need the tick to affect the 1207 * actual RR behaviour. 1208 */ 1209 if (rq->rt.rr_nr_running) { 1210 if (rq->rt.rr_nr_running == 1) 1211 return true; 1212 else 1213 return false; 1214 } 1215 1216 /* 1217 * If there's no RR tasks, but FIFO tasks, we can skip the tick, no 1218 * forced preemption between FIFO tasks. 1219 */ 1220 fifo_nr_running = rq->rt.rt_nr_running - rq->rt.rr_nr_running; 1221 if (fifo_nr_running) 1222 return true; 1223 1224 /* 1225 * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left; 1226 * if there's more than one we need the tick for involuntary 1227 * preemption. 1228 */ 1229 if (rq->nr_running > 1) 1230 return false; 1231 1232 return true; 1233 } 1234 #endif /* CONFIG_NO_HZ_FULL */ 1235 #endif /* CONFIG_SMP */ 1236 1237 #if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \ 1238 (defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH))) 1239 /* 1240 * Iterate task_group tree rooted at *from, calling @down when first entering a 1241 * node and @up when leaving it for the final time. 1242 * 1243 * Caller must hold rcu_lock or sufficient equivalent. 1244 */ 1245 int walk_tg_tree_from(struct task_group *from, 1246 tg_visitor down, tg_visitor up, void *data) 1247 { 1248 struct task_group *parent, *child; 1249 int ret; 1250 1251 parent = from; 1252 1253 down: 1254 ret = (*down)(parent, data); 1255 if (ret) 1256 goto out; 1257 list_for_each_entry_rcu(child, &parent->children, siblings) { 1258 parent = child; 1259 goto down; 1260 1261 up: 1262 continue; 1263 } 1264 ret = (*up)(parent, data); 1265 if (ret || parent == from) 1266 goto out; 1267 1268 child = parent; 1269 parent = parent->parent; 1270 if (parent) 1271 goto up; 1272 out: 1273 return ret; 1274 } 1275 1276 int tg_nop(struct task_group *tg, void *data) 1277 { 1278 return 0; 1279 } 1280 #endif 1281 1282 static void set_load_weight(struct task_struct *p, bool update_load) 1283 { 1284 int prio = p->static_prio - MAX_RT_PRIO; 1285 struct load_weight *load = &p->se.load; 1286 1287 /* 1288 * SCHED_IDLE tasks get minimal weight: 1289 */ 1290 if (task_has_idle_policy(p)) { 1291 load->weight = scale_load(WEIGHT_IDLEPRIO); 1292 load->inv_weight = WMULT_IDLEPRIO; 1293 return; 1294 } 1295 1296 /* 1297 * SCHED_OTHER tasks have to update their load when changing their 1298 * weight 1299 */ 1300 if (update_load && p->sched_class == &fair_sched_class) { 1301 reweight_task(p, prio); 1302 } else { 1303 load->weight = scale_load(sched_prio_to_weight[prio]); 1304 load->inv_weight = sched_prio_to_wmult[prio]; 1305 } 1306 } 1307 1308 #ifdef CONFIG_UCLAMP_TASK 1309 /* 1310 * Serializes updates of utilization clamp values 1311 * 1312 * The (slow-path) user-space triggers utilization clamp value updates which 1313 * can require updates on (fast-path) scheduler's data structures used to 1314 * support enqueue/dequeue operations. 1315 * While the per-CPU rq lock protects fast-path update operations, user-space 1316 * requests are serialized using a mutex to reduce the risk of conflicting 1317 * updates or API abuses. 1318 */ 1319 static DEFINE_MUTEX(uclamp_mutex); 1320 1321 /* Max allowed minimum utilization */ 1322 static unsigned int __maybe_unused sysctl_sched_uclamp_util_min = SCHED_CAPACITY_SCALE; 1323 1324 /* Max allowed maximum utilization */ 1325 static unsigned int __maybe_unused sysctl_sched_uclamp_util_max = SCHED_CAPACITY_SCALE; 1326 1327 /* 1328 * By default RT tasks run at the maximum performance point/capacity of the 1329 * system. Uclamp enforces this by always setting UCLAMP_MIN of RT tasks to 1330 * SCHED_CAPACITY_SCALE. 1331 * 1332 * This knob allows admins to change the default behavior when uclamp is being 1333 * used. In battery powered devices, particularly, running at the maximum 1334 * capacity and frequency will increase energy consumption and shorten the 1335 * battery life. 1336 * 1337 * This knob only affects RT tasks that their uclamp_se->user_defined == false. 1338 * 1339 * This knob will not override the system default sched_util_clamp_min defined 1340 * above. 1341 */ 1342 static unsigned int sysctl_sched_uclamp_util_min_rt_default = SCHED_CAPACITY_SCALE; 1343 1344 /* All clamps are required to be less or equal than these values */ 1345 static struct uclamp_se uclamp_default[UCLAMP_CNT]; 1346 1347 /* 1348 * This static key is used to reduce the uclamp overhead in the fast path. It 1349 * primarily disables the call to uclamp_rq_{inc, dec}() in 1350 * enqueue/dequeue_task(). 1351 * 1352 * This allows users to continue to enable uclamp in their kernel config with 1353 * minimum uclamp overhead in the fast path. 1354 * 1355 * As soon as userspace modifies any of the uclamp knobs, the static key is 1356 * enabled, since we have an actual users that make use of uclamp 1357 * functionality. 1358 * 1359 * The knobs that would enable this static key are: 1360 * 1361 * * A task modifying its uclamp value with sched_setattr(). 1362 * * An admin modifying the sysctl_sched_uclamp_{min, max} via procfs. 1363 * * An admin modifying the cgroup cpu.uclamp.{min, max} 1364 */ 1365 DEFINE_STATIC_KEY_FALSE(sched_uclamp_used); 1366 1367 /* Integer rounded range for each bucket */ 1368 #define UCLAMP_BUCKET_DELTA DIV_ROUND_CLOSEST(SCHED_CAPACITY_SCALE, UCLAMP_BUCKETS) 1369 1370 #define for_each_clamp_id(clamp_id) \ 1371 for ((clamp_id) = 0; (clamp_id) < UCLAMP_CNT; (clamp_id)++) 1372 1373 static inline unsigned int uclamp_bucket_id(unsigned int clamp_value) 1374 { 1375 return min_t(unsigned int, clamp_value / UCLAMP_BUCKET_DELTA, UCLAMP_BUCKETS - 1); 1376 } 1377 1378 static inline unsigned int uclamp_none(enum uclamp_id clamp_id) 1379 { 1380 if (clamp_id == UCLAMP_MIN) 1381 return 0; 1382 return SCHED_CAPACITY_SCALE; 1383 } 1384 1385 static inline void uclamp_se_set(struct uclamp_se *uc_se, 1386 unsigned int value, bool user_defined) 1387 { 1388 uc_se->value = value; 1389 uc_se->bucket_id = uclamp_bucket_id(value); 1390 uc_se->user_defined = user_defined; 1391 } 1392 1393 static inline unsigned int 1394 uclamp_idle_value(struct rq *rq, enum uclamp_id clamp_id, 1395 unsigned int clamp_value) 1396 { 1397 /* 1398 * Avoid blocked utilization pushing up the frequency when we go 1399 * idle (which drops the max-clamp) by retaining the last known 1400 * max-clamp. 1401 */ 1402 if (clamp_id == UCLAMP_MAX) { 1403 rq->uclamp_flags |= UCLAMP_FLAG_IDLE; 1404 return clamp_value; 1405 } 1406 1407 return uclamp_none(UCLAMP_MIN); 1408 } 1409 1410 static inline void uclamp_idle_reset(struct rq *rq, enum uclamp_id clamp_id, 1411 unsigned int clamp_value) 1412 { 1413 /* Reset max-clamp retention only on idle exit */ 1414 if (!(rq->uclamp_flags & UCLAMP_FLAG_IDLE)) 1415 return; 1416 1417 uclamp_rq_set(rq, clamp_id, clamp_value); 1418 } 1419 1420 static inline 1421 unsigned int uclamp_rq_max_value(struct rq *rq, enum uclamp_id clamp_id, 1422 unsigned int clamp_value) 1423 { 1424 struct uclamp_bucket *bucket = rq->uclamp[clamp_id].bucket; 1425 int bucket_id = UCLAMP_BUCKETS - 1; 1426 1427 /* 1428 * Since both min and max clamps are max aggregated, find the 1429 * top most bucket with tasks in. 1430 */ 1431 for ( ; bucket_id >= 0; bucket_id--) { 1432 if (!bucket[bucket_id].tasks) 1433 continue; 1434 return bucket[bucket_id].value; 1435 } 1436 1437 /* No tasks -- default clamp values */ 1438 return uclamp_idle_value(rq, clamp_id, clamp_value); 1439 } 1440 1441 static void __uclamp_update_util_min_rt_default(struct task_struct *p) 1442 { 1443 unsigned int default_util_min; 1444 struct uclamp_se *uc_se; 1445 1446 lockdep_assert_held(&p->pi_lock); 1447 1448 uc_se = &p->uclamp_req[UCLAMP_MIN]; 1449 1450 /* Only sync if user didn't override the default */ 1451 if (uc_se->user_defined) 1452 return; 1453 1454 default_util_min = sysctl_sched_uclamp_util_min_rt_default; 1455 uclamp_se_set(uc_se, default_util_min, false); 1456 } 1457 1458 static void uclamp_update_util_min_rt_default(struct task_struct *p) 1459 { 1460 struct rq_flags rf; 1461 struct rq *rq; 1462 1463 if (!rt_task(p)) 1464 return; 1465 1466 /* Protect updates to p->uclamp_* */ 1467 rq = task_rq_lock(p, &rf); 1468 __uclamp_update_util_min_rt_default(p); 1469 task_rq_unlock(rq, p, &rf); 1470 } 1471 1472 static inline struct uclamp_se 1473 uclamp_tg_restrict(struct task_struct *p, enum uclamp_id clamp_id) 1474 { 1475 /* Copy by value as we could modify it */ 1476 struct uclamp_se uc_req = p->uclamp_req[clamp_id]; 1477 #ifdef CONFIG_UCLAMP_TASK_GROUP 1478 unsigned int tg_min, tg_max, value; 1479 1480 /* 1481 * Tasks in autogroups or root task group will be 1482 * restricted by system defaults. 1483 */ 1484 if (task_group_is_autogroup(task_group(p))) 1485 return uc_req; 1486 if (task_group(p) == &root_task_group) 1487 return uc_req; 1488 1489 tg_min = task_group(p)->uclamp[UCLAMP_MIN].value; 1490 tg_max = task_group(p)->uclamp[UCLAMP_MAX].value; 1491 value = uc_req.value; 1492 value = clamp(value, tg_min, tg_max); 1493 uclamp_se_set(&uc_req, value, false); 1494 #endif 1495 1496 return uc_req; 1497 } 1498 1499 /* 1500 * The effective clamp bucket index of a task depends on, by increasing 1501 * priority: 1502 * - the task specific clamp value, when explicitly requested from userspace 1503 * - the task group effective clamp value, for tasks not either in the root 1504 * group or in an autogroup 1505 * - the system default clamp value, defined by the sysadmin 1506 */ 1507 static inline struct uclamp_se 1508 uclamp_eff_get(struct task_struct *p, enum uclamp_id clamp_id) 1509 { 1510 struct uclamp_se uc_req = uclamp_tg_restrict(p, clamp_id); 1511 struct uclamp_se uc_max = uclamp_default[clamp_id]; 1512 1513 /* System default restrictions always apply */ 1514 if (unlikely(uc_req.value > uc_max.value)) 1515 return uc_max; 1516 1517 return uc_req; 1518 } 1519 1520 unsigned long uclamp_eff_value(struct task_struct *p, enum uclamp_id clamp_id) 1521 { 1522 struct uclamp_se uc_eff; 1523 1524 /* Task currently refcounted: use back-annotated (effective) value */ 1525 if (p->uclamp[clamp_id].active) 1526 return (unsigned long)p->uclamp[clamp_id].value; 1527 1528 uc_eff = uclamp_eff_get(p, clamp_id); 1529 1530 return (unsigned long)uc_eff.value; 1531 } 1532 1533 /* 1534 * When a task is enqueued on a rq, the clamp bucket currently defined by the 1535 * task's uclamp::bucket_id is refcounted on that rq. This also immediately 1536 * updates the rq's clamp value if required. 1537 * 1538 * Tasks can have a task-specific value requested from user-space, track 1539 * within each bucket the maximum value for tasks refcounted in it. 1540 * This "local max aggregation" allows to track the exact "requested" value 1541 * for each bucket when all its RUNNABLE tasks require the same clamp. 1542 */ 1543 static inline void uclamp_rq_inc_id(struct rq *rq, struct task_struct *p, 1544 enum uclamp_id clamp_id) 1545 { 1546 struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id]; 1547 struct uclamp_se *uc_se = &p->uclamp[clamp_id]; 1548 struct uclamp_bucket *bucket; 1549 1550 lockdep_assert_rq_held(rq); 1551 1552 /* Update task effective clamp */ 1553 p->uclamp[clamp_id] = uclamp_eff_get(p, clamp_id); 1554 1555 bucket = &uc_rq->bucket[uc_se->bucket_id]; 1556 bucket->tasks++; 1557 uc_se->active = true; 1558 1559 uclamp_idle_reset(rq, clamp_id, uc_se->value); 1560 1561 /* 1562 * Local max aggregation: rq buckets always track the max 1563 * "requested" clamp value of its RUNNABLE tasks. 1564 */ 1565 if (bucket->tasks == 1 || uc_se->value > bucket->value) 1566 bucket->value = uc_se->value; 1567 1568 if (uc_se->value > uclamp_rq_get(rq, clamp_id)) 1569 uclamp_rq_set(rq, clamp_id, uc_se->value); 1570 } 1571 1572 /* 1573 * When a task is dequeued from a rq, the clamp bucket refcounted by the task 1574 * is released. If this is the last task reference counting the rq's max 1575 * active clamp value, then the rq's clamp value is updated. 1576 * 1577 * Both refcounted tasks and rq's cached clamp values are expected to be 1578 * always valid. If it's detected they are not, as defensive programming, 1579 * enforce the expected state and warn. 1580 */ 1581 static inline void uclamp_rq_dec_id(struct rq *rq, struct task_struct *p, 1582 enum uclamp_id clamp_id) 1583 { 1584 struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id]; 1585 struct uclamp_se *uc_se = &p->uclamp[clamp_id]; 1586 struct uclamp_bucket *bucket; 1587 unsigned int bkt_clamp; 1588 unsigned int rq_clamp; 1589 1590 lockdep_assert_rq_held(rq); 1591 1592 /* 1593 * If sched_uclamp_used was enabled after task @p was enqueued, 1594 * we could end up with unbalanced call to uclamp_rq_dec_id(). 1595 * 1596 * In this case the uc_se->active flag should be false since no uclamp 1597 * accounting was performed at enqueue time and we can just return 1598 * here. 1599 * 1600 * Need to be careful of the following enqueue/dequeue ordering 1601 * problem too 1602 * 1603 * enqueue(taskA) 1604 * // sched_uclamp_used gets enabled 1605 * enqueue(taskB) 1606 * dequeue(taskA) 1607 * // Must not decrement bucket->tasks here 1608 * dequeue(taskB) 1609 * 1610 * where we could end up with stale data in uc_se and 1611 * bucket[uc_se->bucket_id]. 1612 * 1613 * The following check here eliminates the possibility of such race. 1614 */ 1615 if (unlikely(!uc_se->active)) 1616 return; 1617 1618 bucket = &uc_rq->bucket[uc_se->bucket_id]; 1619 1620 SCHED_WARN_ON(!bucket->tasks); 1621 if (likely(bucket->tasks)) 1622 bucket->tasks--; 1623 1624 uc_se->active = false; 1625 1626 /* 1627 * Keep "local max aggregation" simple and accept to (possibly) 1628 * overboost some RUNNABLE tasks in the same bucket. 1629 * The rq clamp bucket value is reset to its base value whenever 1630 * there are no more RUNNABLE tasks refcounting it. 1631 */ 1632 if (likely(bucket->tasks)) 1633 return; 1634 1635 rq_clamp = uclamp_rq_get(rq, clamp_id); 1636 /* 1637 * Defensive programming: this should never happen. If it happens, 1638 * e.g. due to future modification, warn and fixup the expected value. 1639 */ 1640 SCHED_WARN_ON(bucket->value > rq_clamp); 1641 if (bucket->value >= rq_clamp) { 1642 bkt_clamp = uclamp_rq_max_value(rq, clamp_id, uc_se->value); 1643 uclamp_rq_set(rq, clamp_id, bkt_clamp); 1644 } 1645 } 1646 1647 static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p) 1648 { 1649 enum uclamp_id clamp_id; 1650 1651 /* 1652 * Avoid any overhead until uclamp is actually used by the userspace. 1653 * 1654 * The condition is constructed such that a NOP is generated when 1655 * sched_uclamp_used is disabled. 1656 */ 1657 if (!static_branch_unlikely(&sched_uclamp_used)) 1658 return; 1659 1660 if (unlikely(!p->sched_class->uclamp_enabled)) 1661 return; 1662 1663 for_each_clamp_id(clamp_id) 1664 uclamp_rq_inc_id(rq, p, clamp_id); 1665 1666 /* Reset clamp idle holding when there is one RUNNABLE task */ 1667 if (rq->uclamp_flags & UCLAMP_FLAG_IDLE) 1668 rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE; 1669 } 1670 1671 static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) 1672 { 1673 enum uclamp_id clamp_id; 1674 1675 /* 1676 * Avoid any overhead until uclamp is actually used by the userspace. 1677 * 1678 * The condition is constructed such that a NOP is generated when 1679 * sched_uclamp_used is disabled. 1680 */ 1681 if (!static_branch_unlikely(&sched_uclamp_used)) 1682 return; 1683 1684 if (unlikely(!p->sched_class->uclamp_enabled)) 1685 return; 1686 1687 for_each_clamp_id(clamp_id) 1688 uclamp_rq_dec_id(rq, p, clamp_id); 1689 } 1690 1691 static inline void uclamp_rq_reinc_id(struct rq *rq, struct task_struct *p, 1692 enum uclamp_id clamp_id) 1693 { 1694 if (!p->uclamp[clamp_id].active) 1695 return; 1696 1697 uclamp_rq_dec_id(rq, p, clamp_id); 1698 uclamp_rq_inc_id(rq, p, clamp_id); 1699 1700 /* 1701 * Make sure to clear the idle flag if we've transiently reached 0 1702 * active tasks on rq. 1703 */ 1704 if (clamp_id == UCLAMP_MAX && (rq->uclamp_flags & UCLAMP_FLAG_IDLE)) 1705 rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE; 1706 } 1707 1708 static inline void 1709 uclamp_update_active(struct task_struct *p) 1710 { 1711 enum uclamp_id clamp_id; 1712 struct rq_flags rf; 1713 struct rq *rq; 1714 1715 /* 1716 * Lock the task and the rq where the task is (or was) queued. 1717 * 1718 * We might lock the (previous) rq of a !RUNNABLE task, but that's the 1719 * price to pay to safely serialize util_{min,max} updates with 1720 * enqueues, dequeues and migration operations. 1721 * This is the same locking schema used by __set_cpus_allowed_ptr(). 1722 */ 1723 rq = task_rq_lock(p, &rf); 1724 1725 /* 1726 * Setting the clamp bucket is serialized by task_rq_lock(). 1727 * If the task is not yet RUNNABLE and its task_struct is not 1728 * affecting a valid clamp bucket, the next time it's enqueued, 1729 * it will already see the updated clamp bucket value. 1730 */ 1731 for_each_clamp_id(clamp_id) 1732 uclamp_rq_reinc_id(rq, p, clamp_id); 1733 1734 task_rq_unlock(rq, p, &rf); 1735 } 1736 1737 #ifdef CONFIG_UCLAMP_TASK_GROUP 1738 static inline void 1739 uclamp_update_active_tasks(struct cgroup_subsys_state *css) 1740 { 1741 struct css_task_iter it; 1742 struct task_struct *p; 1743 1744 css_task_iter_start(css, 0, &it); 1745 while ((p = css_task_iter_next(&it))) 1746 uclamp_update_active(p); 1747 css_task_iter_end(&it); 1748 } 1749 1750 static void cpu_util_update_eff(struct cgroup_subsys_state *css); 1751 #endif 1752 1753 #ifdef CONFIG_SYSCTL 1754 #ifdef CONFIG_UCLAMP_TASK 1755 #ifdef CONFIG_UCLAMP_TASK_GROUP 1756 static void uclamp_update_root_tg(void) 1757 { 1758 struct task_group *tg = &root_task_group; 1759 1760 uclamp_se_set(&tg->uclamp_req[UCLAMP_MIN], 1761 sysctl_sched_uclamp_util_min, false); 1762 uclamp_se_set(&tg->uclamp_req[UCLAMP_MAX], 1763 sysctl_sched_uclamp_util_max, false); 1764 1765 rcu_read_lock(); 1766 cpu_util_update_eff(&root_task_group.css); 1767 rcu_read_unlock(); 1768 } 1769 #else 1770 static void uclamp_update_root_tg(void) { } 1771 #endif 1772 1773 static void uclamp_sync_util_min_rt_default(void) 1774 { 1775 struct task_struct *g, *p; 1776 1777 /* 1778 * copy_process() sysctl_uclamp 1779 * uclamp_min_rt = X; 1780 * write_lock(&tasklist_lock) read_lock(&tasklist_lock) 1781 * // link thread smp_mb__after_spinlock() 1782 * write_unlock(&tasklist_lock) read_unlock(&tasklist_lock); 1783 * sched_post_fork() for_each_process_thread() 1784 * __uclamp_sync_rt() __uclamp_sync_rt() 1785 * 1786 * Ensures that either sched_post_fork() will observe the new 1787 * uclamp_min_rt or for_each_process_thread() will observe the new 1788 * task. 1789 */ 1790 read_lock(&tasklist_lock); 1791 smp_mb__after_spinlock(); 1792 read_unlock(&tasklist_lock); 1793 1794 rcu_read_lock(); 1795 for_each_process_thread(g, p) 1796 uclamp_update_util_min_rt_default(p); 1797 rcu_read_unlock(); 1798 } 1799 1800 static int sysctl_sched_uclamp_handler(struct ctl_table *table, int write, 1801 void *buffer, size_t *lenp, loff_t *ppos) 1802 { 1803 bool update_root_tg = false; 1804 int old_min, old_max, old_min_rt; 1805 int result; 1806 1807 mutex_lock(&uclamp_mutex); 1808 old_min = sysctl_sched_uclamp_util_min; 1809 old_max = sysctl_sched_uclamp_util_max; 1810 old_min_rt = sysctl_sched_uclamp_util_min_rt_default; 1811 1812 result = proc_dointvec(table, write, buffer, lenp, ppos); 1813 if (result) 1814 goto undo; 1815 if (!write) 1816 goto done; 1817 1818 if (sysctl_sched_uclamp_util_min > sysctl_sched_uclamp_util_max || 1819 sysctl_sched_uclamp_util_max > SCHED_CAPACITY_SCALE || 1820 sysctl_sched_uclamp_util_min_rt_default > SCHED_CAPACITY_SCALE) { 1821 1822 result = -EINVAL; 1823 goto undo; 1824 } 1825 1826 if (old_min != sysctl_sched_uclamp_util_min) { 1827 uclamp_se_set(&uclamp_default[UCLAMP_MIN], 1828 sysctl_sched_uclamp_util_min, false); 1829 update_root_tg = true; 1830 } 1831 if (old_max != sysctl_sched_uclamp_util_max) { 1832 uclamp_se_set(&uclamp_default[UCLAMP_MAX], 1833 sysctl_sched_uclamp_util_max, false); 1834 update_root_tg = true; 1835 } 1836 1837 if (update_root_tg) { 1838 static_branch_enable(&sched_uclamp_used); 1839 uclamp_update_root_tg(); 1840 } 1841 1842 if (old_min_rt != sysctl_sched_uclamp_util_min_rt_default) { 1843 static_branch_enable(&sched_uclamp_used); 1844 uclamp_sync_util_min_rt_default(); 1845 } 1846 1847 /* 1848 * We update all RUNNABLE tasks only when task groups are in use. 1849 * Otherwise, keep it simple and do just a lazy update at each next 1850 * task enqueue time. 1851 */ 1852 1853 goto done; 1854 1855 undo: 1856 sysctl_sched_uclamp_util_min = old_min; 1857 sysctl_sched_uclamp_util_max = old_max; 1858 sysctl_sched_uclamp_util_min_rt_default = old_min_rt; 1859 done: 1860 mutex_unlock(&uclamp_mutex); 1861 1862 return result; 1863 } 1864 #endif 1865 #endif 1866 1867 static int uclamp_validate(struct task_struct *p, 1868 const struct sched_attr *attr) 1869 { 1870 int util_min = p->uclamp_req[UCLAMP_MIN].value; 1871 int util_max = p->uclamp_req[UCLAMP_MAX].value; 1872 1873 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MIN) { 1874 util_min = attr->sched_util_min; 1875 1876 if (util_min + 1 > SCHED_CAPACITY_SCALE + 1) 1877 return -EINVAL; 1878 } 1879 1880 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MAX) { 1881 util_max = attr->sched_util_max; 1882 1883 if (util_max + 1 > SCHED_CAPACITY_SCALE + 1) 1884 return -EINVAL; 1885 } 1886 1887 if (util_min != -1 && util_max != -1 && util_min > util_max) 1888 return -EINVAL; 1889 1890 /* 1891 * We have valid uclamp attributes; make sure uclamp is enabled. 1892 * 1893 * We need to do that here, because enabling static branches is a 1894 * blocking operation which obviously cannot be done while holding 1895 * scheduler locks. 1896 */ 1897 static_branch_enable(&sched_uclamp_used); 1898 1899 return 0; 1900 } 1901 1902 static bool uclamp_reset(const struct sched_attr *attr, 1903 enum uclamp_id clamp_id, 1904 struct uclamp_se *uc_se) 1905 { 1906 /* Reset on sched class change for a non user-defined clamp value. */ 1907 if (likely(!(attr->sched_flags & SCHED_FLAG_UTIL_CLAMP)) && 1908 !uc_se->user_defined) 1909 return true; 1910 1911 /* Reset on sched_util_{min,max} == -1. */ 1912 if (clamp_id == UCLAMP_MIN && 1913 attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MIN && 1914 attr->sched_util_min == -1) { 1915 return true; 1916 } 1917 1918 if (clamp_id == UCLAMP_MAX && 1919 attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MAX && 1920 attr->sched_util_max == -1) { 1921 return true; 1922 } 1923 1924 return false; 1925 } 1926 1927 static void __setscheduler_uclamp(struct task_struct *p, 1928 const struct sched_attr *attr) 1929 { 1930 enum uclamp_id clamp_id; 1931 1932 for_each_clamp_id(clamp_id) { 1933 struct uclamp_se *uc_se = &p->uclamp_req[clamp_id]; 1934 unsigned int value; 1935 1936 if (!uclamp_reset(attr, clamp_id, uc_se)) 1937 continue; 1938 1939 /* 1940 * RT by default have a 100% boost value that could be modified 1941 * at runtime. 1942 */ 1943 if (unlikely(rt_task(p) && clamp_id == UCLAMP_MIN)) 1944 value = sysctl_sched_uclamp_util_min_rt_default; 1945 else 1946 value = uclamp_none(clamp_id); 1947 1948 uclamp_se_set(uc_se, value, false); 1949 1950 } 1951 1952 if (likely(!(attr->sched_flags & SCHED_FLAG_UTIL_CLAMP))) 1953 return; 1954 1955 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MIN && 1956 attr->sched_util_min != -1) { 1957 uclamp_se_set(&p->uclamp_req[UCLAMP_MIN], 1958 attr->sched_util_min, true); 1959 } 1960 1961 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MAX && 1962 attr->sched_util_max != -1) { 1963 uclamp_se_set(&p->uclamp_req[UCLAMP_MAX], 1964 attr->sched_util_max, true); 1965 } 1966 } 1967 1968 static void uclamp_fork(struct task_struct *p) 1969 { 1970 enum uclamp_id clamp_id; 1971 1972 /* 1973 * We don't need to hold task_rq_lock() when updating p->uclamp_* here 1974 * as the task is still at its early fork stages. 1975 */ 1976 for_each_clamp_id(clamp_id) 1977 p->uclamp[clamp_id].active = false; 1978 1979 if (likely(!p->sched_reset_on_fork)) 1980 return; 1981 1982 for_each_clamp_id(clamp_id) { 1983 uclamp_se_set(&p->uclamp_req[clamp_id], 1984 uclamp_none(clamp_id), false); 1985 } 1986 } 1987 1988 static void uclamp_post_fork(struct task_struct *p) 1989 { 1990 uclamp_update_util_min_rt_default(p); 1991 } 1992 1993 static void __init init_uclamp_rq(struct rq *rq) 1994 { 1995 enum uclamp_id clamp_id; 1996 struct uclamp_rq *uc_rq = rq->uclamp; 1997 1998 for_each_clamp_id(clamp_id) { 1999 uc_rq[clamp_id] = (struct uclamp_rq) { 2000 .value = uclamp_none(clamp_id) 2001 }; 2002 } 2003 2004 rq->uclamp_flags = UCLAMP_FLAG_IDLE; 2005 } 2006 2007 static void __init init_uclamp(void) 2008 { 2009 struct uclamp_se uc_max = {}; 2010 enum uclamp_id clamp_id; 2011 int cpu; 2012 2013 for_each_possible_cpu(cpu) 2014 init_uclamp_rq(cpu_rq(cpu)); 2015 2016 for_each_clamp_id(clamp_id) { 2017 uclamp_se_set(&init_task.uclamp_req[clamp_id], 2018 uclamp_none(clamp_id), false); 2019 } 2020 2021 /* System defaults allow max clamp values for both indexes */ 2022 uclamp_se_set(&uc_max, uclamp_none(UCLAMP_MAX), false); 2023 for_each_clamp_id(clamp_id) { 2024 uclamp_default[clamp_id] = uc_max; 2025 #ifdef CONFIG_UCLAMP_TASK_GROUP 2026 root_task_group.uclamp_req[clamp_id] = uc_max; 2027 root_task_group.uclamp[clamp_id] = uc_max; 2028 #endif 2029 } 2030 } 2031 2032 #else /* CONFIG_UCLAMP_TASK */ 2033 static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p) { } 2034 static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) { } 2035 static inline int uclamp_validate(struct task_struct *p, 2036 const struct sched_attr *attr) 2037 { 2038 return -EOPNOTSUPP; 2039 } 2040 static void __setscheduler_uclamp(struct task_struct *p, 2041 const struct sched_attr *attr) { } 2042 static inline void uclamp_fork(struct task_struct *p) { } 2043 static inline void uclamp_post_fork(struct task_struct *p) { } 2044 static inline void init_uclamp(void) { } 2045 #endif /* CONFIG_UCLAMP_TASK */ 2046 2047 bool sched_task_on_rq(struct task_struct *p) 2048 { 2049 return task_on_rq_queued(p); 2050 } 2051 2052 unsigned long get_wchan(struct task_struct *p) 2053 { 2054 unsigned long ip = 0; 2055 unsigned int state; 2056 2057 if (!p || p == current) 2058 return 0; 2059 2060 /* Only get wchan if task is blocked and we can keep it that way. */ 2061 raw_spin_lock_irq(&p->pi_lock); 2062 state = READ_ONCE(p->__state); 2063 smp_rmb(); /* see try_to_wake_up() */ 2064 if (state != TASK_RUNNING && state != TASK_WAKING && !p->on_rq) 2065 ip = __get_wchan(p); 2066 raw_spin_unlock_irq(&p->pi_lock); 2067 2068 return ip; 2069 } 2070 2071 static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags) 2072 { 2073 if (!(flags & ENQUEUE_NOCLOCK)) 2074 update_rq_clock(rq); 2075 2076 if (!(flags & ENQUEUE_RESTORE)) { 2077 sched_info_enqueue(rq, p); 2078 psi_enqueue(p, (flags & ENQUEUE_WAKEUP) && !(flags & ENQUEUE_MIGRATED)); 2079 } 2080 2081 uclamp_rq_inc(rq, p); 2082 p->sched_class->enqueue_task(rq, p, flags); 2083 2084 if (sched_core_enabled(rq)) 2085 sched_core_enqueue(rq, p); 2086 } 2087 2088 static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags) 2089 { 2090 if (sched_core_enabled(rq)) 2091 sched_core_dequeue(rq, p, flags); 2092 2093 if (!(flags & DEQUEUE_NOCLOCK)) 2094 update_rq_clock(rq); 2095 2096 if (!(flags & DEQUEUE_SAVE)) { 2097 sched_info_dequeue(rq, p); 2098 psi_dequeue(p, flags & DEQUEUE_SLEEP); 2099 } 2100 2101 uclamp_rq_dec(rq, p); 2102 p->sched_class->dequeue_task(rq, p, flags); 2103 } 2104 2105 void activate_task(struct rq *rq, struct task_struct *p, int flags) 2106 { 2107 if (task_on_rq_migrating(p)) 2108 flags |= ENQUEUE_MIGRATED; 2109 if (flags & ENQUEUE_MIGRATED) 2110 sched_mm_cid_migrate_to(rq, p); 2111 2112 enqueue_task(rq, p, flags); 2113 2114 p->on_rq = TASK_ON_RQ_QUEUED; 2115 } 2116 2117 void deactivate_task(struct rq *rq, struct task_struct *p, int flags) 2118 { 2119 p->on_rq = (flags & DEQUEUE_SLEEP) ? 0 : TASK_ON_RQ_MIGRATING; 2120 2121 dequeue_task(rq, p, flags); 2122 } 2123 2124 static inline int __normal_prio(int policy, int rt_prio, int nice) 2125 { 2126 int prio; 2127 2128 if (dl_policy(policy)) 2129 prio = MAX_DL_PRIO - 1; 2130 else if (rt_policy(policy)) 2131 prio = MAX_RT_PRIO - 1 - rt_prio; 2132 else 2133 prio = NICE_TO_PRIO(nice); 2134 2135 return prio; 2136 } 2137 2138 /* 2139 * Calculate the expected normal priority: i.e. priority 2140 * without taking RT-inheritance into account. Might be 2141 * boosted by interactivity modifiers. Changes upon fork, 2142 * setprio syscalls, and whenever the interactivity 2143 * estimator recalculates. 2144 */ 2145 static inline int normal_prio(struct task_struct *p) 2146 { 2147 return __normal_prio(p->policy, p->rt_priority, PRIO_TO_NICE(p->static_prio)); 2148 } 2149 2150 /* 2151 * Calculate the current priority, i.e. the priority 2152 * taken into account by the scheduler. This value might 2153 * be boosted by RT tasks, or might be boosted by 2154 * interactivity modifiers. Will be RT if the task got 2155 * RT-boosted. If not then it returns p->normal_prio. 2156 */ 2157 static int effective_prio(struct task_struct *p) 2158 { 2159 p->normal_prio = normal_prio(p); 2160 /* 2161 * If we are RT tasks or we were boosted to RT priority, 2162 * keep the priority unchanged. Otherwise, update priority 2163 * to the normal priority: 2164 */ 2165 if (!rt_prio(p->prio)) 2166 return p->normal_prio; 2167 return p->prio; 2168 } 2169 2170 /** 2171 * task_curr - is this task currently executing on a CPU? 2172 * @p: the task in question. 2173 * 2174 * Return: 1 if the task is currently executing. 0 otherwise. 2175 */ 2176 inline int task_curr(const struct task_struct *p) 2177 { 2178 return cpu_curr(task_cpu(p)) == p; 2179 } 2180 2181 /* 2182 * switched_from, switched_to and prio_changed must _NOT_ drop rq->lock, 2183 * use the balance_callback list if you want balancing. 2184 * 2185 * this means any call to check_class_changed() must be followed by a call to 2186 * balance_callback(). 2187 */ 2188 static inline void check_class_changed(struct rq *rq, struct task_struct *p, 2189 const struct sched_class *prev_class, 2190 int oldprio) 2191 { 2192 if (prev_class != p->sched_class) { 2193 if (prev_class->switched_from) 2194 prev_class->switched_from(rq, p); 2195 2196 p->sched_class->switched_to(rq, p); 2197 } else if (oldprio != p->prio || dl_task(p)) 2198 p->sched_class->prio_changed(rq, p, oldprio); 2199 } 2200 2201 void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags) 2202 { 2203 if (p->sched_class == rq->curr->sched_class) 2204 rq->curr->sched_class->check_preempt_curr(rq, p, flags); 2205 else if (sched_class_above(p->sched_class, rq->curr->sched_class)) 2206 resched_curr(rq); 2207 2208 /* 2209 * A queue event has occurred, and we're going to schedule. In 2210 * this case, we can save a useless back to back clock update. 2211 */ 2212 if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr)) 2213 rq_clock_skip_update(rq); 2214 } 2215 2216 static __always_inline 2217 int __task_state_match(struct task_struct *p, unsigned int state) 2218 { 2219 if (READ_ONCE(p->__state) & state) 2220 return 1; 2221 2222 #ifdef CONFIG_PREEMPT_RT 2223 if (READ_ONCE(p->saved_state) & state) 2224 return -1; 2225 #endif 2226 return 0; 2227 } 2228 2229 static __always_inline 2230 int task_state_match(struct task_struct *p, unsigned int state) 2231 { 2232 #ifdef CONFIG_PREEMPT_RT 2233 int match; 2234 2235 /* 2236 * Serialize against current_save_and_set_rtlock_wait_state() and 2237 * current_restore_rtlock_saved_state(). 2238 */ 2239 raw_spin_lock_irq(&p->pi_lock); 2240 match = __task_state_match(p, state); 2241 raw_spin_unlock_irq(&p->pi_lock); 2242 2243 return match; 2244 #else 2245 return __task_state_match(p, state); 2246 #endif 2247 } 2248 2249 /* 2250 * wait_task_inactive - wait for a thread to unschedule. 2251 * 2252 * Wait for the thread to block in any of the states set in @match_state. 2253 * If it changes, i.e. @p might have woken up, then return zero. When we 2254 * succeed in waiting for @p to be off its CPU, we return a positive number 2255 * (its total switch count). If a second call a short while later returns the 2256 * same number, the caller can be sure that @p has remained unscheduled the 2257 * whole time. 2258 * 2259 * The caller must ensure that the task *will* unschedule sometime soon, 2260 * else this function might spin for a *long* time. This function can't 2261 * be called with interrupts off, or it may introduce deadlock with 2262 * smp_call_function() if an IPI is sent by the same process we are 2263 * waiting to become inactive. 2264 */ 2265 unsigned long wait_task_inactive(struct task_struct *p, unsigned int match_state) 2266 { 2267 int running, queued, match; 2268 struct rq_flags rf; 2269 unsigned long ncsw; 2270 struct rq *rq; 2271 2272 for (;;) { 2273 /* 2274 * We do the initial early heuristics without holding 2275 * any task-queue locks at all. We'll only try to get 2276 * the runqueue lock when things look like they will 2277 * work out! 2278 */ 2279 rq = task_rq(p); 2280 2281 /* 2282 * If the task is actively running on another CPU 2283 * still, just relax and busy-wait without holding 2284 * any locks. 2285 * 2286 * NOTE! Since we don't hold any locks, it's not 2287 * even sure that "rq" stays as the right runqueue! 2288 * But we don't care, since "task_on_cpu()" will 2289 * return false if the runqueue has changed and p 2290 * is actually now running somewhere else! 2291 */ 2292 while (task_on_cpu(rq, p)) { 2293 if (!task_state_match(p, match_state)) 2294 return 0; 2295 cpu_relax(); 2296 } 2297 2298 /* 2299 * Ok, time to look more closely! We need the rq 2300 * lock now, to be *sure*. If we're wrong, we'll 2301 * just go back and repeat. 2302 */ 2303 rq = task_rq_lock(p, &rf); 2304 trace_sched_wait_task(p); 2305 running = task_on_cpu(rq, p); 2306 queued = task_on_rq_queued(p); 2307 ncsw = 0; 2308 if ((match = __task_state_match(p, match_state))) { 2309 /* 2310 * When matching on p->saved_state, consider this task 2311 * still queued so it will wait. 2312 */ 2313 if (match < 0) 2314 queued = 1; 2315 ncsw = p->nvcsw | LONG_MIN; /* sets MSB */ 2316 } 2317 task_rq_unlock(rq, p, &rf); 2318 2319 /* 2320 * If it changed from the expected state, bail out now. 2321 */ 2322 if (unlikely(!ncsw)) 2323 break; 2324 2325 /* 2326 * Was it really running after all now that we 2327 * checked with the proper locks actually held? 2328 * 2329 * Oops. Go back and try again.. 2330 */ 2331 if (unlikely(running)) { 2332 cpu_relax(); 2333 continue; 2334 } 2335 2336 /* 2337 * It's not enough that it's not actively running, 2338 * it must be off the runqueue _entirely_, and not 2339 * preempted! 2340 * 2341 * So if it was still runnable (but just not actively 2342 * running right now), it's preempted, and we should 2343 * yield - it could be a while. 2344 */ 2345 if (unlikely(queued)) { 2346 ktime_t to = NSEC_PER_SEC / HZ; 2347 2348 set_current_state(TASK_UNINTERRUPTIBLE); 2349 schedule_hrtimeout(&to, HRTIMER_MODE_REL_HARD); 2350 continue; 2351 } 2352 2353 /* 2354 * Ahh, all good. It wasn't running, and it wasn't 2355 * runnable, which means that it will never become 2356 * running in the future either. We're all done! 2357 */ 2358 break; 2359 } 2360 2361 return ncsw; 2362 } 2363 2364 #ifdef CONFIG_SMP 2365 2366 static void 2367 __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx); 2368 2369 static int __set_cpus_allowed_ptr(struct task_struct *p, 2370 struct affinity_context *ctx); 2371 2372 static void migrate_disable_switch(struct rq *rq, struct task_struct *p) 2373 { 2374 struct affinity_context ac = { 2375 .new_mask = cpumask_of(rq->cpu), 2376 .flags = SCA_MIGRATE_DISABLE, 2377 }; 2378 2379 if (likely(!p->migration_disabled)) 2380 return; 2381 2382 if (p->cpus_ptr != &p->cpus_mask) 2383 return; 2384 2385 /* 2386 * Violates locking rules! see comment in __do_set_cpus_allowed(). 2387 */ 2388 __do_set_cpus_allowed(p, &ac); 2389 } 2390 2391 void migrate_disable(void) 2392 { 2393 struct task_struct *p = current; 2394 2395 if (p->migration_disabled) { 2396 p->migration_disabled++; 2397 return; 2398 } 2399 2400 preempt_disable(); 2401 this_rq()->nr_pinned++; 2402 p->migration_disabled = 1; 2403 preempt_enable(); 2404 } 2405 EXPORT_SYMBOL_GPL(migrate_disable); 2406 2407 void migrate_enable(void) 2408 { 2409 struct task_struct *p = current; 2410 struct affinity_context ac = { 2411 .new_mask = &p->cpus_mask, 2412 .flags = SCA_MIGRATE_ENABLE, 2413 }; 2414 2415 if (p->migration_disabled > 1) { 2416 p->migration_disabled--; 2417 return; 2418 } 2419 2420 if (WARN_ON_ONCE(!p->migration_disabled)) 2421 return; 2422 2423 /* 2424 * Ensure stop_task runs either before or after this, and that 2425 * __set_cpus_allowed_ptr(SCA_MIGRATE_ENABLE) doesn't schedule(). 2426 */ 2427 preempt_disable(); 2428 if (p->cpus_ptr != &p->cpus_mask) 2429 __set_cpus_allowed_ptr(p, &ac); 2430 /* 2431 * Mustn't clear migration_disabled() until cpus_ptr points back at the 2432 * regular cpus_mask, otherwise things that race (eg. 2433 * select_fallback_rq) get confused. 2434 */ 2435 barrier(); 2436 p->migration_disabled = 0; 2437 this_rq()->nr_pinned--; 2438 preempt_enable(); 2439 } 2440 EXPORT_SYMBOL_GPL(migrate_enable); 2441 2442 static inline bool rq_has_pinned_tasks(struct rq *rq) 2443 { 2444 return rq->nr_pinned; 2445 } 2446 2447 /* 2448 * Per-CPU kthreads are allowed to run on !active && online CPUs, see 2449 * __set_cpus_allowed_ptr() and select_fallback_rq(). 2450 */ 2451 static inline bool is_cpu_allowed(struct task_struct *p, int cpu) 2452 { 2453 /* When not in the task's cpumask, no point in looking further. */ 2454 if (!cpumask_test_cpu(cpu, p->cpus_ptr)) 2455 return false; 2456 2457 /* migrate_disabled() must be allowed to finish. */ 2458 if (is_migration_disabled(p)) 2459 return cpu_online(cpu); 2460 2461 /* Non kernel threads are not allowed during either online or offline. */ 2462 if (!(p->flags & PF_KTHREAD)) 2463 return cpu_active(cpu) && task_cpu_possible(cpu, p); 2464 2465 /* KTHREAD_IS_PER_CPU is always allowed. */ 2466 if (kthread_is_per_cpu(p)) 2467 return cpu_online(cpu); 2468 2469 /* Regular kernel threads don't get to stay during offline. */ 2470 if (cpu_dying(cpu)) 2471 return false; 2472 2473 /* But are allowed during online. */ 2474 return cpu_online(cpu); 2475 } 2476 2477 /* 2478 * This is how migration works: 2479 * 2480 * 1) we invoke migration_cpu_stop() on the target CPU using 2481 * stop_one_cpu(). 2482 * 2) stopper starts to run (implicitly forcing the migrated thread 2483 * off the CPU) 2484 * 3) it checks whether the migrated task is still in the wrong runqueue. 2485 * 4) if it's in the wrong runqueue then the migration thread removes 2486 * it and puts it into the right queue. 2487 * 5) stopper completes and stop_one_cpu() returns and the migration 2488 * is done. 2489 */ 2490 2491 /* 2492 * move_queued_task - move a queued task to new rq. 2493 * 2494 * Returns (locked) new rq. Old rq's lock is released. 2495 */ 2496 static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf, 2497 struct task_struct *p, int new_cpu) 2498 { 2499 lockdep_assert_rq_held(rq); 2500 2501 deactivate_task(rq, p, DEQUEUE_NOCLOCK); 2502 set_task_cpu(p, new_cpu); 2503 rq_unlock(rq, rf); 2504 2505 rq = cpu_rq(new_cpu); 2506 2507 rq_lock(rq, rf); 2508 WARN_ON_ONCE(task_cpu(p) != new_cpu); 2509 activate_task(rq, p, 0); 2510 check_preempt_curr(rq, p, 0); 2511 2512 return rq; 2513 } 2514 2515 struct migration_arg { 2516 struct task_struct *task; 2517 int dest_cpu; 2518 struct set_affinity_pending *pending; 2519 }; 2520 2521 /* 2522 * @refs: number of wait_for_completion() 2523 * @stop_pending: is @stop_work in use 2524 */ 2525 struct set_affinity_pending { 2526 refcount_t refs; 2527 unsigned int stop_pending; 2528 struct completion done; 2529 struct cpu_stop_work stop_work; 2530 struct migration_arg arg; 2531 }; 2532 2533 /* 2534 * Move (not current) task off this CPU, onto the destination CPU. We're doing 2535 * this because either it can't run here any more (set_cpus_allowed() 2536 * away from this CPU, or CPU going down), or because we're 2537 * attempting to rebalance this task on exec (sched_exec). 2538 * 2539 * So we race with normal scheduler movements, but that's OK, as long 2540 * as the task is no longer on this CPU. 2541 */ 2542 static struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf, 2543 struct task_struct *p, int dest_cpu) 2544 { 2545 /* Affinity changed (again). */ 2546 if (!is_cpu_allowed(p, dest_cpu)) 2547 return rq; 2548 2549 rq = move_queued_task(rq, rf, p, dest_cpu); 2550 2551 return rq; 2552 } 2553 2554 /* 2555 * migration_cpu_stop - this will be executed by a highprio stopper thread 2556 * and performs thread migration by bumping thread off CPU then 2557 * 'pushing' onto another runqueue. 2558 */ 2559 static int migration_cpu_stop(void *data) 2560 { 2561 struct migration_arg *arg = data; 2562 struct set_affinity_pending *pending = arg->pending; 2563 struct task_struct *p = arg->task; 2564 struct rq *rq = this_rq(); 2565 bool complete = false; 2566 struct rq_flags rf; 2567 2568 /* 2569 * The original target CPU might have gone down and we might 2570 * be on another CPU but it doesn't matter. 2571 */ 2572 local_irq_save(rf.flags); 2573 /* 2574 * We need to explicitly wake pending tasks before running 2575 * __migrate_task() such that we will not miss enforcing cpus_ptr 2576 * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test. 2577 */ 2578 flush_smp_call_function_queue(); 2579 2580 raw_spin_lock(&p->pi_lock); 2581 rq_lock(rq, &rf); 2582 2583 /* 2584 * If we were passed a pending, then ->stop_pending was set, thus 2585 * p->migration_pending must have remained stable. 2586 */ 2587 WARN_ON_ONCE(pending && pending != p->migration_pending); 2588 2589 /* 2590 * If task_rq(p) != rq, it cannot be migrated here, because we're 2591 * holding rq->lock, if p->on_rq == 0 it cannot get enqueued because 2592 * we're holding p->pi_lock. 2593 */ 2594 if (task_rq(p) == rq) { 2595 if (is_migration_disabled(p)) 2596 goto out; 2597 2598 if (pending) { 2599 p->migration_pending = NULL; 2600 complete = true; 2601 2602 if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask)) 2603 goto out; 2604 } 2605 2606 if (task_on_rq_queued(p)) { 2607 update_rq_clock(rq); 2608 rq = __migrate_task(rq, &rf, p, arg->dest_cpu); 2609 } else { 2610 p->wake_cpu = arg->dest_cpu; 2611 } 2612 2613 /* 2614 * XXX __migrate_task() can fail, at which point we might end 2615 * up running on a dodgy CPU, AFAICT this can only happen 2616 * during CPU hotplug, at which point we'll get pushed out 2617 * anyway, so it's probably not a big deal. 2618 */ 2619 2620 } else if (pending) { 2621 /* 2622 * This happens when we get migrated between migrate_enable()'s 2623 * preempt_enable() and scheduling the stopper task. At that 2624 * point we're a regular task again and not current anymore. 2625 * 2626 * A !PREEMPT kernel has a giant hole here, which makes it far 2627 * more likely. 2628 */ 2629 2630 /* 2631 * The task moved before the stopper got to run. We're holding 2632 * ->pi_lock, so the allowed mask is stable - if it got 2633 * somewhere allowed, we're done. 2634 */ 2635 if (cpumask_test_cpu(task_cpu(p), p->cpus_ptr)) { 2636 p->migration_pending = NULL; 2637 complete = true; 2638 goto out; 2639 } 2640 2641 /* 2642 * When migrate_enable() hits a rq mis-match we can't reliably 2643 * determine is_migration_disabled() and so have to chase after 2644 * it. 2645 */ 2646 WARN_ON_ONCE(!pending->stop_pending); 2647 task_rq_unlock(rq, p, &rf); 2648 stop_one_cpu_nowait(task_cpu(p), migration_cpu_stop, 2649 &pending->arg, &pending->stop_work); 2650 return 0; 2651 } 2652 out: 2653 if (pending) 2654 pending->stop_pending = false; 2655 task_rq_unlock(rq, p, &rf); 2656 2657 if (complete) 2658 complete_all(&pending->done); 2659 2660 return 0; 2661 } 2662 2663 int push_cpu_stop(void *arg) 2664 { 2665 struct rq *lowest_rq = NULL, *rq = this_rq(); 2666 struct task_struct *p = arg; 2667 2668 raw_spin_lock_irq(&p->pi_lock); 2669 raw_spin_rq_lock(rq); 2670 2671 if (task_rq(p) != rq) 2672 goto out_unlock; 2673 2674 if (is_migration_disabled(p)) { 2675 p->migration_flags |= MDF_PUSH; 2676 goto out_unlock; 2677 } 2678 2679 p->migration_flags &= ~MDF_PUSH; 2680 2681 if (p->sched_class->find_lock_rq) 2682 lowest_rq = p->sched_class->find_lock_rq(p, rq); 2683 2684 if (!lowest_rq) 2685 goto out_unlock; 2686 2687 // XXX validate p is still the highest prio task 2688 if (task_rq(p) == rq) { 2689 deactivate_task(rq, p, 0); 2690 set_task_cpu(p, lowest_rq->cpu); 2691 activate_task(lowest_rq, p, 0); 2692 resched_curr(lowest_rq); 2693 } 2694 2695 double_unlock_balance(rq, lowest_rq); 2696 2697 out_unlock: 2698 rq->push_busy = false; 2699 raw_spin_rq_unlock(rq); 2700 raw_spin_unlock_irq(&p->pi_lock); 2701 2702 put_task_struct(p); 2703 return 0; 2704 } 2705 2706 /* 2707 * sched_class::set_cpus_allowed must do the below, but is not required to 2708 * actually call this function. 2709 */ 2710 void set_cpus_allowed_common(struct task_struct *p, struct affinity_context *ctx) 2711 { 2712 if (ctx->flags & (SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) { 2713 p->cpus_ptr = ctx->new_mask; 2714 return; 2715 } 2716 2717 cpumask_copy(&p->cpus_mask, ctx->new_mask); 2718 p->nr_cpus_allowed = cpumask_weight(ctx->new_mask); 2719 2720 /* 2721 * Swap in a new user_cpus_ptr if SCA_USER flag set 2722 */ 2723 if (ctx->flags & SCA_USER) 2724 swap(p->user_cpus_ptr, ctx->user_mask); 2725 } 2726 2727 static void 2728 __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) 2729 { 2730 struct rq *rq = task_rq(p); 2731 bool queued, running; 2732 2733 /* 2734 * This here violates the locking rules for affinity, since we're only 2735 * supposed to change these variables while holding both rq->lock and 2736 * p->pi_lock. 2737 * 2738 * HOWEVER, it magically works, because ttwu() is the only code that 2739 * accesses these variables under p->pi_lock and only does so after 2740 * smp_cond_load_acquire(&p->on_cpu, !VAL), and we're in __schedule() 2741 * before finish_task(). 2742 * 2743 * XXX do further audits, this smells like something putrid. 2744 */ 2745 if (ctx->flags & SCA_MIGRATE_DISABLE) 2746 SCHED_WARN_ON(!p->on_cpu); 2747 else 2748 lockdep_assert_held(&p->pi_lock); 2749 2750 queued = task_on_rq_queued(p); 2751 running = task_current(rq, p); 2752 2753 if (queued) { 2754 /* 2755 * Because __kthread_bind() calls this on blocked tasks without 2756 * holding rq->lock. 2757 */ 2758 lockdep_assert_rq_held(rq); 2759 dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); 2760 } 2761 if (running) 2762 put_prev_task(rq, p); 2763 2764 p->sched_class->set_cpus_allowed(p, ctx); 2765 2766 if (queued) 2767 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); 2768 if (running) 2769 set_next_task(rq, p); 2770 } 2771 2772 /* 2773 * Used for kthread_bind() and select_fallback_rq(), in both cases the user 2774 * affinity (if any) should be destroyed too. 2775 */ 2776 void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) 2777 { 2778 struct affinity_context ac = { 2779 .new_mask = new_mask, 2780 .user_mask = NULL, 2781 .flags = SCA_USER, /* clear the user requested mask */ 2782 }; 2783 union cpumask_rcuhead { 2784 cpumask_t cpumask; 2785 struct rcu_head rcu; 2786 }; 2787 2788 __do_set_cpus_allowed(p, &ac); 2789 2790 /* 2791 * Because this is called with p->pi_lock held, it is not possible 2792 * to use kfree() here (when PREEMPT_RT=y), therefore punt to using 2793 * kfree_rcu(). 2794 */ 2795 kfree_rcu((union cpumask_rcuhead *)ac.user_mask, rcu); 2796 } 2797 2798 static cpumask_t *alloc_user_cpus_ptr(int node) 2799 { 2800 /* 2801 * See do_set_cpus_allowed() above for the rcu_head usage. 2802 */ 2803 int size = max_t(int, cpumask_size(), sizeof(struct rcu_head)); 2804 2805 return kmalloc_node(size, GFP_KERNEL, node); 2806 } 2807 2808 int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src, 2809 int node) 2810 { 2811 cpumask_t *user_mask; 2812 unsigned long flags; 2813 2814 /* 2815 * Always clear dst->user_cpus_ptr first as their user_cpus_ptr's 2816 * may differ by now due to racing. 2817 */ 2818 dst->user_cpus_ptr = NULL; 2819 2820 /* 2821 * This check is racy and losing the race is a valid situation. 2822 * It is not worth the extra overhead of taking the pi_lock on 2823 * every fork/clone. 2824 */ 2825 if (data_race(!src->user_cpus_ptr)) 2826 return 0; 2827 2828 user_mask = alloc_user_cpus_ptr(node); 2829 if (!user_mask) 2830 return -ENOMEM; 2831 2832 /* 2833 * Use pi_lock to protect content of user_cpus_ptr 2834 * 2835 * Though unlikely, user_cpus_ptr can be reset to NULL by a concurrent 2836 * do_set_cpus_allowed(). 2837 */ 2838 raw_spin_lock_irqsave(&src->pi_lock, flags); 2839 if (src->user_cpus_ptr) { 2840 swap(dst->user_cpus_ptr, user_mask); 2841 cpumask_copy(dst->user_cpus_ptr, src->user_cpus_ptr); 2842 } 2843 raw_spin_unlock_irqrestore(&src->pi_lock, flags); 2844 2845 if (unlikely(user_mask)) 2846 kfree(user_mask); 2847 2848 return 0; 2849 } 2850 2851 static inline struct cpumask *clear_user_cpus_ptr(struct task_struct *p) 2852 { 2853 struct cpumask *user_mask = NULL; 2854 2855 swap(p->user_cpus_ptr, user_mask); 2856 2857 return user_mask; 2858 } 2859 2860 void release_user_cpus_ptr(struct task_struct *p) 2861 { 2862 kfree(clear_user_cpus_ptr(p)); 2863 } 2864 2865 /* 2866 * This function is wildly self concurrent; here be dragons. 2867 * 2868 * 2869 * When given a valid mask, __set_cpus_allowed_ptr() must block until the 2870 * designated task is enqueued on an allowed CPU. If that task is currently 2871 * running, we have to kick it out using the CPU stopper. 2872 * 2873 * Migrate-Disable comes along and tramples all over our nice sandcastle. 2874 * Consider: 2875 * 2876 * Initial conditions: P0->cpus_mask = [0, 1] 2877 * 2878 * P0@CPU0 P1 2879 * 2880 * migrate_disable(); 2881 * <preempted> 2882 * set_cpus_allowed_ptr(P0, [1]); 2883 * 2884 * P1 *cannot* return from this set_cpus_allowed_ptr() call until P0 executes 2885 * its outermost migrate_enable() (i.e. it exits its Migrate-Disable region). 2886 * This means we need the following scheme: 2887 * 2888 * P0@CPU0 P1 2889 * 2890 * migrate_disable(); 2891 * <preempted> 2892 * set_cpus_allowed_ptr(P0, [1]); 2893 * <blocks> 2894 * <resumes> 2895 * migrate_enable(); 2896 * __set_cpus_allowed_ptr(); 2897 * <wakes local stopper> 2898 * `--> <woken on migration completion> 2899 * 2900 * Now the fun stuff: there may be several P1-like tasks, i.e. multiple 2901 * concurrent set_cpus_allowed_ptr(P0, [*]) calls. CPU affinity changes of any 2902 * task p are serialized by p->pi_lock, which we can leverage: the one that 2903 * should come into effect at the end of the Migrate-Disable region is the last 2904 * one. This means we only need to track a single cpumask (i.e. p->cpus_mask), 2905 * but we still need to properly signal those waiting tasks at the appropriate 2906 * moment. 2907 * 2908 * This is implemented using struct set_affinity_pending. The first 2909 * __set_cpus_allowed_ptr() caller within a given Migrate-Disable region will 2910 * setup an instance of that struct and install it on the targeted task_struct. 2911 * Any and all further callers will reuse that instance. Those then wait for 2912 * a completion signaled at the tail of the CPU stopper callback (1), triggered 2913 * on the end of the Migrate-Disable region (i.e. outermost migrate_enable()). 2914 * 2915 * 2916 * (1) In the cases covered above. There is one more where the completion is 2917 * signaled within affine_move_task() itself: when a subsequent affinity request 2918 * occurs after the stopper bailed out due to the targeted task still being 2919 * Migrate-Disable. Consider: 2920 * 2921 * Initial conditions: P0->cpus_mask = [0, 1] 2922 * 2923 * CPU0 P1 P2 2924 * <P0> 2925 * migrate_disable(); 2926 * <preempted> 2927 * set_cpus_allowed_ptr(P0, [1]); 2928 * <blocks> 2929 * <migration/0> 2930 * migration_cpu_stop() 2931 * is_migration_disabled() 2932 * <bails> 2933 * set_cpus_allowed_ptr(P0, [0, 1]); 2934 * <signal completion> 2935 * <awakes> 2936 * 2937 * Note that the above is safe vs a concurrent migrate_enable(), as any 2938 * pending affinity completion is preceded by an uninstallation of 2939 * p->migration_pending done with p->pi_lock held. 2940 */ 2941 static int affine_move_task(struct rq *rq, struct task_struct *p, struct rq_flags *rf, 2942 int dest_cpu, unsigned int flags) 2943 __releases(rq->lock) 2944 __releases(p->pi_lock) 2945 { 2946 struct set_affinity_pending my_pending = { }, *pending = NULL; 2947 bool stop_pending, complete = false; 2948 2949 /* Can the task run on the task's current CPU? If so, we're done */ 2950 if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask)) { 2951 struct task_struct *push_task = NULL; 2952 2953 if ((flags & SCA_MIGRATE_ENABLE) && 2954 (p->migration_flags & MDF_PUSH) && !rq->push_busy) { 2955 rq->push_busy = true; 2956 push_task = get_task_struct(p); 2957 } 2958 2959 /* 2960 * If there are pending waiters, but no pending stop_work, 2961 * then complete now. 2962 */ 2963 pending = p->migration_pending; 2964 if (pending && !pending->stop_pending) { 2965 p->migration_pending = NULL; 2966 complete = true; 2967 } 2968 2969 task_rq_unlock(rq, p, rf); 2970 2971 if (push_task) { 2972 stop_one_cpu_nowait(rq->cpu, push_cpu_stop, 2973 p, &rq->push_work); 2974 } 2975 2976 if (complete) 2977 complete_all(&pending->done); 2978 2979 return 0; 2980 } 2981 2982 if (!(flags & SCA_MIGRATE_ENABLE)) { 2983 /* serialized by p->pi_lock */ 2984 if (!p->migration_pending) { 2985 /* Install the request */ 2986 refcount_set(&my_pending.refs, 1); 2987 init_completion(&my_pending.done); 2988 my_pending.arg = (struct migration_arg) { 2989 .task = p, 2990 .dest_cpu = dest_cpu, 2991 .pending = &my_pending, 2992 }; 2993 2994 p->migration_pending = &my_pending; 2995 } else { 2996 pending = p->migration_pending; 2997 refcount_inc(&pending->refs); 2998 /* 2999 * Affinity has changed, but we've already installed a 3000 * pending. migration_cpu_stop() *must* see this, else 3001 * we risk a completion of the pending despite having a 3002 * task on a disallowed CPU. 3003 * 3004 * Serialized by p->pi_lock, so this is safe. 3005 */ 3006 pending->arg.dest_cpu = dest_cpu; 3007 } 3008 } 3009 pending = p->migration_pending; 3010 /* 3011 * - !MIGRATE_ENABLE: 3012 * we'll have installed a pending if there wasn't one already. 3013 * 3014 * - MIGRATE_ENABLE: 3015 * we're here because the current CPU isn't matching anymore, 3016 * the only way that can happen is because of a concurrent 3017 * set_cpus_allowed_ptr() call, which should then still be 3018 * pending completion. 3019 * 3020 * Either way, we really should have a @pending here. 3021 */ 3022 if (WARN_ON_ONCE(!pending)) { 3023 task_rq_unlock(rq, p, rf); 3024 return -EINVAL; 3025 } 3026 3027 if (task_on_cpu(rq, p) || READ_ONCE(p->__state) == TASK_WAKING) { 3028 /* 3029 * MIGRATE_ENABLE gets here because 'p == current', but for 3030 * anything else we cannot do is_migration_disabled(), punt 3031 * and have the stopper function handle it all race-free. 3032 */ 3033 stop_pending = pending->stop_pending; 3034 if (!stop_pending) 3035 pending->stop_pending = true; 3036 3037 if (flags & SCA_MIGRATE_ENABLE) 3038 p->migration_flags &= ~MDF_PUSH; 3039 3040 task_rq_unlock(rq, p, rf); 3041 3042 if (!stop_pending) { 3043 stop_one_cpu_nowait(cpu_of(rq), migration_cpu_stop, 3044 &pending->arg, &pending->stop_work); 3045 } 3046 3047 if (flags & SCA_MIGRATE_ENABLE) 3048 return 0; 3049 } else { 3050 3051 if (!is_migration_disabled(p)) { 3052 if (task_on_rq_queued(p)) 3053 rq = move_queued_task(rq, rf, p, dest_cpu); 3054 3055 if (!pending->stop_pending) { 3056 p->migration_pending = NULL; 3057 complete = true; 3058 } 3059 } 3060 task_rq_unlock(rq, p, rf); 3061 3062 if (complete) 3063 complete_all(&pending->done); 3064 } 3065 3066 wait_for_completion(&pending->done); 3067 3068 if (refcount_dec_and_test(&pending->refs)) 3069 wake_up_var(&pending->refs); /* No UaF, just an address */ 3070 3071 /* 3072 * Block the original owner of &pending until all subsequent callers 3073 * have seen the completion and decremented the refcount 3074 */ 3075 wait_var_event(&my_pending.refs, !refcount_read(&my_pending.refs)); 3076 3077 /* ARGH */ 3078 WARN_ON_ONCE(my_pending.stop_pending); 3079 3080 return 0; 3081 } 3082 3083 /* 3084 * Called with both p->pi_lock and rq->lock held; drops both before returning. 3085 */ 3086 static int __set_cpus_allowed_ptr_locked(struct task_struct *p, 3087 struct affinity_context *ctx, 3088 struct rq *rq, 3089 struct rq_flags *rf) 3090 __releases(rq->lock) 3091 __releases(p->pi_lock) 3092 { 3093 const struct cpumask *cpu_allowed_mask = task_cpu_possible_mask(p); 3094 const struct cpumask *cpu_valid_mask = cpu_active_mask; 3095 bool kthread = p->flags & PF_KTHREAD; 3096 unsigned int dest_cpu; 3097 int ret = 0; 3098 3099 update_rq_clock(rq); 3100 3101 if (kthread || is_migration_disabled(p)) { 3102 /* 3103 * Kernel threads are allowed on online && !active CPUs, 3104 * however, during cpu-hot-unplug, even these might get pushed 3105 * away if not KTHREAD_IS_PER_CPU. 3106 * 3107 * Specifically, migration_disabled() tasks must not fail the 3108 * cpumask_any_and_distribute() pick below, esp. so on 3109 * SCA_MIGRATE_ENABLE, otherwise we'll not call 3110 * set_cpus_allowed_common() and actually reset p->cpus_ptr. 3111 */ 3112 cpu_valid_mask = cpu_online_mask; 3113 } 3114 3115 if (!kthread && !cpumask_subset(ctx->new_mask, cpu_allowed_mask)) { 3116 ret = -EINVAL; 3117 goto out; 3118 } 3119 3120 /* 3121 * Must re-check here, to close a race against __kthread_bind(), 3122 * sched_setaffinity() is not guaranteed to observe the flag. 3123 */ 3124 if ((ctx->flags & SCA_CHECK) && (p->flags & PF_NO_SETAFFINITY)) { 3125 ret = -EINVAL; 3126 goto out; 3127 } 3128 3129 if (!(ctx->flags & SCA_MIGRATE_ENABLE)) { 3130 if (cpumask_equal(&p->cpus_mask, ctx->new_mask)) { 3131 if (ctx->flags & SCA_USER) 3132 swap(p->user_cpus_ptr, ctx->user_mask); 3133 goto out; 3134 } 3135 3136 if (WARN_ON_ONCE(p == current && 3137 is_migration_disabled(p) && 3138 !cpumask_test_cpu(task_cpu(p), ctx->new_mask))) { 3139 ret = -EBUSY; 3140 goto out; 3141 } 3142 } 3143 3144 /* 3145 * Picking a ~random cpu helps in cases where we are changing affinity 3146 * for groups of tasks (ie. cpuset), so that load balancing is not 3147 * immediately required to distribute the tasks within their new mask. 3148 */ 3149 dest_cpu = cpumask_any_and_distribute(cpu_valid_mask, ctx->new_mask); 3150 if (dest_cpu >= nr_cpu_ids) { 3151 ret = -EINVAL; 3152 goto out; 3153 } 3154 3155 __do_set_cpus_allowed(p, ctx); 3156 3157 return affine_move_task(rq, p, rf, dest_cpu, ctx->flags); 3158 3159 out: 3160 task_rq_unlock(rq, p, rf); 3161 3162 return ret; 3163 } 3164 3165 /* 3166 * Change a given task's CPU affinity. Migrate the thread to a 3167 * proper CPU and schedule it away if the CPU it's executing on 3168 * is removed from the allowed bitmask. 3169 * 3170 * NOTE: the caller must have a valid reference to the task, the 3171 * task must not exit() & deallocate itself prematurely. The 3172 * call is not atomic; no spinlocks may be held. 3173 */ 3174 static int __set_cpus_allowed_ptr(struct task_struct *p, 3175 struct affinity_context *ctx) 3176 { 3177 struct rq_flags rf; 3178 struct rq *rq; 3179 3180 rq = task_rq_lock(p, &rf); 3181 /* 3182 * Masking should be skipped if SCA_USER or any of the SCA_MIGRATE_* 3183 * flags are set. 3184 */ 3185 if (p->user_cpus_ptr && 3186 !(ctx->flags & (SCA_USER | SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) && 3187 cpumask_and(rq->scratch_mask, ctx->new_mask, p->user_cpus_ptr)) 3188 ctx->new_mask = rq->scratch_mask; 3189 3190 return __set_cpus_allowed_ptr_locked(p, ctx, rq, &rf); 3191 } 3192 3193 int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) 3194 { 3195 struct affinity_context ac = { 3196 .new_mask = new_mask, 3197 .flags = 0, 3198 }; 3199 3200 return __set_cpus_allowed_ptr(p, &ac); 3201 } 3202 EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr); 3203 3204 /* 3205 * Change a given task's CPU affinity to the intersection of its current 3206 * affinity mask and @subset_mask, writing the resulting mask to @new_mask. 3207 * If user_cpus_ptr is defined, use it as the basis for restricting CPU 3208 * affinity or use cpu_online_mask instead. 3209 * 3210 * If the resulting mask is empty, leave the affinity unchanged and return 3211 * -EINVAL. 3212 */ 3213 static int restrict_cpus_allowed_ptr(struct task_struct *p, 3214 struct cpumask *new_mask, 3215 const struct cpumask *subset_mask) 3216 { 3217 struct affinity_context ac = { 3218 .new_mask = new_mask, 3219 .flags = 0, 3220 }; 3221 struct rq_flags rf; 3222 struct rq *rq; 3223 int err; 3224 3225 rq = task_rq_lock(p, &rf); 3226 3227 /* 3228 * Forcefully restricting the affinity of a deadline task is 3229 * likely to cause problems, so fail and noisily override the 3230 * mask entirely. 3231 */ 3232 if (task_has_dl_policy(p) && dl_bandwidth_enabled()) { 3233 err = -EPERM; 3234 goto err_unlock; 3235 } 3236 3237 if (!cpumask_and(new_mask, task_user_cpus(p), subset_mask)) { 3238 err = -EINVAL; 3239 goto err_unlock; 3240 } 3241 3242 return __set_cpus_allowed_ptr_locked(p, &ac, rq, &rf); 3243 3244 err_unlock: 3245 task_rq_unlock(rq, p, &rf); 3246 return err; 3247 } 3248 3249 /* 3250 * Restrict the CPU affinity of task @p so that it is a subset of 3251 * task_cpu_possible_mask() and point @p->user_cpus_ptr to a copy of the 3252 * old affinity mask. If the resulting mask is empty, we warn and walk 3253 * up the cpuset hierarchy until we find a suitable mask. 3254 */ 3255 void force_compatible_cpus_allowed_ptr(struct task_struct *p) 3256 { 3257 cpumask_var_t new_mask; 3258 const struct cpumask *override_mask = task_cpu_possible_mask(p); 3259 3260 alloc_cpumask_var(&new_mask, GFP_KERNEL); 3261 3262 /* 3263 * __migrate_task() can fail silently in the face of concurrent 3264 * offlining of the chosen destination CPU, so take the hotplug 3265 * lock to ensure that the migration succeeds. 3266 */ 3267 cpus_read_lock(); 3268 if (!cpumask_available(new_mask)) 3269 goto out_set_mask; 3270 3271 if (!restrict_cpus_allowed_ptr(p, new_mask, override_mask)) 3272 goto out_free_mask; 3273 3274 /* 3275 * We failed to find a valid subset of the affinity mask for the 3276 * task, so override it based on its cpuset hierarchy. 3277 */ 3278 cpuset_cpus_allowed(p, new_mask); 3279 override_mask = new_mask; 3280 3281 out_set_mask: 3282 if (printk_ratelimit()) { 3283 printk_deferred("Overriding affinity for process %d (%s) to CPUs %*pbl\n", 3284 task_pid_nr(p), p->comm, 3285 cpumask_pr_args(override_mask)); 3286 } 3287 3288 WARN_ON(set_cpus_allowed_ptr(p, override_mask)); 3289 out_free_mask: 3290 cpus_read_unlock(); 3291 free_cpumask_var(new_mask); 3292 } 3293 3294 static int 3295 __sched_setaffinity(struct task_struct *p, struct affinity_context *ctx); 3296 3297 /* 3298 * Restore the affinity of a task @p which was previously restricted by a 3299 * call to force_compatible_cpus_allowed_ptr(). 3300 * 3301 * It is the caller's responsibility to serialise this with any calls to 3302 * force_compatible_cpus_allowed_ptr(@p). 3303 */ 3304 void relax_compatible_cpus_allowed_ptr(struct task_struct *p) 3305 { 3306 struct affinity_context ac = { 3307 .new_mask = task_user_cpus(p), 3308 .flags = 0, 3309 }; 3310 int ret; 3311 3312 /* 3313 * Try to restore the old affinity mask with __sched_setaffinity(). 3314 * Cpuset masking will be done there too. 3315 */ 3316 ret = __sched_setaffinity(p, &ac); 3317 WARN_ON_ONCE(ret); 3318 } 3319 3320 void set_task_cpu(struct task_struct *p, unsigned int new_cpu) 3321 { 3322 #ifdef CONFIG_SCHED_DEBUG 3323 unsigned int state = READ_ONCE(p->__state); 3324 3325 /* 3326 * We should never call set_task_cpu() on a blocked task, 3327 * ttwu() will sort out the placement. 3328 */ 3329 WARN_ON_ONCE(state != TASK_RUNNING && state != TASK_WAKING && !p->on_rq); 3330 3331 /* 3332 * Migrating fair class task must have p->on_rq = TASK_ON_RQ_MIGRATING, 3333 * because schedstat_wait_{start,end} rebase migrating task's wait_start 3334 * time relying on p->on_rq. 3335 */ 3336 WARN_ON_ONCE(state == TASK_RUNNING && 3337 p->sched_class == &fair_sched_class && 3338 (p->on_rq && !task_on_rq_migrating(p))); 3339 3340 #ifdef CONFIG_LOCKDEP 3341 /* 3342 * The caller should hold either p->pi_lock or rq->lock, when changing 3343 * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks. 3344 * 3345 * sched_move_task() holds both and thus holding either pins the cgroup, 3346 * see task_group(). 3347 * 3348 * Furthermore, all task_rq users should acquire both locks, see 3349 * task_rq_lock(). 3350 */ 3351 WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) || 3352 lockdep_is_held(__rq_lockp(task_rq(p))))); 3353 #endif 3354 /* 3355 * Clearly, migrating tasks to offline CPUs is a fairly daft thing. 3356 */ 3357 WARN_ON_ONCE(!cpu_online(new_cpu)); 3358 3359 WARN_ON_ONCE(is_migration_disabled(p)); 3360 #endif 3361 3362 trace_sched_migrate_task(p, new_cpu); 3363 3364 if (task_cpu(p) != new_cpu) { 3365 if (p->sched_class->migrate_task_rq) 3366 p->sched_class->migrate_task_rq(p, new_cpu); 3367 p->se.nr_migrations++; 3368 rseq_migrate(p); 3369 sched_mm_cid_migrate_from(p); 3370 perf_event_task_migrate(p); 3371 } 3372 3373 __set_task_cpu(p, new_cpu); 3374 } 3375 3376 #ifdef CONFIG_NUMA_BALANCING 3377 static void __migrate_swap_task(struct task_struct *p, int cpu) 3378 { 3379 if (task_on_rq_queued(p)) { 3380 struct rq *src_rq, *dst_rq; 3381 struct rq_flags srf, drf; 3382 3383 src_rq = task_rq(p); 3384 dst_rq = cpu_rq(cpu); 3385 3386 rq_pin_lock(src_rq, &srf); 3387 rq_pin_lock(dst_rq, &drf); 3388 3389 deactivate_task(src_rq, p, 0); 3390 set_task_cpu(p, cpu); 3391 activate_task(dst_rq, p, 0); 3392 check_preempt_curr(dst_rq, p, 0); 3393 3394 rq_unpin_lock(dst_rq, &drf); 3395 rq_unpin_lock(src_rq, &srf); 3396 3397 } else { 3398 /* 3399 * Task isn't running anymore; make it appear like we migrated 3400 * it before it went to sleep. This means on wakeup we make the 3401 * previous CPU our target instead of where it really is. 3402 */ 3403 p->wake_cpu = cpu; 3404 } 3405 } 3406 3407 struct migration_swap_arg { 3408 struct task_struct *src_task, *dst_task; 3409 int src_cpu, dst_cpu; 3410 }; 3411 3412 static int migrate_swap_stop(void *data) 3413 { 3414 struct migration_swap_arg *arg = data; 3415 struct rq *src_rq, *dst_rq; 3416 int ret = -EAGAIN; 3417 3418 if (!cpu_active(arg->src_cpu) || !cpu_active(arg->dst_cpu)) 3419 return -EAGAIN; 3420 3421 src_rq = cpu_rq(arg->src_cpu); 3422 dst_rq = cpu_rq(arg->dst_cpu); 3423 3424 double_raw_lock(&arg->src_task->pi_lock, 3425 &arg->dst_task->pi_lock); 3426 double_rq_lock(src_rq, dst_rq); 3427 3428 if (task_cpu(arg->dst_task) != arg->dst_cpu) 3429 goto unlock; 3430 3431 if (task_cpu(arg->src_task) != arg->src_cpu) 3432 goto unlock; 3433 3434 if (!cpumask_test_cpu(arg->dst_cpu, arg->src_task->cpus_ptr)) 3435 goto unlock; 3436 3437 if (!cpumask_test_cpu(arg->src_cpu, arg->dst_task->cpus_ptr)) 3438 goto unlock; 3439 3440 __migrate_swap_task(arg->src_task, arg->dst_cpu); 3441 __migrate_swap_task(arg->dst_task, arg->src_cpu); 3442 3443 ret = 0; 3444 3445 unlock: 3446 double_rq_unlock(src_rq, dst_rq); 3447 raw_spin_unlock(&arg->dst_task->pi_lock); 3448 raw_spin_unlock(&arg->src_task->pi_lock); 3449 3450 return ret; 3451 } 3452 3453 /* 3454 * Cross migrate two tasks 3455 */ 3456 int migrate_swap(struct task_struct *cur, struct task_struct *p, 3457 int target_cpu, int curr_cpu) 3458 { 3459 struct migration_swap_arg arg; 3460 int ret = -EINVAL; 3461 3462 arg = (struct migration_swap_arg){ 3463 .src_task = cur, 3464 .src_cpu = curr_cpu, 3465 .dst_task = p, 3466 .dst_cpu = target_cpu, 3467 }; 3468 3469 if (arg.src_cpu == arg.dst_cpu) 3470 goto out; 3471 3472 /* 3473 * These three tests are all lockless; this is OK since all of them 3474 * will be re-checked with proper locks held further down the line. 3475 */ 3476 if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu)) 3477 goto out; 3478 3479 if (!cpumask_test_cpu(arg.dst_cpu, arg.src_task->cpus_ptr)) 3480 goto out; 3481 3482 if (!cpumask_test_cpu(arg.src_cpu, arg.dst_task->cpus_ptr)) 3483 goto out; 3484 3485 trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu); 3486 ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg); 3487 3488 out: 3489 return ret; 3490 } 3491 #endif /* CONFIG_NUMA_BALANCING */ 3492 3493 /*** 3494 * kick_process - kick a running thread to enter/exit the kernel 3495 * @p: the to-be-kicked thread 3496 * 3497 * Cause a process which is running on another CPU to enter 3498 * kernel-mode, without any delay. (to get signals handled.) 3499 * 3500 * NOTE: this function doesn't have to take the runqueue lock, 3501 * because all it wants to ensure is that the remote task enters 3502 * the kernel. If the IPI races and the task has been migrated 3503 * to another CPU then no harm is done and the purpose has been 3504 * achieved as well. 3505 */ 3506 void kick_process(struct task_struct *p) 3507 { 3508 int cpu; 3509 3510 preempt_disable(); 3511 cpu = task_cpu(p); 3512 if ((cpu != smp_processor_id()) && task_curr(p)) 3513 smp_send_reschedule(cpu); 3514 preempt_enable(); 3515 } 3516 EXPORT_SYMBOL_GPL(kick_process); 3517 3518 /* 3519 * ->cpus_ptr is protected by both rq->lock and p->pi_lock 3520 * 3521 * A few notes on cpu_active vs cpu_online: 3522 * 3523 * - cpu_active must be a subset of cpu_online 3524 * 3525 * - on CPU-up we allow per-CPU kthreads on the online && !active CPU, 3526 * see __set_cpus_allowed_ptr(). At this point the newly online 3527 * CPU isn't yet part of the sched domains, and balancing will not 3528 * see it. 3529 * 3530 * - on CPU-down we clear cpu_active() to mask the sched domains and 3531 * avoid the load balancer to place new tasks on the to be removed 3532 * CPU. Existing tasks will remain running there and will be taken 3533 * off. 3534 * 3535 * This means that fallback selection must not select !active CPUs. 3536 * And can assume that any active CPU must be online. Conversely 3537 * select_task_rq() below may allow selection of !active CPUs in order 3538 * to satisfy the above rules. 3539 */ 3540 static int select_fallback_rq(int cpu, struct task_struct *p) 3541 { 3542 int nid = cpu_to_node(cpu); 3543 const struct cpumask *nodemask = NULL; 3544 enum { cpuset, possible, fail } state = cpuset; 3545 int dest_cpu; 3546 3547 /* 3548 * If the node that the CPU is on has been offlined, cpu_to_node() 3549 * will return -1. There is no CPU on the node, and we should 3550 * select the CPU on the other node. 3551 */ 3552 if (nid != -1) { 3553 nodemask = cpumask_of_node(nid); 3554 3555 /* Look for allowed, online CPU in same node. */ 3556 for_each_cpu(dest_cpu, nodemask) { 3557 if (is_cpu_allowed(p, dest_cpu)) 3558 return dest_cpu; 3559 } 3560 } 3561 3562 for (;;) { 3563 /* Any allowed, online CPU? */ 3564 for_each_cpu(dest_cpu, p->cpus_ptr) { 3565 if (!is_cpu_allowed(p, dest_cpu)) 3566 continue; 3567 3568 goto out; 3569 } 3570 3571 /* No more Mr. Nice Guy. */ 3572 switch (state) { 3573 case cpuset: 3574 if (cpuset_cpus_allowed_fallback(p)) { 3575 state = possible; 3576 break; 3577 } 3578 fallthrough; 3579 case possible: 3580 /* 3581 * XXX When called from select_task_rq() we only 3582 * hold p->pi_lock and again violate locking order. 3583 * 3584 * More yuck to audit. 3585 */ 3586 do_set_cpus_allowed(p, task_cpu_possible_mask(p)); 3587 state = fail; 3588 break; 3589 case fail: 3590 BUG(); 3591 break; 3592 } 3593 } 3594 3595 out: 3596 if (state != cpuset) { 3597 /* 3598 * Don't tell them about moving exiting tasks or 3599 * kernel threads (both mm NULL), since they never 3600 * leave kernel. 3601 */ 3602 if (p->mm && printk_ratelimit()) { 3603 printk_deferred("process %d (%s) no longer affine to cpu%d\n", 3604 task_pid_nr(p), p->comm, cpu); 3605 } 3606 } 3607 3608 return dest_cpu; 3609 } 3610 3611 /* 3612 * The caller (fork, wakeup) owns p->pi_lock, ->cpus_ptr is stable. 3613 */ 3614 static inline 3615 int select_task_rq(struct task_struct *p, int cpu, int wake_flags) 3616 { 3617 lockdep_assert_held(&p->pi_lock); 3618 3619 if (p->nr_cpus_allowed > 1 && !is_migration_disabled(p)) 3620 cpu = p->sched_class->select_task_rq(p, cpu, wake_flags); 3621 else 3622 cpu = cpumask_any(p->cpus_ptr); 3623 3624 /* 3625 * In order not to call set_task_cpu() on a blocking task we need 3626 * to rely on ttwu() to place the task on a valid ->cpus_ptr 3627 * CPU. 3628 * 3629 * Since this is common to all placement strategies, this lives here. 3630 * 3631 * [ this allows ->select_task() to simply return task_cpu(p) and 3632 * not worry about this generic constraint ] 3633 */ 3634 if (unlikely(!is_cpu_allowed(p, cpu))) 3635 cpu = select_fallback_rq(task_cpu(p), p); 3636 3637 return cpu; 3638 } 3639 3640 void sched_set_stop_task(int cpu, struct task_struct *stop) 3641 { 3642 static struct lock_class_key stop_pi_lock; 3643 struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 }; 3644 struct task_struct *old_stop = cpu_rq(cpu)->stop; 3645 3646 if (stop) { 3647 /* 3648 * Make it appear like a SCHED_FIFO task, its something 3649 * userspace knows about and won't get confused about. 3650 * 3651 * Also, it will make PI more or less work without too 3652 * much confusion -- but then, stop work should not 3653 * rely on PI working anyway. 3654 */ 3655 sched_setscheduler_nocheck(stop, SCHED_FIFO, ¶m); 3656 3657 stop->sched_class = &stop_sched_class; 3658 3659 /* 3660 * The PI code calls rt_mutex_setprio() with ->pi_lock held to 3661 * adjust the effective priority of a task. As a result, 3662 * rt_mutex_setprio() can trigger (RT) balancing operations, 3663 * which can then trigger wakeups of the stop thread to push 3664 * around the current task. 3665 * 3666 * The stop task itself will never be part of the PI-chain, it 3667 * never blocks, therefore that ->pi_lock recursion is safe. 3668 * Tell lockdep about this by placing the stop->pi_lock in its 3669 * own class. 3670 */ 3671 lockdep_set_class(&stop->pi_lock, &stop_pi_lock); 3672 } 3673 3674 cpu_rq(cpu)->stop = stop; 3675 3676 if (old_stop) { 3677 /* 3678 * Reset it back to a normal scheduling class so that 3679 * it can die in pieces. 3680 */ 3681 old_stop->sched_class = &rt_sched_class; 3682 } 3683 } 3684 3685 #else /* CONFIG_SMP */ 3686 3687 static inline int __set_cpus_allowed_ptr(struct task_struct *p, 3688 struct affinity_context *ctx) 3689 { 3690 return set_cpus_allowed_ptr(p, ctx->new_mask); 3691 } 3692 3693 static inline void migrate_disable_switch(struct rq *rq, struct task_struct *p) { } 3694 3695 static inline bool rq_has_pinned_tasks(struct rq *rq) 3696 { 3697 return false; 3698 } 3699 3700 static inline cpumask_t *alloc_user_cpus_ptr(int node) 3701 { 3702 return NULL; 3703 } 3704 3705 #endif /* !CONFIG_SMP */ 3706 3707 static void 3708 ttwu_stat(struct task_struct *p, int cpu, int wake_flags) 3709 { 3710 struct rq *rq; 3711 3712 if (!schedstat_enabled()) 3713 return; 3714 3715 rq = this_rq(); 3716 3717 #ifdef CONFIG_SMP 3718 if (cpu == rq->cpu) { 3719 __schedstat_inc(rq->ttwu_local); 3720 __schedstat_inc(p->stats.nr_wakeups_local); 3721 } else { 3722 struct sched_domain *sd; 3723 3724 __schedstat_inc(p->stats.nr_wakeups_remote); 3725 rcu_read_lock(); 3726 for_each_domain(rq->cpu, sd) { 3727 if (cpumask_test_cpu(cpu, sched_domain_span(sd))) { 3728 __schedstat_inc(sd->ttwu_wake_remote); 3729 break; 3730 } 3731 } 3732 rcu_read_unlock(); 3733 } 3734 3735 if (wake_flags & WF_MIGRATED) 3736 __schedstat_inc(p->stats.nr_wakeups_migrate); 3737 #endif /* CONFIG_SMP */ 3738 3739 __schedstat_inc(rq->ttwu_count); 3740 __schedstat_inc(p->stats.nr_wakeups); 3741 3742 if (wake_flags & WF_SYNC) 3743 __schedstat_inc(p->stats.nr_wakeups_sync); 3744 } 3745 3746 /* 3747 * Mark the task runnable. 3748 */ 3749 static inline void ttwu_do_wakeup(struct task_struct *p) 3750 { 3751 WRITE_ONCE(p->__state, TASK_RUNNING); 3752 trace_sched_wakeup(p); 3753 } 3754 3755 static void 3756 ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags, 3757 struct rq_flags *rf) 3758 { 3759 int en_flags = ENQUEUE_WAKEUP | ENQUEUE_NOCLOCK; 3760 3761 lockdep_assert_rq_held(rq); 3762 3763 if (p->sched_contributes_to_load) 3764 rq->nr_uninterruptible--; 3765 3766 #ifdef CONFIG_SMP 3767 if (wake_flags & WF_MIGRATED) 3768 en_flags |= ENQUEUE_MIGRATED; 3769 else 3770 #endif 3771 if (p->in_iowait) { 3772 delayacct_blkio_end(p); 3773 atomic_dec(&task_rq(p)->nr_iowait); 3774 } 3775 3776 activate_task(rq, p, en_flags); 3777 check_preempt_curr(rq, p, wake_flags); 3778 3779 ttwu_do_wakeup(p); 3780 3781 #ifdef CONFIG_SMP 3782 if (p->sched_class->task_woken) { 3783 /* 3784 * Our task @p is fully woken up and running; so it's safe to 3785 * drop the rq->lock, hereafter rq is only used for statistics. 3786 */ 3787 rq_unpin_lock(rq, rf); 3788 p->sched_class->task_woken(rq, p); 3789 rq_repin_lock(rq, rf); 3790 } 3791 3792 if (rq->idle_stamp) { 3793 u64 delta = rq_clock(rq) - rq->idle_stamp; 3794 u64 max = 2*rq->max_idle_balance_cost; 3795 3796 update_avg(&rq->avg_idle, delta); 3797 3798 if (rq->avg_idle > max) 3799 rq->avg_idle = max; 3800 3801 rq->wake_stamp = jiffies; 3802 rq->wake_avg_idle = rq->avg_idle / 2; 3803 3804 rq->idle_stamp = 0; 3805 } 3806 #endif 3807 } 3808 3809 /* 3810 * Consider @p being inside a wait loop: 3811 * 3812 * for (;;) { 3813 * set_current_state(TASK_UNINTERRUPTIBLE); 3814 * 3815 * if (CONDITION) 3816 * break; 3817 * 3818 * schedule(); 3819 * } 3820 * __set_current_state(TASK_RUNNING); 3821 * 3822 * between set_current_state() and schedule(). In this case @p is still 3823 * runnable, so all that needs doing is change p->state back to TASK_RUNNING in 3824 * an atomic manner. 3825 * 3826 * By taking task_rq(p)->lock we serialize against schedule(), if @p->on_rq 3827 * then schedule() must still happen and p->state can be changed to 3828 * TASK_RUNNING. Otherwise we lost the race, schedule() has happened, and we 3829 * need to do a full wakeup with enqueue. 3830 * 3831 * Returns: %true when the wakeup is done, 3832 * %false otherwise. 3833 */ 3834 static int ttwu_runnable(struct task_struct *p, int wake_flags) 3835 { 3836 struct rq_flags rf; 3837 struct rq *rq; 3838 int ret = 0; 3839 3840 rq = __task_rq_lock(p, &rf); 3841 if (task_on_rq_queued(p)) { 3842 if (!task_on_cpu(rq, p)) { 3843 /* 3844 * When on_rq && !on_cpu the task is preempted, see if 3845 * it should preempt the task that is current now. 3846 */ 3847 update_rq_clock(rq); 3848 check_preempt_curr(rq, p, wake_flags); 3849 } 3850 ttwu_do_wakeup(p); 3851 ret = 1; 3852 } 3853 __task_rq_unlock(rq, &rf); 3854 3855 return ret; 3856 } 3857 3858 #ifdef CONFIG_SMP 3859 void sched_ttwu_pending(void *arg) 3860 { 3861 struct llist_node *llist = arg; 3862 struct rq *rq = this_rq(); 3863 struct task_struct *p, *t; 3864 struct rq_flags rf; 3865 3866 if (!llist) 3867 return; 3868 3869 rq_lock_irqsave(rq, &rf); 3870 update_rq_clock(rq); 3871 3872 llist_for_each_entry_safe(p, t, llist, wake_entry.llist) { 3873 if (WARN_ON_ONCE(p->on_cpu)) 3874 smp_cond_load_acquire(&p->on_cpu, !VAL); 3875 3876 if (WARN_ON_ONCE(task_cpu(p) != cpu_of(rq))) 3877 set_task_cpu(p, cpu_of(rq)); 3878 3879 ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0, &rf); 3880 } 3881 3882 /* 3883 * Must be after enqueueing at least once task such that 3884 * idle_cpu() does not observe a false-negative -- if it does, 3885 * it is possible for select_idle_siblings() to stack a number 3886 * of tasks on this CPU during that window. 3887 * 3888 * It is ok to clear ttwu_pending when another task pending. 3889 * We will receive IPI after local irq enabled and then enqueue it. 3890 * Since now nr_running > 0, idle_cpu() will always get correct result. 3891 */ 3892 WRITE_ONCE(rq->ttwu_pending, 0); 3893 rq_unlock_irqrestore(rq, &rf); 3894 } 3895 3896 /* 3897 * Prepare the scene for sending an IPI for a remote smp_call 3898 * 3899 * Returns true if the caller can proceed with sending the IPI. 3900 * Returns false otherwise. 3901 */ 3902 bool call_function_single_prep_ipi(int cpu) 3903 { 3904 if (set_nr_if_polling(cpu_rq(cpu)->idle)) { 3905 trace_sched_wake_idle_without_ipi(cpu); 3906 return false; 3907 } 3908 3909 return true; 3910 } 3911 3912 /* 3913 * Queue a task on the target CPUs wake_list and wake the CPU via IPI if 3914 * necessary. The wakee CPU on receipt of the IPI will queue the task 3915 * via sched_ttwu_wakeup() for activation so the wakee incurs the cost 3916 * of the wakeup instead of the waker. 3917 */ 3918 static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) 3919 { 3920 struct rq *rq = cpu_rq(cpu); 3921 3922 p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED); 3923 3924 WRITE_ONCE(rq->ttwu_pending, 1); 3925 __smp_call_single_queue(cpu, &p->wake_entry.llist); 3926 } 3927 3928 void wake_up_if_idle(int cpu) 3929 { 3930 struct rq *rq = cpu_rq(cpu); 3931 struct rq_flags rf; 3932 3933 rcu_read_lock(); 3934 3935 if (!is_idle_task(rcu_dereference(rq->curr))) 3936 goto out; 3937 3938 rq_lock_irqsave(rq, &rf); 3939 if (is_idle_task(rq->curr)) 3940 resched_curr(rq); 3941 /* Else CPU is not idle, do nothing here: */ 3942 rq_unlock_irqrestore(rq, &rf); 3943 3944 out: 3945 rcu_read_unlock(); 3946 } 3947 3948 bool cpus_share_cache(int this_cpu, int that_cpu) 3949 { 3950 if (this_cpu == that_cpu) 3951 return true; 3952 3953 return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu); 3954 } 3955 3956 static inline bool ttwu_queue_cond(struct task_struct *p, int cpu) 3957 { 3958 /* 3959 * Do not complicate things with the async wake_list while the CPU is 3960 * in hotplug state. 3961 */ 3962 if (!cpu_active(cpu)) 3963 return false; 3964 3965 /* Ensure the task will still be allowed to run on the CPU. */ 3966 if (!cpumask_test_cpu(cpu, p->cpus_ptr)) 3967 return false; 3968 3969 /* 3970 * If the CPU does not share cache, then queue the task on the 3971 * remote rqs wakelist to avoid accessing remote data. 3972 */ 3973 if (!cpus_share_cache(smp_processor_id(), cpu)) 3974 return true; 3975 3976 if (cpu == smp_processor_id()) 3977 return false; 3978 3979 /* 3980 * If the wakee cpu is idle, or the task is descheduling and the 3981 * only running task on the CPU, then use the wakelist to offload 3982 * the task activation to the idle (or soon-to-be-idle) CPU as 3983 * the current CPU is likely busy. nr_running is checked to 3984 * avoid unnecessary task stacking. 3985 * 3986 * Note that we can only get here with (wakee) p->on_rq=0, 3987 * p->on_cpu can be whatever, we've done the dequeue, so 3988 * the wakee has been accounted out of ->nr_running. 3989 */ 3990 if (!cpu_rq(cpu)->nr_running) 3991 return true; 3992 3993 return false; 3994 } 3995 3996 static bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) 3997 { 3998 if (sched_feat(TTWU_QUEUE) && ttwu_queue_cond(p, cpu)) { 3999 sched_clock_cpu(cpu); /* Sync clocks across CPUs */ 4000 __ttwu_queue_wakelist(p, cpu, wake_flags); 4001 return true; 4002 } 4003 4004 return false; 4005 } 4006 4007 #else /* !CONFIG_SMP */ 4008 4009 static inline bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) 4010 { 4011 return false; 4012 } 4013 4014 #endif /* CONFIG_SMP */ 4015 4016 static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags) 4017 { 4018 struct rq *rq = cpu_rq(cpu); 4019 struct rq_flags rf; 4020 4021 if (ttwu_queue_wakelist(p, cpu, wake_flags)) 4022 return; 4023 4024 rq_lock(rq, &rf); 4025 update_rq_clock(rq); 4026 ttwu_do_activate(rq, p, wake_flags, &rf); 4027 rq_unlock(rq, &rf); 4028 } 4029 4030 /* 4031 * Invoked from try_to_wake_up() to check whether the task can be woken up. 4032 * 4033 * The caller holds p::pi_lock if p != current or has preemption 4034 * disabled when p == current. 4035 * 4036 * The rules of PREEMPT_RT saved_state: 4037 * 4038 * The related locking code always holds p::pi_lock when updating 4039 * p::saved_state, which means the code is fully serialized in both cases. 4040 * 4041 * The lock wait and lock wakeups happen via TASK_RTLOCK_WAIT. No other 4042 * bits set. This allows to distinguish all wakeup scenarios. 4043 */ 4044 static __always_inline 4045 bool ttwu_state_match(struct task_struct *p, unsigned int state, int *success) 4046 { 4047 int match; 4048 4049 if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)) { 4050 WARN_ON_ONCE((state & TASK_RTLOCK_WAIT) && 4051 state != TASK_RTLOCK_WAIT); 4052 } 4053 4054 *success = !!(match = __task_state_match(p, state)); 4055 4056 #ifdef CONFIG_PREEMPT_RT 4057 /* 4058 * Saved state preserves the task state across blocking on 4059 * an RT lock. If the state matches, set p::saved_state to 4060 * TASK_RUNNING, but do not wake the task because it waits 4061 * for a lock wakeup. Also indicate success because from 4062 * the regular waker's point of view this has succeeded. 4063 * 4064 * After acquiring the lock the task will restore p::__state 4065 * from p::saved_state which ensures that the regular 4066 * wakeup is not lost. The restore will also set 4067 * p::saved_state to TASK_RUNNING so any further tests will 4068 * not result in false positives vs. @success 4069 */ 4070 if (match < 0) 4071 p->saved_state = TASK_RUNNING; 4072 #endif 4073 return match > 0; 4074 } 4075 4076 /* 4077 * Notes on Program-Order guarantees on SMP systems. 4078 * 4079 * MIGRATION 4080 * 4081 * The basic program-order guarantee on SMP systems is that when a task [t] 4082 * migrates, all its activity on its old CPU [c0] happens-before any subsequent 4083 * execution on its new CPU [c1]. 4084 * 4085 * For migration (of runnable tasks) this is provided by the following means: 4086 * 4087 * A) UNLOCK of the rq(c0)->lock scheduling out task t 4088 * B) migration for t is required to synchronize *both* rq(c0)->lock and 4089 * rq(c1)->lock (if not at the same time, then in that order). 4090 * C) LOCK of the rq(c1)->lock scheduling in task 4091 * 4092 * Release/acquire chaining guarantees that B happens after A and C after B. 4093 * Note: the CPU doing B need not be c0 or c1 4094 * 4095 * Example: 4096 * 4097 * CPU0 CPU1 CPU2 4098 * 4099 * LOCK rq(0)->lock 4100 * sched-out X 4101 * sched-in Y 4102 * UNLOCK rq(0)->lock 4103 * 4104 * LOCK rq(0)->lock // orders against CPU0 4105 * dequeue X 4106 * UNLOCK rq(0)->lock 4107 * 4108 * LOCK rq(1)->lock 4109 * enqueue X 4110 * UNLOCK rq(1)->lock 4111 * 4112 * LOCK rq(1)->lock // orders against CPU2 4113 * sched-out Z 4114 * sched-in X 4115 * UNLOCK rq(1)->lock 4116 * 4117 * 4118 * BLOCKING -- aka. SLEEP + WAKEUP 4119 * 4120 * For blocking we (obviously) need to provide the same guarantee as for 4121 * migration. However the means are completely different as there is no lock 4122 * chain to provide order. Instead we do: 4123 * 4124 * 1) smp_store_release(X->on_cpu, 0) -- finish_task() 4125 * 2) smp_cond_load_acquire(!X->on_cpu) -- try_to_wake_up() 4126 * 4127 * Example: 4128 * 4129 * CPU0 (schedule) CPU1 (try_to_wake_up) CPU2 (schedule) 4130 * 4131 * LOCK rq(0)->lock LOCK X->pi_lock 4132 * dequeue X 4133 * sched-out X 4134 * smp_store_release(X->on_cpu, 0); 4135 * 4136 * smp_cond_load_acquire(&X->on_cpu, !VAL); 4137 * X->state = WAKING 4138 * set_task_cpu(X,2) 4139 * 4140 * LOCK rq(2)->lock 4141 * enqueue X 4142 * X->state = RUNNING 4143 * UNLOCK rq(2)->lock 4144 * 4145 * LOCK rq(2)->lock // orders against CPU1 4146 * sched-out Z 4147 * sched-in X 4148 * UNLOCK rq(2)->lock 4149 * 4150 * UNLOCK X->pi_lock 4151 * UNLOCK rq(0)->lock 4152 * 4153 * 4154 * However, for wakeups there is a second guarantee we must provide, namely we 4155 * must ensure that CONDITION=1 done by the caller can not be reordered with 4156 * accesses to the task state; see try_to_wake_up() and set_current_state(). 4157 */ 4158 4159 /** 4160 * try_to_wake_up - wake up a thread 4161 * @p: the thread to be awakened 4162 * @state: the mask of task states that can be woken 4163 * @wake_flags: wake modifier flags (WF_*) 4164 * 4165 * Conceptually does: 4166 * 4167 * If (@state & @p->state) @p->state = TASK_RUNNING. 4168 * 4169 * If the task was not queued/runnable, also place it back on a runqueue. 4170 * 4171 * This function is atomic against schedule() which would dequeue the task. 4172 * 4173 * It issues a full memory barrier before accessing @p->state, see the comment 4174 * with set_current_state(). 4175 * 4176 * Uses p->pi_lock to serialize against concurrent wake-ups. 4177 * 4178 * Relies on p->pi_lock stabilizing: 4179 * - p->sched_class 4180 * - p->cpus_ptr 4181 * - p->sched_task_group 4182 * in order to do migration, see its use of select_task_rq()/set_task_cpu(). 4183 * 4184 * Tries really hard to only take one task_rq(p)->lock for performance. 4185 * Takes rq->lock in: 4186 * - ttwu_runnable() -- old rq, unavoidable, see comment there; 4187 * - ttwu_queue() -- new rq, for enqueue of the task; 4188 * - psi_ttwu_dequeue() -- much sadness :-( accounting will kill us. 4189 * 4190 * As a consequence we race really badly with just about everything. See the 4191 * many memory barriers and their comments for details. 4192 * 4193 * Return: %true if @p->state changes (an actual wakeup was done), 4194 * %false otherwise. 4195 */ 4196 static int 4197 try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) 4198 { 4199 unsigned long flags; 4200 int cpu, success = 0; 4201 4202 preempt_disable(); 4203 if (p == current) { 4204 /* 4205 * We're waking current, this means 'p->on_rq' and 'task_cpu(p) 4206 * == smp_processor_id()'. Together this means we can special 4207 * case the whole 'p->on_rq && ttwu_runnable()' case below 4208 * without taking any locks. 4209 * 4210 * In particular: 4211 * - we rely on Program-Order guarantees for all the ordering, 4212 * - we're serialized against set_special_state() by virtue of 4213 * it disabling IRQs (this allows not taking ->pi_lock). 4214 */ 4215 if (!ttwu_state_match(p, state, &success)) 4216 goto out; 4217 4218 trace_sched_waking(p); 4219 ttwu_do_wakeup(p); 4220 goto out; 4221 } 4222 4223 /* 4224 * If we are going to wake up a thread waiting for CONDITION we 4225 * need to ensure that CONDITION=1 done by the caller can not be 4226 * reordered with p->state check below. This pairs with smp_store_mb() 4227 * in set_current_state() that the waiting thread does. 4228 */ 4229 raw_spin_lock_irqsave(&p->pi_lock, flags); 4230 smp_mb__after_spinlock(); 4231 if (!ttwu_state_match(p, state, &success)) 4232 goto unlock; 4233 4234 trace_sched_waking(p); 4235 4236 /* 4237 * Ensure we load p->on_rq _after_ p->state, otherwise it would 4238 * be possible to, falsely, observe p->on_rq == 0 and get stuck 4239 * in smp_cond_load_acquire() below. 4240 * 4241 * sched_ttwu_pending() try_to_wake_up() 4242 * STORE p->on_rq = 1 LOAD p->state 4243 * UNLOCK rq->lock 4244 * 4245 * __schedule() (switch to task 'p') 4246 * LOCK rq->lock smp_rmb(); 4247 * smp_mb__after_spinlock(); 4248 * UNLOCK rq->lock 4249 * 4250 * [task p] 4251 * STORE p->state = UNINTERRUPTIBLE LOAD p->on_rq 4252 * 4253 * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in 4254 * __schedule(). See the comment for smp_mb__after_spinlock(). 4255 * 4256 * A similar smb_rmb() lives in try_invoke_on_locked_down_task(). 4257 */ 4258 smp_rmb(); 4259 if (READ_ONCE(p->on_rq) && ttwu_runnable(p, wake_flags)) 4260 goto unlock; 4261 4262 #ifdef CONFIG_SMP 4263 /* 4264 * Ensure we load p->on_cpu _after_ p->on_rq, otherwise it would be 4265 * possible to, falsely, observe p->on_cpu == 0. 4266 * 4267 * One must be running (->on_cpu == 1) in order to remove oneself 4268 * from the runqueue. 4269 * 4270 * __schedule() (switch to task 'p') try_to_wake_up() 4271 * STORE p->on_cpu = 1 LOAD p->on_rq 4272 * UNLOCK rq->lock 4273 * 4274 * __schedule() (put 'p' to sleep) 4275 * LOCK rq->lock smp_rmb(); 4276 * smp_mb__after_spinlock(); 4277 * STORE p->on_rq = 0 LOAD p->on_cpu 4278 * 4279 * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in 4280 * __schedule(). See the comment for smp_mb__after_spinlock(). 4281 * 4282 * Form a control-dep-acquire with p->on_rq == 0 above, to ensure 4283 * schedule()'s deactivate_task() has 'happened' and p will no longer 4284 * care about it's own p->state. See the comment in __schedule(). 4285 */ 4286 smp_acquire__after_ctrl_dep(); 4287 4288 /* 4289 * We're doing the wakeup (@success == 1), they did a dequeue (p->on_rq 4290 * == 0), which means we need to do an enqueue, change p->state to 4291 * TASK_WAKING such that we can unlock p->pi_lock before doing the 4292 * enqueue, such as ttwu_queue_wakelist(). 4293 */ 4294 WRITE_ONCE(p->__state, TASK_WAKING); 4295 4296 /* 4297 * If the owning (remote) CPU is still in the middle of schedule() with 4298 * this task as prev, considering queueing p on the remote CPUs wake_list 4299 * which potentially sends an IPI instead of spinning on p->on_cpu to 4300 * let the waker make forward progress. This is safe because IRQs are 4301 * disabled and the IPI will deliver after on_cpu is cleared. 4302 * 4303 * Ensure we load task_cpu(p) after p->on_cpu: 4304 * 4305 * set_task_cpu(p, cpu); 4306 * STORE p->cpu = @cpu 4307 * __schedule() (switch to task 'p') 4308 * LOCK rq->lock 4309 * smp_mb__after_spin_lock() smp_cond_load_acquire(&p->on_cpu) 4310 * STORE p->on_cpu = 1 LOAD p->cpu 4311 * 4312 * to ensure we observe the correct CPU on which the task is currently 4313 * scheduling. 4314 */ 4315 if (smp_load_acquire(&p->on_cpu) && 4316 ttwu_queue_wakelist(p, task_cpu(p), wake_flags)) 4317 goto unlock; 4318 4319 /* 4320 * If the owning (remote) CPU is still in the middle of schedule() with 4321 * this task as prev, wait until it's done referencing the task. 4322 * 4323 * Pairs with the smp_store_release() in finish_task(). 4324 * 4325 * This ensures that tasks getting woken will be fully ordered against 4326 * their previous state and preserve Program Order. 4327 */ 4328 smp_cond_load_acquire(&p->on_cpu, !VAL); 4329 4330 cpu = select_task_rq(p, p->wake_cpu, wake_flags | WF_TTWU); 4331 if (task_cpu(p) != cpu) { 4332 if (p->in_iowait) { 4333 delayacct_blkio_end(p); 4334 atomic_dec(&task_rq(p)->nr_iowait); 4335 } 4336 4337 wake_flags |= WF_MIGRATED; 4338 psi_ttwu_dequeue(p); 4339 set_task_cpu(p, cpu); 4340 } 4341 #else 4342 cpu = task_cpu(p); 4343 #endif /* CONFIG_SMP */ 4344 4345 ttwu_queue(p, cpu, wake_flags); 4346 unlock: 4347 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 4348 out: 4349 if (success) 4350 ttwu_stat(p, task_cpu(p), wake_flags); 4351 preempt_enable(); 4352 4353 return success; 4354 } 4355 4356 static bool __task_needs_rq_lock(struct task_struct *p) 4357 { 4358 unsigned int state = READ_ONCE(p->__state); 4359 4360 /* 4361 * Since pi->lock blocks try_to_wake_up(), we don't need rq->lock when 4362 * the task is blocked. Make sure to check @state since ttwu() can drop 4363 * locks at the end, see ttwu_queue_wakelist(). 4364 */ 4365 if (state == TASK_RUNNING || state == TASK_WAKING) 4366 return true; 4367 4368 /* 4369 * Ensure we load p->on_rq after p->__state, otherwise it would be 4370 * possible to, falsely, observe p->on_rq == 0. 4371 * 4372 * See try_to_wake_up() for a longer comment. 4373 */ 4374 smp_rmb(); 4375 if (p->on_rq) 4376 return true; 4377 4378 #ifdef CONFIG_SMP 4379 /* 4380 * Ensure the task has finished __schedule() and will not be referenced 4381 * anymore. Again, see try_to_wake_up() for a longer comment. 4382 */ 4383 smp_rmb(); 4384 smp_cond_load_acquire(&p->on_cpu, !VAL); 4385 #endif 4386 4387 return false; 4388 } 4389 4390 /** 4391 * task_call_func - Invoke a function on task in fixed state 4392 * @p: Process for which the function is to be invoked, can be @current. 4393 * @func: Function to invoke. 4394 * @arg: Argument to function. 4395 * 4396 * Fix the task in it's current state by avoiding wakeups and or rq operations 4397 * and call @func(@arg) on it. This function can use ->on_rq and task_curr() 4398 * to work out what the state is, if required. Given that @func can be invoked 4399 * with a runqueue lock held, it had better be quite lightweight. 4400 * 4401 * Returns: 4402 * Whatever @func returns 4403 */ 4404 int task_call_func(struct task_struct *p, task_call_f func, void *arg) 4405 { 4406 struct rq *rq = NULL; 4407 struct rq_flags rf; 4408 int ret; 4409 4410 raw_spin_lock_irqsave(&p->pi_lock, rf.flags); 4411 4412 if (__task_needs_rq_lock(p)) 4413 rq = __task_rq_lock(p, &rf); 4414 4415 /* 4416 * At this point the task is pinned; either: 4417 * - blocked and we're holding off wakeups (pi->lock) 4418 * - woken, and we're holding off enqueue (rq->lock) 4419 * - queued, and we're holding off schedule (rq->lock) 4420 * - running, and we're holding off de-schedule (rq->lock) 4421 * 4422 * The called function (@func) can use: task_curr(), p->on_rq and 4423 * p->__state to differentiate between these states. 4424 */ 4425 ret = func(p, arg); 4426 4427 if (rq) 4428 rq_unlock(rq, &rf); 4429 4430 raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags); 4431 return ret; 4432 } 4433 4434 /** 4435 * cpu_curr_snapshot - Return a snapshot of the currently running task 4436 * @cpu: The CPU on which to snapshot the task. 4437 * 4438 * Returns the task_struct pointer of the task "currently" running on 4439 * the specified CPU. If the same task is running on that CPU throughout, 4440 * the return value will be a pointer to that task's task_struct structure. 4441 * If the CPU did any context switches even vaguely concurrently with the 4442 * execution of this function, the return value will be a pointer to the 4443 * task_struct structure of a randomly chosen task that was running on 4444 * that CPU somewhere around the time that this function was executing. 4445 * 4446 * If the specified CPU was offline, the return value is whatever it 4447 * is, perhaps a pointer to the task_struct structure of that CPU's idle 4448 * task, but there is no guarantee. Callers wishing a useful return 4449 * value must take some action to ensure that the specified CPU remains 4450 * online throughout. 4451 * 4452 * This function executes full memory barriers before and after fetching 4453 * the pointer, which permits the caller to confine this function's fetch 4454 * with respect to the caller's accesses to other shared variables. 4455 */ 4456 struct task_struct *cpu_curr_snapshot(int cpu) 4457 { 4458 struct task_struct *t; 4459 4460 smp_mb(); /* Pairing determined by caller's synchronization design. */ 4461 t = rcu_dereference(cpu_curr(cpu)); 4462 smp_mb(); /* Pairing determined by caller's synchronization design. */ 4463 return t; 4464 } 4465 4466 /** 4467 * wake_up_process - Wake up a specific process 4468 * @p: The process to be woken up. 4469 * 4470 * Attempt to wake up the nominated process and move it to the set of runnable 4471 * processes. 4472 * 4473 * Return: 1 if the process was woken up, 0 if it was already running. 4474 * 4475 * This function executes a full memory barrier before accessing the task state. 4476 */ 4477 int wake_up_process(struct task_struct *p) 4478 { 4479 return try_to_wake_up(p, TASK_NORMAL, 0); 4480 } 4481 EXPORT_SYMBOL(wake_up_process); 4482 4483 int wake_up_state(struct task_struct *p, unsigned int state) 4484 { 4485 return try_to_wake_up(p, state, 0); 4486 } 4487 4488 /* 4489 * Perform scheduler related setup for a newly forked process p. 4490 * p is forked by current. 4491 * 4492 * __sched_fork() is basic setup used by init_idle() too: 4493 */ 4494 static void __sched_fork(unsigned long clone_flags, struct task_struct *p) 4495 { 4496 p->on_rq = 0; 4497 4498 p->se.on_rq = 0; 4499 p->se.exec_start = 0; 4500 p->se.sum_exec_runtime = 0; 4501 p->se.prev_sum_exec_runtime = 0; 4502 p->se.nr_migrations = 0; 4503 p->se.vruntime = 0; 4504 p->se.vlag = 0; 4505 p->se.slice = sysctl_sched_base_slice; 4506 INIT_LIST_HEAD(&p->se.group_node); 4507 4508 #ifdef CONFIG_FAIR_GROUP_SCHED 4509 p->se.cfs_rq = NULL; 4510 #endif 4511 4512 #ifdef CONFIG_SCHEDSTATS 4513 /* Even if schedstat is disabled, there should not be garbage */ 4514 memset(&p->stats, 0, sizeof(p->stats)); 4515 #endif 4516 4517 RB_CLEAR_NODE(&p->dl.rb_node); 4518 init_dl_task_timer(&p->dl); 4519 init_dl_inactive_task_timer(&p->dl); 4520 __dl_clear_params(p); 4521 4522 INIT_LIST_HEAD(&p->rt.run_list); 4523 p->rt.timeout = 0; 4524 p->rt.time_slice = sched_rr_timeslice; 4525 p->rt.on_rq = 0; 4526 p->rt.on_list = 0; 4527 4528 #ifdef CONFIG_PREEMPT_NOTIFIERS 4529 INIT_HLIST_HEAD(&p->preempt_notifiers); 4530 #endif 4531 4532 #ifdef CONFIG_COMPACTION 4533 p->capture_control = NULL; 4534 #endif 4535 init_numa_balancing(clone_flags, p); 4536 #ifdef CONFIG_SMP 4537 p->wake_entry.u_flags = CSD_TYPE_TTWU; 4538 p->migration_pending = NULL; 4539 #endif 4540 init_sched_mm_cid(p); 4541 } 4542 4543 DEFINE_STATIC_KEY_FALSE(sched_numa_balancing); 4544 4545 #ifdef CONFIG_NUMA_BALANCING 4546 4547 int sysctl_numa_balancing_mode; 4548 4549 static void __set_numabalancing_state(bool enabled) 4550 { 4551 if (enabled) 4552 static_branch_enable(&sched_numa_balancing); 4553 else 4554 static_branch_disable(&sched_numa_balancing); 4555 } 4556 4557 void set_numabalancing_state(bool enabled) 4558 { 4559 if (enabled) 4560 sysctl_numa_balancing_mode = NUMA_BALANCING_NORMAL; 4561 else 4562 sysctl_numa_balancing_mode = NUMA_BALANCING_DISABLED; 4563 __set_numabalancing_state(enabled); 4564 } 4565 4566 #ifdef CONFIG_PROC_SYSCTL 4567 static void reset_memory_tiering(void) 4568 { 4569 struct pglist_data *pgdat; 4570 4571 for_each_online_pgdat(pgdat) { 4572 pgdat->nbp_threshold = 0; 4573 pgdat->nbp_th_nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE); 4574 pgdat->nbp_th_start = jiffies_to_msecs(jiffies); 4575 } 4576 } 4577 4578 static int sysctl_numa_balancing(struct ctl_table *table, int write, 4579 void *buffer, size_t *lenp, loff_t *ppos) 4580 { 4581 struct ctl_table t; 4582 int err; 4583 int state = sysctl_numa_balancing_mode; 4584 4585 if (write && !capable(CAP_SYS_ADMIN)) 4586 return -EPERM; 4587 4588 t = *table; 4589 t.data = &state; 4590 err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); 4591 if (err < 0) 4592 return err; 4593 if (write) { 4594 if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) && 4595 (state & NUMA_BALANCING_MEMORY_TIERING)) 4596 reset_memory_tiering(); 4597 sysctl_numa_balancing_mode = state; 4598 __set_numabalancing_state(state); 4599 } 4600 return err; 4601 } 4602 #endif 4603 #endif 4604 4605 #ifdef CONFIG_SCHEDSTATS 4606 4607 DEFINE_STATIC_KEY_FALSE(sched_schedstats); 4608 4609 static void set_schedstats(bool enabled) 4610 { 4611 if (enabled) 4612 static_branch_enable(&sched_schedstats); 4613 else 4614 static_branch_disable(&sched_schedstats); 4615 } 4616 4617 void force_schedstat_enabled(void) 4618 { 4619 if (!schedstat_enabled()) { 4620 pr_info("kernel profiling enabled schedstats, disable via kernel.sched_schedstats.\n"); 4621 static_branch_enable(&sched_schedstats); 4622 } 4623 } 4624 4625 static int __init setup_schedstats(char *str) 4626 { 4627 int ret = 0; 4628 if (!str) 4629 goto out; 4630 4631 if (!strcmp(str, "enable")) { 4632 set_schedstats(true); 4633 ret = 1; 4634 } else if (!strcmp(str, "disable")) { 4635 set_schedstats(false); 4636 ret = 1; 4637 } 4638 out: 4639 if (!ret) 4640 pr_warn("Unable to parse schedstats=\n"); 4641 4642 return ret; 4643 } 4644 __setup("schedstats=", setup_schedstats); 4645 4646 #ifdef CONFIG_PROC_SYSCTL 4647 static int sysctl_schedstats(struct ctl_table *table, int write, void *buffer, 4648 size_t *lenp, loff_t *ppos) 4649 { 4650 struct ctl_table t; 4651 int err; 4652 int state = static_branch_likely(&sched_schedstats); 4653 4654 if (write && !capable(CAP_SYS_ADMIN)) 4655 return -EPERM; 4656 4657 t = *table; 4658 t.data = &state; 4659 err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); 4660 if (err < 0) 4661 return err; 4662 if (write) 4663 set_schedstats(state); 4664 return err; 4665 } 4666 #endif /* CONFIG_PROC_SYSCTL */ 4667 #endif /* CONFIG_SCHEDSTATS */ 4668 4669 #ifdef CONFIG_SYSCTL 4670 static struct ctl_table sched_core_sysctls[] = { 4671 #ifdef CONFIG_SCHEDSTATS 4672 { 4673 .procname = "sched_schedstats", 4674 .data = NULL, 4675 .maxlen = sizeof(unsigned int), 4676 .mode = 0644, 4677 .proc_handler = sysctl_schedstats, 4678 .extra1 = SYSCTL_ZERO, 4679 .extra2 = SYSCTL_ONE, 4680 }, 4681 #endif /* CONFIG_SCHEDSTATS */ 4682 #ifdef CONFIG_UCLAMP_TASK 4683 { 4684 .procname = "sched_util_clamp_min", 4685 .data = &sysctl_sched_uclamp_util_min, 4686 .maxlen = sizeof(unsigned int), 4687 .mode = 0644, 4688 .proc_handler = sysctl_sched_uclamp_handler, 4689 }, 4690 { 4691 .procname = "sched_util_clamp_max", 4692 .data = &sysctl_sched_uclamp_util_max, 4693 .maxlen = sizeof(unsigned int), 4694 .mode = 0644, 4695 .proc_handler = sysctl_sched_uclamp_handler, 4696 }, 4697 { 4698 .procname = "sched_util_clamp_min_rt_default", 4699 .data = &sysctl_sched_uclamp_util_min_rt_default, 4700 .maxlen = sizeof(unsigned int), 4701 .mode = 0644, 4702 .proc_handler = sysctl_sched_uclamp_handler, 4703 }, 4704 #endif /* CONFIG_UCLAMP_TASK */ 4705 #ifdef CONFIG_NUMA_BALANCING 4706 { 4707 .procname = "numa_balancing", 4708 .data = NULL, /* filled in by handler */ 4709 .maxlen = sizeof(unsigned int), 4710 .mode = 0644, 4711 .proc_handler = sysctl_numa_balancing, 4712 .extra1 = SYSCTL_ZERO, 4713 .extra2 = SYSCTL_FOUR, 4714 }, 4715 #endif /* CONFIG_NUMA_BALANCING */ 4716 {} 4717 }; 4718 static int __init sched_core_sysctl_init(void) 4719 { 4720 register_sysctl_init("kernel", sched_core_sysctls); 4721 return 0; 4722 } 4723 late_initcall(sched_core_sysctl_init); 4724 #endif /* CONFIG_SYSCTL */ 4725 4726 /* 4727 * fork()/clone()-time setup: 4728 */ 4729 int sched_fork(unsigned long clone_flags, struct task_struct *p) 4730 { 4731 __sched_fork(clone_flags, p); 4732 /* 4733 * We mark the process as NEW here. This guarantees that 4734 * nobody will actually run it, and a signal or other external 4735 * event cannot wake it up and insert it on the runqueue either. 4736 */ 4737 p->__state = TASK_NEW; 4738 4739 /* 4740 * Make sure we do not leak PI boosting priority to the child. 4741 */ 4742 p->prio = current->normal_prio; 4743 4744 uclamp_fork(p); 4745 4746 /* 4747 * Revert to default priority/policy on fork if requested. 4748 */ 4749 if (unlikely(p->sched_reset_on_fork)) { 4750 if (task_has_dl_policy(p) || task_has_rt_policy(p)) { 4751 p->policy = SCHED_NORMAL; 4752 p->static_prio = NICE_TO_PRIO(0); 4753 p->rt_priority = 0; 4754 } else if (PRIO_TO_NICE(p->static_prio) < 0) 4755 p->static_prio = NICE_TO_PRIO(0); 4756 4757 p->prio = p->normal_prio = p->static_prio; 4758 set_load_weight(p, false); 4759 4760 /* 4761 * We don't need the reset flag anymore after the fork. It has 4762 * fulfilled its duty: 4763 */ 4764 p->sched_reset_on_fork = 0; 4765 } 4766 4767 if (dl_prio(p->prio)) 4768 return -EAGAIN; 4769 else if (rt_prio(p->prio)) 4770 p->sched_class = &rt_sched_class; 4771 else 4772 p->sched_class = &fair_sched_class; 4773 4774 init_entity_runnable_average(&p->se); 4775 4776 4777 #ifdef CONFIG_SCHED_INFO 4778 if (likely(sched_info_on())) 4779 memset(&p->sched_info, 0, sizeof(p->sched_info)); 4780 #endif 4781 #if defined(CONFIG_SMP) 4782 p->on_cpu = 0; 4783 #endif 4784 init_task_preempt_count(p); 4785 #ifdef CONFIG_SMP 4786 plist_node_init(&p->pushable_tasks, MAX_PRIO); 4787 RB_CLEAR_NODE(&p->pushable_dl_tasks); 4788 #endif 4789 return 0; 4790 } 4791 4792 void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) 4793 { 4794 unsigned long flags; 4795 4796 /* 4797 * Because we're not yet on the pid-hash, p->pi_lock isn't strictly 4798 * required yet, but lockdep gets upset if rules are violated. 4799 */ 4800 raw_spin_lock_irqsave(&p->pi_lock, flags); 4801 #ifdef CONFIG_CGROUP_SCHED 4802 if (1) { 4803 struct task_group *tg; 4804 tg = container_of(kargs->cset->subsys[cpu_cgrp_id], 4805 struct task_group, css); 4806 tg = autogroup_task_group(p, tg); 4807 p->sched_task_group = tg; 4808 } 4809 #endif 4810 rseq_migrate(p); 4811 /* 4812 * We're setting the CPU for the first time, we don't migrate, 4813 * so use __set_task_cpu(). 4814 */ 4815 __set_task_cpu(p, smp_processor_id()); 4816 if (p->sched_class->task_fork) 4817 p->sched_class->task_fork(p); 4818 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 4819 } 4820 4821 void sched_post_fork(struct task_struct *p) 4822 { 4823 uclamp_post_fork(p); 4824 } 4825 4826 unsigned long to_ratio(u64 period, u64 runtime) 4827 { 4828 if (runtime == RUNTIME_INF) 4829 return BW_UNIT; 4830 4831 /* 4832 * Doing this here saves a lot of checks in all 4833 * the calling paths, and returning zero seems 4834 * safe for them anyway. 4835 */ 4836 if (period == 0) 4837 return 0; 4838 4839 return div64_u64(runtime << BW_SHIFT, period); 4840 } 4841 4842 /* 4843 * wake_up_new_task - wake up a newly created task for the first time. 4844 * 4845 * This function will do some initial scheduler statistics housekeeping 4846 * that must be done for every newly created context, then puts the task 4847 * on the runqueue and wakes it. 4848 */ 4849 void wake_up_new_task(struct task_struct *p) 4850 { 4851 struct rq_flags rf; 4852 struct rq *rq; 4853 4854 raw_spin_lock_irqsave(&p->pi_lock, rf.flags); 4855 WRITE_ONCE(p->__state, TASK_RUNNING); 4856 #ifdef CONFIG_SMP 4857 /* 4858 * Fork balancing, do it here and not earlier because: 4859 * - cpus_ptr can change in the fork path 4860 * - any previously selected CPU might disappear through hotplug 4861 * 4862 * Use __set_task_cpu() to avoid calling sched_class::migrate_task_rq, 4863 * as we're not fully set-up yet. 4864 */ 4865 p->recent_used_cpu = task_cpu(p); 4866 rseq_migrate(p); 4867 __set_task_cpu(p, select_task_rq(p, task_cpu(p), WF_FORK)); 4868 #endif 4869 rq = __task_rq_lock(p, &rf); 4870 update_rq_clock(rq); 4871 post_init_entity_util_avg(p); 4872 4873 activate_task(rq, p, ENQUEUE_NOCLOCK); 4874 trace_sched_wakeup_new(p); 4875 check_preempt_curr(rq, p, WF_FORK); 4876 #ifdef CONFIG_SMP 4877 if (p->sched_class->task_woken) { 4878 /* 4879 * Nothing relies on rq->lock after this, so it's fine to 4880 * drop it. 4881 */ 4882 rq_unpin_lock(rq, &rf); 4883 p->sched_class->task_woken(rq, p); 4884 rq_repin_lock(rq, &rf); 4885 } 4886 #endif 4887 task_rq_unlock(rq, p, &rf); 4888 } 4889 4890 #ifdef CONFIG_PREEMPT_NOTIFIERS 4891 4892 static DEFINE_STATIC_KEY_FALSE(preempt_notifier_key); 4893 4894 void preempt_notifier_inc(void) 4895 { 4896 static_branch_inc(&preempt_notifier_key); 4897 } 4898 EXPORT_SYMBOL_GPL(preempt_notifier_inc); 4899 4900 void preempt_notifier_dec(void) 4901 { 4902 static_branch_dec(&preempt_notifier_key); 4903 } 4904 EXPORT_SYMBOL_GPL(preempt_notifier_dec); 4905 4906 /** 4907 * preempt_notifier_register - tell me when current is being preempted & rescheduled 4908 * @notifier: notifier struct to register 4909 */ 4910 void preempt_notifier_register(struct preempt_notifier *notifier) 4911 { 4912 if (!static_branch_unlikely(&preempt_notifier_key)) 4913 WARN(1, "registering preempt_notifier while notifiers disabled\n"); 4914 4915 hlist_add_head(¬ifier->link, ¤t->preempt_notifiers); 4916 } 4917 EXPORT_SYMBOL_GPL(preempt_notifier_register); 4918 4919 /** 4920 * preempt_notifier_unregister - no longer interested in preemption notifications 4921 * @notifier: notifier struct to unregister 4922 * 4923 * This is *not* safe to call from within a preemption notifier. 4924 */ 4925 void preempt_notifier_unregister(struct preempt_notifier *notifier) 4926 { 4927 hlist_del(¬ifier->link); 4928 } 4929 EXPORT_SYMBOL_GPL(preempt_notifier_unregister); 4930 4931 static void __fire_sched_in_preempt_notifiers(struct task_struct *curr) 4932 { 4933 struct preempt_notifier *notifier; 4934 4935 hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) 4936 notifier->ops->sched_in(notifier, raw_smp_processor_id()); 4937 } 4938 4939 static __always_inline void fire_sched_in_preempt_notifiers(struct task_struct *curr) 4940 { 4941 if (static_branch_unlikely(&preempt_notifier_key)) 4942 __fire_sched_in_preempt_notifiers(curr); 4943 } 4944 4945 static void 4946 __fire_sched_out_preempt_notifiers(struct task_struct *curr, 4947 struct task_struct *next) 4948 { 4949 struct preempt_notifier *notifier; 4950 4951 hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) 4952 notifier->ops->sched_out(notifier, next); 4953 } 4954 4955 static __always_inline void 4956 fire_sched_out_preempt_notifiers(struct task_struct *curr, 4957 struct task_struct *next) 4958 { 4959 if (static_branch_unlikely(&preempt_notifier_key)) 4960 __fire_sched_out_preempt_notifiers(curr, next); 4961 } 4962 4963 #else /* !CONFIG_PREEMPT_NOTIFIERS */ 4964 4965 static inline void fire_sched_in_preempt_notifiers(struct task_struct *curr) 4966 { 4967 } 4968 4969 static inline void 4970 fire_sched_out_preempt_notifiers(struct task_struct *curr, 4971 struct task_struct *next) 4972 { 4973 } 4974 4975 #endif /* CONFIG_PREEMPT_NOTIFIERS */ 4976 4977 static inline void prepare_task(struct task_struct *next) 4978 { 4979 #ifdef CONFIG_SMP 4980 /* 4981 * Claim the task as running, we do this before switching to it 4982 * such that any running task will have this set. 4983 * 4984 * See the smp_load_acquire(&p->on_cpu) case in ttwu() and 4985 * its ordering comment. 4986 */ 4987 WRITE_ONCE(next->on_cpu, 1); 4988 #endif 4989 } 4990 4991 static inline void finish_task(struct task_struct *prev) 4992 { 4993 #ifdef CONFIG_SMP 4994 /* 4995 * This must be the very last reference to @prev from this CPU. After 4996 * p->on_cpu is cleared, the task can be moved to a different CPU. We 4997 * must ensure this doesn't happen until the switch is completely 4998 * finished. 4999 * 5000 * In particular, the load of prev->state in finish_task_switch() must 5001 * happen before this. 5002 * 5003 * Pairs with the smp_cond_load_acquire() in try_to_wake_up(). 5004 */ 5005 smp_store_release(&prev->on_cpu, 0); 5006 #endif 5007 } 5008 5009 #ifdef CONFIG_SMP 5010 5011 static void do_balance_callbacks(struct rq *rq, struct balance_callback *head) 5012 { 5013 void (*func)(struct rq *rq); 5014 struct balance_callback *next; 5015 5016 lockdep_assert_rq_held(rq); 5017 5018 while (head) { 5019 func = (void (*)(struct rq *))head->func; 5020 next = head->next; 5021 head->next = NULL; 5022 head = next; 5023 5024 func(rq); 5025 } 5026 } 5027 5028 static void balance_push(struct rq *rq); 5029 5030 /* 5031 * balance_push_callback is a right abuse of the callback interface and plays 5032 * by significantly different rules. 5033 * 5034 * Where the normal balance_callback's purpose is to be ran in the same context 5035 * that queued it (only later, when it's safe to drop rq->lock again), 5036 * balance_push_callback is specifically targeted at __schedule(). 5037 * 5038 * This abuse is tolerated because it places all the unlikely/odd cases behind 5039 * a single test, namely: rq->balance_callback == NULL. 5040 */ 5041 struct balance_callback balance_push_callback = { 5042 .next = NULL, 5043 .func = balance_push, 5044 }; 5045 5046 static inline struct balance_callback * 5047 __splice_balance_callbacks(struct rq *rq, bool split) 5048 { 5049 struct balance_callback *head = rq->balance_callback; 5050 5051 if (likely(!head)) 5052 return NULL; 5053 5054 lockdep_assert_rq_held(rq); 5055 /* 5056 * Must not take balance_push_callback off the list when 5057 * splice_balance_callbacks() and balance_callbacks() are not 5058 * in the same rq->lock section. 5059 * 5060 * In that case it would be possible for __schedule() to interleave 5061 * and observe the list empty. 5062 */ 5063 if (split && head == &balance_push_callback) 5064 head = NULL; 5065 else 5066 rq->balance_callback = NULL; 5067 5068 return head; 5069 } 5070 5071 static inline struct balance_callback *splice_balance_callbacks(struct rq *rq) 5072 { 5073 return __splice_balance_callbacks(rq, true); 5074 } 5075 5076 static void __balance_callbacks(struct rq *rq) 5077 { 5078 do_balance_callbacks(rq, __splice_balance_callbacks(rq, false)); 5079 } 5080 5081 static inline void balance_callbacks(struct rq *rq, struct balance_callback *head) 5082 { 5083 unsigned long flags; 5084 5085 if (unlikely(head)) { 5086 raw_spin_rq_lock_irqsave(rq, flags); 5087 do_balance_callbacks(rq, head); 5088 raw_spin_rq_unlock_irqrestore(rq, flags); 5089 } 5090 } 5091 5092 #else 5093 5094 static inline void __balance_callbacks(struct rq *rq) 5095 { 5096 } 5097 5098 static inline struct balance_callback *splice_balance_callbacks(struct rq *rq) 5099 { 5100 return NULL; 5101 } 5102 5103 static inline void balance_callbacks(struct rq *rq, struct balance_callback *head) 5104 { 5105 } 5106 5107 #endif 5108 5109 static inline void 5110 prepare_lock_switch(struct rq *rq, struct task_struct *next, struct rq_flags *rf) 5111 { 5112 /* 5113 * Since the runqueue lock will be released by the next 5114 * task (which is an invalid locking op but in the case 5115 * of the scheduler it's an obvious special-case), so we 5116 * do an early lockdep release here: 5117 */ 5118 rq_unpin_lock(rq, rf); 5119 spin_release(&__rq_lockp(rq)->dep_map, _THIS_IP_); 5120 #ifdef CONFIG_DEBUG_SPINLOCK 5121 /* this is a valid case when another task releases the spinlock */ 5122 rq_lockp(rq)->owner = next; 5123 #endif 5124 } 5125 5126 static inline void finish_lock_switch(struct rq *rq) 5127 { 5128 /* 5129 * If we are tracking spinlock dependencies then we have to 5130 * fix up the runqueue lock - which gets 'carried over' from 5131 * prev into current: 5132 */ 5133 spin_acquire(&__rq_lockp(rq)->dep_map, 0, 0, _THIS_IP_); 5134 __balance_callbacks(rq); 5135 raw_spin_rq_unlock_irq(rq); 5136 } 5137 5138 /* 5139 * NOP if the arch has not defined these: 5140 */ 5141 5142 #ifndef prepare_arch_switch 5143 # define prepare_arch_switch(next) do { } while (0) 5144 #endif 5145 5146 #ifndef finish_arch_post_lock_switch 5147 # define finish_arch_post_lock_switch() do { } while (0) 5148 #endif 5149 5150 static inline void kmap_local_sched_out(void) 5151 { 5152 #ifdef CONFIG_KMAP_LOCAL 5153 if (unlikely(current->kmap_ctrl.idx)) 5154 __kmap_local_sched_out(); 5155 #endif 5156 } 5157 5158 static inline void kmap_local_sched_in(void) 5159 { 5160 #ifdef CONFIG_KMAP_LOCAL 5161 if (unlikely(current->kmap_ctrl.idx)) 5162 __kmap_local_sched_in(); 5163 #endif 5164 } 5165 5166 /** 5167 * prepare_task_switch - prepare to switch tasks 5168 * @rq: the runqueue preparing to switch 5169 * @prev: the current task that is being switched out 5170 * @next: the task we are going to switch to. 5171 * 5172 * This is called with the rq lock held and interrupts off. It must 5173 * be paired with a subsequent finish_task_switch after the context 5174 * switch. 5175 * 5176 * prepare_task_switch sets up locking and calls architecture specific 5177 * hooks. 5178 */ 5179 static inline void 5180 prepare_task_switch(struct rq *rq, struct task_struct *prev, 5181 struct task_struct *next) 5182 { 5183 kcov_prepare_switch(prev); 5184 sched_info_switch(rq, prev, next); 5185 perf_event_task_sched_out(prev, next); 5186 rseq_preempt(prev); 5187 fire_sched_out_preempt_notifiers(prev, next); 5188 kmap_local_sched_out(); 5189 prepare_task(next); 5190 prepare_arch_switch(next); 5191 } 5192 5193 /** 5194 * finish_task_switch - clean up after a task-switch 5195 * @prev: the thread we just switched away from. 5196 * 5197 * finish_task_switch must be called after the context switch, paired 5198 * with a prepare_task_switch call before the context switch. 5199 * finish_task_switch will reconcile locking set up by prepare_task_switch, 5200 * and do any other architecture-specific cleanup actions. 5201 * 5202 * Note that we may have delayed dropping an mm in context_switch(). If 5203 * so, we finish that here outside of the runqueue lock. (Doing it 5204 * with the lock held can cause deadlocks; see schedule() for 5205 * details.) 5206 * 5207 * The context switch have flipped the stack from under us and restored the 5208 * local variables which were saved when this task called schedule() in the 5209 * past. prev == current is still correct but we need to recalculate this_rq 5210 * because prev may have moved to another CPU. 5211 */ 5212 static struct rq *finish_task_switch(struct task_struct *prev) 5213 __releases(rq->lock) 5214 { 5215 struct rq *rq = this_rq(); 5216 struct mm_struct *mm = rq->prev_mm; 5217 unsigned int prev_state; 5218 5219 /* 5220 * The previous task will have left us with a preempt_count of 2 5221 * because it left us after: 5222 * 5223 * schedule() 5224 * preempt_disable(); // 1 5225 * __schedule() 5226 * raw_spin_lock_irq(&rq->lock) // 2 5227 * 5228 * Also, see FORK_PREEMPT_COUNT. 5229 */ 5230 if (WARN_ONCE(preempt_count() != 2*PREEMPT_DISABLE_OFFSET, 5231 "corrupted preempt_count: %s/%d/0x%x\n", 5232 current->comm, current->pid, preempt_count())) 5233 preempt_count_set(FORK_PREEMPT_COUNT); 5234 5235 rq->prev_mm = NULL; 5236 5237 /* 5238 * A task struct has one reference for the use as "current". 5239 * If a task dies, then it sets TASK_DEAD in tsk->state and calls 5240 * schedule one last time. The schedule call will never return, and 5241 * the scheduled task must drop that reference. 5242 * 5243 * We must observe prev->state before clearing prev->on_cpu (in 5244 * finish_task), otherwise a concurrent wakeup can get prev 5245 * running on another CPU and we could rave with its RUNNING -> DEAD 5246 * transition, resulting in a double drop. 5247 */ 5248 prev_state = READ_ONCE(prev->__state); 5249 vtime_task_switch(prev); 5250 perf_event_task_sched_in(prev, current); 5251 finish_task(prev); 5252 tick_nohz_task_switch(); 5253 finish_lock_switch(rq); 5254 finish_arch_post_lock_switch(); 5255 kcov_finish_switch(current); 5256 /* 5257 * kmap_local_sched_out() is invoked with rq::lock held and 5258 * interrupts disabled. There is no requirement for that, but the 5259 * sched out code does not have an interrupt enabled section. 5260 * Restoring the maps on sched in does not require interrupts being 5261 * disabled either. 5262 */ 5263 kmap_local_sched_in(); 5264 5265 fire_sched_in_preempt_notifiers(current); 5266 /* 5267 * When switching through a kernel thread, the loop in 5268 * membarrier_{private,global}_expedited() may have observed that 5269 * kernel thread and not issued an IPI. It is therefore possible to 5270 * schedule between user->kernel->user threads without passing though 5271 * switch_mm(). Membarrier requires a barrier after storing to 5272 * rq->curr, before returning to userspace, so provide them here: 5273 * 5274 * - a full memory barrier for {PRIVATE,GLOBAL}_EXPEDITED, implicitly 5275 * provided by mmdrop_lazy_tlb(), 5276 * - a sync_core for SYNC_CORE. 5277 */ 5278 if (mm) { 5279 membarrier_mm_sync_core_before_usermode(mm); 5280 mmdrop_lazy_tlb_sched(mm); 5281 } 5282 5283 if (unlikely(prev_state == TASK_DEAD)) { 5284 if (prev->sched_class->task_dead) 5285 prev->sched_class->task_dead(prev); 5286 5287 /* Task is done with its stack. */ 5288 put_task_stack(prev); 5289 5290 put_task_struct_rcu_user(prev); 5291 } 5292 5293 return rq; 5294 } 5295 5296 /** 5297 * schedule_tail - first thing a freshly forked thread must call. 5298 * @prev: the thread we just switched away from. 5299 */ 5300 asmlinkage __visible void schedule_tail(struct task_struct *prev) 5301 __releases(rq->lock) 5302 { 5303 /* 5304 * New tasks start with FORK_PREEMPT_COUNT, see there and 5305 * finish_task_switch() for details. 5306 * 5307 * finish_task_switch() will drop rq->lock() and lower preempt_count 5308 * and the preempt_enable() will end up enabling preemption (on 5309 * PREEMPT_COUNT kernels). 5310 */ 5311 5312 finish_task_switch(prev); 5313 preempt_enable(); 5314 5315 if (current->set_child_tid) 5316 put_user(task_pid_vnr(current), current->set_child_tid); 5317 5318 calculate_sigpending(); 5319 } 5320 5321 /* 5322 * context_switch - switch to the new MM and the new thread's register state. 5323 */ 5324 static __always_inline struct rq * 5325 context_switch(struct rq *rq, struct task_struct *prev, 5326 struct task_struct *next, struct rq_flags *rf) 5327 { 5328 prepare_task_switch(rq, prev, next); 5329 5330 /* 5331 * For paravirt, this is coupled with an exit in switch_to to 5332 * combine the page table reload and the switch backend into 5333 * one hypercall. 5334 */ 5335 arch_start_context_switch(prev); 5336 5337 /* 5338 * kernel -> kernel lazy + transfer active 5339 * user -> kernel lazy + mmgrab_lazy_tlb() active 5340 * 5341 * kernel -> user switch + mmdrop_lazy_tlb() active 5342 * user -> user switch 5343 * 5344 * switch_mm_cid() needs to be updated if the barriers provided 5345 * by context_switch() are modified. 5346 */ 5347 if (!next->mm) { // to kernel 5348 enter_lazy_tlb(prev->active_mm, next); 5349 5350 next->active_mm = prev->active_mm; 5351 if (prev->mm) // from user 5352 mmgrab_lazy_tlb(prev->active_mm); 5353 else 5354 prev->active_mm = NULL; 5355 } else { // to user 5356 membarrier_switch_mm(rq, prev->active_mm, next->mm); 5357 /* 5358 * sys_membarrier() requires an smp_mb() between setting 5359 * rq->curr / membarrier_switch_mm() and returning to userspace. 5360 * 5361 * The below provides this either through switch_mm(), or in 5362 * case 'prev->active_mm == next->mm' through 5363 * finish_task_switch()'s mmdrop(). 5364 */ 5365 switch_mm_irqs_off(prev->active_mm, next->mm, next); 5366 lru_gen_use_mm(next->mm); 5367 5368 if (!prev->mm) { // from kernel 5369 /* will mmdrop_lazy_tlb() in finish_task_switch(). */ 5370 rq->prev_mm = prev->active_mm; 5371 prev->active_mm = NULL; 5372 } 5373 } 5374 5375 /* switch_mm_cid() requires the memory barriers above. */ 5376 switch_mm_cid(rq, prev, next); 5377 5378 rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP); 5379 5380 prepare_lock_switch(rq, next, rf); 5381 5382 /* Here we just switch the register state and the stack. */ 5383 switch_to(prev, next, prev); 5384 barrier(); 5385 5386 return finish_task_switch(prev); 5387 } 5388 5389 /* 5390 * nr_running and nr_context_switches: 5391 * 5392 * externally visible scheduler statistics: current number of runnable 5393 * threads, total number of context switches performed since bootup. 5394 */ 5395 unsigned int nr_running(void) 5396 { 5397 unsigned int i, sum = 0; 5398 5399 for_each_online_cpu(i) 5400 sum += cpu_rq(i)->nr_running; 5401 5402 return sum; 5403 } 5404 5405 /* 5406 * Check if only the current task is running on the CPU. 5407 * 5408 * Caution: this function does not check that the caller has disabled 5409 * preemption, thus the result might have a time-of-check-to-time-of-use 5410 * race. The caller is responsible to use it correctly, for example: 5411 * 5412 * - from a non-preemptible section (of course) 5413 * 5414 * - from a thread that is bound to a single CPU 5415 * 5416 * - in a loop with very short iterations (e.g. a polling loop) 5417 */ 5418 bool single_task_running(void) 5419 { 5420 return raw_rq()->nr_running == 1; 5421 } 5422 EXPORT_SYMBOL(single_task_running); 5423 5424 unsigned long long nr_context_switches_cpu(int cpu) 5425 { 5426 return cpu_rq(cpu)->nr_switches; 5427 } 5428 5429 unsigned long long nr_context_switches(void) 5430 { 5431 int i; 5432 unsigned long long sum = 0; 5433 5434 for_each_possible_cpu(i) 5435 sum += cpu_rq(i)->nr_switches; 5436 5437 return sum; 5438 } 5439 5440 /* 5441 * Consumers of these two interfaces, like for example the cpuidle menu 5442 * governor, are using nonsensical data. Preferring shallow idle state selection 5443 * for a CPU that has IO-wait which might not even end up running the task when 5444 * it does become runnable. 5445 */ 5446 5447 unsigned int nr_iowait_cpu(int cpu) 5448 { 5449 return atomic_read(&cpu_rq(cpu)->nr_iowait); 5450 } 5451 5452 /* 5453 * IO-wait accounting, and how it's mostly bollocks (on SMP). 5454 * 5455 * The idea behind IO-wait account is to account the idle time that we could 5456 * have spend running if it were not for IO. That is, if we were to improve the 5457 * storage performance, we'd have a proportional reduction in IO-wait time. 5458 * 5459 * This all works nicely on UP, where, when a task blocks on IO, we account 5460 * idle time as IO-wait, because if the storage were faster, it could've been 5461 * running and we'd not be idle. 5462 * 5463 * This has been extended to SMP, by doing the same for each CPU. This however 5464 * is broken. 5465 * 5466 * Imagine for instance the case where two tasks block on one CPU, only the one 5467 * CPU will have IO-wait accounted, while the other has regular idle. Even 5468 * though, if the storage were faster, both could've ran at the same time, 5469 * utilising both CPUs. 5470 * 5471 * This means, that when looking globally, the current IO-wait accounting on 5472 * SMP is a lower bound, by reason of under accounting. 5473 * 5474 * Worse, since the numbers are provided per CPU, they are sometimes 5475 * interpreted per CPU, and that is nonsensical. A blocked task isn't strictly 5476 * associated with any one particular CPU, it can wake to another CPU than it 5477 * blocked on. This means the per CPU IO-wait number is meaningless. 5478 * 5479 * Task CPU affinities can make all that even more 'interesting'. 5480 */ 5481 5482 unsigned int nr_iowait(void) 5483 { 5484 unsigned int i, sum = 0; 5485 5486 for_each_possible_cpu(i) 5487 sum += nr_iowait_cpu(i); 5488 5489 return sum; 5490 } 5491 5492 #ifdef CONFIG_SMP 5493 5494 /* 5495 * sched_exec - execve() is a valuable balancing opportunity, because at 5496 * this point the task has the smallest effective memory and cache footprint. 5497 */ 5498 void sched_exec(void) 5499 { 5500 struct task_struct *p = current; 5501 unsigned long flags; 5502 int dest_cpu; 5503 5504 raw_spin_lock_irqsave(&p->pi_lock, flags); 5505 dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), WF_EXEC); 5506 if (dest_cpu == smp_processor_id()) 5507 goto unlock; 5508 5509 if (likely(cpu_active(dest_cpu))) { 5510 struct migration_arg arg = { p, dest_cpu }; 5511 5512 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 5513 stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg); 5514 return; 5515 } 5516 unlock: 5517 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 5518 } 5519 5520 #endif 5521 5522 DEFINE_PER_CPU(struct kernel_stat, kstat); 5523 DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat); 5524 5525 EXPORT_PER_CPU_SYMBOL(kstat); 5526 EXPORT_PER_CPU_SYMBOL(kernel_cpustat); 5527 5528 /* 5529 * The function fair_sched_class.update_curr accesses the struct curr 5530 * and its field curr->exec_start; when called from task_sched_runtime(), 5531 * we observe a high rate of cache misses in practice. 5532 * Prefetching this data results in improved performance. 5533 */ 5534 static inline void prefetch_curr_exec_start(struct task_struct *p) 5535 { 5536 #ifdef CONFIG_FAIR_GROUP_SCHED 5537 struct sched_entity *curr = (&p->se)->cfs_rq->curr; 5538 #else 5539 struct sched_entity *curr = (&task_rq(p)->cfs)->curr; 5540 #endif 5541 prefetch(curr); 5542 prefetch(&curr->exec_start); 5543 } 5544 5545 /* 5546 * Return accounted runtime for the task. 5547 * In case the task is currently running, return the runtime plus current's 5548 * pending runtime that have not been accounted yet. 5549 */ 5550 unsigned long long task_sched_runtime(struct task_struct *p) 5551 { 5552 struct rq_flags rf; 5553 struct rq *rq; 5554 u64 ns; 5555 5556 #if defined(CONFIG_64BIT) && defined(CONFIG_SMP) 5557 /* 5558 * 64-bit doesn't need locks to atomically read a 64-bit value. 5559 * So we have a optimization chance when the task's delta_exec is 0. 5560 * Reading ->on_cpu is racy, but this is ok. 5561 * 5562 * If we race with it leaving CPU, we'll take a lock. So we're correct. 5563 * If we race with it entering CPU, unaccounted time is 0. This is 5564 * indistinguishable from the read occurring a few cycles earlier. 5565 * If we see ->on_cpu without ->on_rq, the task is leaving, and has 5566 * been accounted, so we're correct here as well. 5567 */ 5568 if (!p->on_cpu || !task_on_rq_queued(p)) 5569 return p->se.sum_exec_runtime; 5570 #endif 5571 5572 rq = task_rq_lock(p, &rf); 5573 /* 5574 * Must be ->curr _and_ ->on_rq. If dequeued, we would 5575 * project cycles that may never be accounted to this 5576 * thread, breaking clock_gettime(). 5577 */ 5578 if (task_current(rq, p) && task_on_rq_queued(p)) { 5579 prefetch_curr_exec_start(p); 5580 update_rq_clock(rq); 5581 p->sched_class->update_curr(rq); 5582 } 5583 ns = p->se.sum_exec_runtime; 5584 task_rq_unlock(rq, p, &rf); 5585 5586 return ns; 5587 } 5588 5589 #ifdef CONFIG_SCHED_DEBUG 5590 static u64 cpu_resched_latency(struct rq *rq) 5591 { 5592 int latency_warn_ms = READ_ONCE(sysctl_resched_latency_warn_ms); 5593 u64 resched_latency, now = rq_clock(rq); 5594 static bool warned_once; 5595 5596 if (sysctl_resched_latency_warn_once && warned_once) 5597 return 0; 5598 5599 if (!need_resched() || !latency_warn_ms) 5600 return 0; 5601 5602 if (system_state == SYSTEM_BOOTING) 5603 return 0; 5604 5605 if (!rq->last_seen_need_resched_ns) { 5606 rq->last_seen_need_resched_ns = now; 5607 rq->ticks_without_resched = 0; 5608 return 0; 5609 } 5610 5611 rq->ticks_without_resched++; 5612 resched_latency = now - rq->last_seen_need_resched_ns; 5613 if (resched_latency <= latency_warn_ms * NSEC_PER_MSEC) 5614 return 0; 5615 5616 warned_once = true; 5617 5618 return resched_latency; 5619 } 5620 5621 static int __init setup_resched_latency_warn_ms(char *str) 5622 { 5623 long val; 5624 5625 if ((kstrtol(str, 0, &val))) { 5626 pr_warn("Unable to set resched_latency_warn_ms\n"); 5627 return 1; 5628 } 5629 5630 sysctl_resched_latency_warn_ms = val; 5631 return 1; 5632 } 5633 __setup("resched_latency_warn_ms=", setup_resched_latency_warn_ms); 5634 #else 5635 static inline u64 cpu_resched_latency(struct rq *rq) { return 0; } 5636 #endif /* CONFIG_SCHED_DEBUG */ 5637 5638 /* 5639 * This function gets called by the timer code, with HZ frequency. 5640 * We call it with interrupts disabled. 5641 */ 5642 void scheduler_tick(void) 5643 { 5644 int cpu = smp_processor_id(); 5645 struct rq *rq = cpu_rq(cpu); 5646 struct task_struct *curr = rq->curr; 5647 struct rq_flags rf; 5648 unsigned long thermal_pressure; 5649 u64 resched_latency; 5650 5651 if (housekeeping_cpu(cpu, HK_TYPE_TICK)) 5652 arch_scale_freq_tick(); 5653 5654 sched_clock_tick(); 5655 5656 rq_lock(rq, &rf); 5657 5658 update_rq_clock(rq); 5659 thermal_pressure = arch_scale_thermal_pressure(cpu_of(rq)); 5660 update_thermal_load_avg(rq_clock_thermal(rq), rq, thermal_pressure); 5661 curr->sched_class->task_tick(rq, curr, 0); 5662 if (sched_feat(LATENCY_WARN)) 5663 resched_latency = cpu_resched_latency(rq); 5664 calc_global_load_tick(rq); 5665 sched_core_tick(rq); 5666 task_tick_mm_cid(rq, curr); 5667 5668 rq_unlock(rq, &rf); 5669 5670 if (sched_feat(LATENCY_WARN) && resched_latency) 5671 resched_latency_warn(cpu, resched_latency); 5672 5673 perf_event_task_tick(); 5674 5675 if (curr->flags & PF_WQ_WORKER) 5676 wq_worker_tick(curr); 5677 5678 #ifdef CONFIG_SMP 5679 rq->idle_balance = idle_cpu(cpu); 5680 trigger_load_balance(rq); 5681 #endif 5682 } 5683 5684 #ifdef CONFIG_NO_HZ_FULL 5685 5686 struct tick_work { 5687 int cpu; 5688 atomic_t state; 5689 struct delayed_work work; 5690 }; 5691 /* Values for ->state, see diagram below. */ 5692 #define TICK_SCHED_REMOTE_OFFLINE 0 5693 #define TICK_SCHED_REMOTE_OFFLINING 1 5694 #define TICK_SCHED_REMOTE_RUNNING 2 5695 5696 /* 5697 * State diagram for ->state: 5698 * 5699 * 5700 * TICK_SCHED_REMOTE_OFFLINE 5701 * | ^ 5702 * | | 5703 * | | sched_tick_remote() 5704 * | | 5705 * | | 5706 * +--TICK_SCHED_REMOTE_OFFLINING 5707 * | ^ 5708 * | | 5709 * sched_tick_start() | | sched_tick_stop() 5710 * | | 5711 * V | 5712 * TICK_SCHED_REMOTE_RUNNING 5713 * 5714 * 5715 * Other transitions get WARN_ON_ONCE(), except that sched_tick_remote() 5716 * and sched_tick_start() are happy to leave the state in RUNNING. 5717 */ 5718 5719 static struct tick_work __percpu *tick_work_cpu; 5720 5721 static void sched_tick_remote(struct work_struct *work) 5722 { 5723 struct delayed_work *dwork = to_delayed_work(work); 5724 struct tick_work *twork = container_of(dwork, struct tick_work, work); 5725 int cpu = twork->cpu; 5726 struct rq *rq = cpu_rq(cpu); 5727 struct task_struct *curr; 5728 struct rq_flags rf; 5729 u64 delta; 5730 int os; 5731 5732 /* 5733 * Handle the tick only if it appears the remote CPU is running in full 5734 * dynticks mode. The check is racy by nature, but missing a tick or 5735 * having one too much is no big deal because the scheduler tick updates 5736 * statistics and checks timeslices in a time-independent way, regardless 5737 * of when exactly it is running. 5738 */ 5739 if (!tick_nohz_tick_stopped_cpu(cpu)) 5740 goto out_requeue; 5741 5742 rq_lock_irq(rq, &rf); 5743 curr = rq->curr; 5744 if (cpu_is_offline(cpu)) 5745 goto out_unlock; 5746 5747 update_rq_clock(rq); 5748 5749 if (!is_idle_task(curr)) { 5750 /* 5751 * Make sure the next tick runs within a reasonable 5752 * amount of time. 5753 */ 5754 delta = rq_clock_task(rq) - curr->se.exec_start; 5755 WARN_ON_ONCE(delta > (u64)NSEC_PER_SEC * 3); 5756 } 5757 curr->sched_class->task_tick(rq, curr, 0); 5758 5759 calc_load_nohz_remote(rq); 5760 out_unlock: 5761 rq_unlock_irq(rq, &rf); 5762 out_requeue: 5763 5764 /* 5765 * Run the remote tick once per second (1Hz). This arbitrary 5766 * frequency is large enough to avoid overload but short enough 5767 * to keep scheduler internal stats reasonably up to date. But 5768 * first update state to reflect hotplug activity if required. 5769 */ 5770 os = atomic_fetch_add_unless(&twork->state, -1, TICK_SCHED_REMOTE_RUNNING); 5771 WARN_ON_ONCE(os == TICK_SCHED_REMOTE_OFFLINE); 5772 if (os == TICK_SCHED_REMOTE_RUNNING) 5773 queue_delayed_work(system_unbound_wq, dwork, HZ); 5774 } 5775 5776 static void sched_tick_start(int cpu) 5777 { 5778 int os; 5779 struct tick_work *twork; 5780 5781 if (housekeeping_cpu(cpu, HK_TYPE_TICK)) 5782 return; 5783 5784 WARN_ON_ONCE(!tick_work_cpu); 5785 5786 twork = per_cpu_ptr(tick_work_cpu, cpu); 5787 os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_RUNNING); 5788 WARN_ON_ONCE(os == TICK_SCHED_REMOTE_RUNNING); 5789 if (os == TICK_SCHED_REMOTE_OFFLINE) { 5790 twork->cpu = cpu; 5791 INIT_DELAYED_WORK(&twork->work, sched_tick_remote); 5792 queue_delayed_work(system_unbound_wq, &twork->work, HZ); 5793 } 5794 } 5795 5796 #ifdef CONFIG_HOTPLUG_CPU 5797 static void sched_tick_stop(int cpu) 5798 { 5799 struct tick_work *twork; 5800 int os; 5801 5802 if (housekeeping_cpu(cpu, HK_TYPE_TICK)) 5803 return; 5804 5805 WARN_ON_ONCE(!tick_work_cpu); 5806 5807 twork = per_cpu_ptr(tick_work_cpu, cpu); 5808 /* There cannot be competing actions, but don't rely on stop-machine. */ 5809 os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_OFFLINING); 5810 WARN_ON_ONCE(os != TICK_SCHED_REMOTE_RUNNING); 5811 /* Don't cancel, as this would mess up the state machine. */ 5812 } 5813 #endif /* CONFIG_HOTPLUG_CPU */ 5814 5815 int __init sched_tick_offload_init(void) 5816 { 5817 tick_work_cpu = alloc_percpu(struct tick_work); 5818 BUG_ON(!tick_work_cpu); 5819 return 0; 5820 } 5821 5822 #else /* !CONFIG_NO_HZ_FULL */ 5823 static inline void sched_tick_start(int cpu) { } 5824 static inline void sched_tick_stop(int cpu) { } 5825 #endif 5826 5827 #if defined(CONFIG_PREEMPTION) && (defined(CONFIG_DEBUG_PREEMPT) || \ 5828 defined(CONFIG_TRACE_PREEMPT_TOGGLE)) 5829 /* 5830 * If the value passed in is equal to the current preempt count 5831 * then we just disabled preemption. Start timing the latency. 5832 */ 5833 static inline void preempt_latency_start(int val) 5834 { 5835 if (preempt_count() == val) { 5836 unsigned long ip = get_lock_parent_ip(); 5837 #ifdef CONFIG_DEBUG_PREEMPT 5838 current->preempt_disable_ip = ip; 5839 #endif 5840 trace_preempt_off(CALLER_ADDR0, ip); 5841 } 5842 } 5843 5844 void preempt_count_add(int val) 5845 { 5846 #ifdef CONFIG_DEBUG_PREEMPT 5847 /* 5848 * Underflow? 5849 */ 5850 if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) 5851 return; 5852 #endif 5853 __preempt_count_add(val); 5854 #ifdef CONFIG_DEBUG_PREEMPT 5855 /* 5856 * Spinlock count overflowing soon? 5857 */ 5858 DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >= 5859 PREEMPT_MASK - 10); 5860 #endif 5861 preempt_latency_start(val); 5862 } 5863 EXPORT_SYMBOL(preempt_count_add); 5864 NOKPROBE_SYMBOL(preempt_count_add); 5865 5866 /* 5867 * If the value passed in equals to the current preempt count 5868 * then we just enabled preemption. Stop timing the latency. 5869 */ 5870 static inline void preempt_latency_stop(int val) 5871 { 5872 if (preempt_count() == val) 5873 trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip()); 5874 } 5875 5876 void preempt_count_sub(int val) 5877 { 5878 #ifdef CONFIG_DEBUG_PREEMPT 5879 /* 5880 * Underflow? 5881 */ 5882 if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) 5883 return; 5884 /* 5885 * Is the spinlock portion underflowing? 5886 */ 5887 if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) && 5888 !(preempt_count() & PREEMPT_MASK))) 5889 return; 5890 #endif 5891 5892 preempt_latency_stop(val); 5893 __preempt_count_sub(val); 5894 } 5895 EXPORT_SYMBOL(preempt_count_sub); 5896 NOKPROBE_SYMBOL(preempt_count_sub); 5897 5898 #else 5899 static inline void preempt_latency_start(int val) { } 5900 static inline void preempt_latency_stop(int val) { } 5901 #endif 5902 5903 static inline unsigned long get_preempt_disable_ip(struct task_struct *p) 5904 { 5905 #ifdef CONFIG_DEBUG_PREEMPT 5906 return p->preempt_disable_ip; 5907 #else 5908 return 0; 5909 #endif 5910 } 5911 5912 /* 5913 * Print scheduling while atomic bug: 5914 */ 5915 static noinline void __schedule_bug(struct task_struct *prev) 5916 { 5917 /* Save this before calling printk(), since that will clobber it */ 5918 unsigned long preempt_disable_ip = get_preempt_disable_ip(current); 5919 5920 if (oops_in_progress) 5921 return; 5922 5923 printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n", 5924 prev->comm, prev->pid, preempt_count()); 5925 5926 debug_show_held_locks(prev); 5927 print_modules(); 5928 if (irqs_disabled()) 5929 print_irqtrace_events(prev); 5930 if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) 5931 && in_atomic_preempt_off()) { 5932 pr_err("Preemption disabled at:"); 5933 print_ip_sym(KERN_ERR, preempt_disable_ip); 5934 } 5935 check_panic_on_warn("scheduling while atomic"); 5936 5937 dump_stack(); 5938 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 5939 } 5940 5941 /* 5942 * Various schedule()-time debugging checks and statistics: 5943 */ 5944 static inline void schedule_debug(struct task_struct *prev, bool preempt) 5945 { 5946 #ifdef CONFIG_SCHED_STACK_END_CHECK 5947 if (task_stack_end_corrupted(prev)) 5948 panic("corrupted stack end detected inside scheduler\n"); 5949 5950 if (task_scs_end_corrupted(prev)) 5951 panic("corrupted shadow stack detected inside scheduler\n"); 5952 #endif 5953 5954 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP 5955 if (!preempt && READ_ONCE(prev->__state) && prev->non_block_count) { 5956 printk(KERN_ERR "BUG: scheduling in a non-blocking section: %s/%d/%i\n", 5957 prev->comm, prev->pid, prev->non_block_count); 5958 dump_stack(); 5959 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 5960 } 5961 #endif 5962 5963 if (unlikely(in_atomic_preempt_off())) { 5964 __schedule_bug(prev); 5965 preempt_count_set(PREEMPT_DISABLED); 5966 } 5967 rcu_sleep_check(); 5968 SCHED_WARN_ON(ct_state() == CONTEXT_USER); 5969 5970 profile_hit(SCHED_PROFILING, __builtin_return_address(0)); 5971 5972 schedstat_inc(this_rq()->sched_count); 5973 } 5974 5975 static void put_prev_task_balance(struct rq *rq, struct task_struct *prev, 5976 struct rq_flags *rf) 5977 { 5978 #ifdef CONFIG_SMP 5979 const struct sched_class *class; 5980 /* 5981 * We must do the balancing pass before put_prev_task(), such 5982 * that when we release the rq->lock the task is in the same 5983 * state as before we took rq->lock. 5984 * 5985 * We can terminate the balance pass as soon as we know there is 5986 * a runnable task of @class priority or higher. 5987 */ 5988 for_class_range(class, prev->sched_class, &idle_sched_class) { 5989 if (class->balance(rq, prev, rf)) 5990 break; 5991 } 5992 #endif 5993 5994 put_prev_task(rq, prev); 5995 } 5996 5997 /* 5998 * Pick up the highest-prio task: 5999 */ 6000 static inline struct task_struct * 6001 __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 6002 { 6003 const struct sched_class *class; 6004 struct task_struct *p; 6005 6006 /* 6007 * Optimization: we know that if all tasks are in the fair class we can 6008 * call that function directly, but only if the @prev task wasn't of a 6009 * higher scheduling class, because otherwise those lose the 6010 * opportunity to pull in more work from other CPUs. 6011 */ 6012 if (likely(!sched_class_above(prev->sched_class, &fair_sched_class) && 6013 rq->nr_running == rq->cfs.h_nr_running)) { 6014 6015 p = pick_next_task_fair(rq, prev, rf); 6016 if (unlikely(p == RETRY_TASK)) 6017 goto restart; 6018 6019 /* Assume the next prioritized class is idle_sched_class */ 6020 if (!p) { 6021 put_prev_task(rq, prev); 6022 p = pick_next_task_idle(rq); 6023 } 6024 6025 return p; 6026 } 6027 6028 restart: 6029 put_prev_task_balance(rq, prev, rf); 6030 6031 for_each_class(class) { 6032 p = class->pick_next_task(rq); 6033 if (p) 6034 return p; 6035 } 6036 6037 BUG(); /* The idle class should always have a runnable task. */ 6038 } 6039 6040 #ifdef CONFIG_SCHED_CORE 6041 static inline bool is_task_rq_idle(struct task_struct *t) 6042 { 6043 return (task_rq(t)->idle == t); 6044 } 6045 6046 static inline bool cookie_equals(struct task_struct *a, unsigned long cookie) 6047 { 6048 return is_task_rq_idle(a) || (a->core_cookie == cookie); 6049 } 6050 6051 static inline bool cookie_match(struct task_struct *a, struct task_struct *b) 6052 { 6053 if (is_task_rq_idle(a) || is_task_rq_idle(b)) 6054 return true; 6055 6056 return a->core_cookie == b->core_cookie; 6057 } 6058 6059 static inline struct task_struct *pick_task(struct rq *rq) 6060 { 6061 const struct sched_class *class; 6062 struct task_struct *p; 6063 6064 for_each_class(class) { 6065 p = class->pick_task(rq); 6066 if (p) 6067 return p; 6068 } 6069 6070 BUG(); /* The idle class should always have a runnable task. */ 6071 } 6072 6073 extern void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi); 6074 6075 static void queue_core_balance(struct rq *rq); 6076 6077 static struct task_struct * 6078 pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 6079 { 6080 struct task_struct *next, *p, *max = NULL; 6081 const struct cpumask *smt_mask; 6082 bool fi_before = false; 6083 bool core_clock_updated = (rq == rq->core); 6084 unsigned long cookie; 6085 int i, cpu, occ = 0; 6086 struct rq *rq_i; 6087 bool need_sync; 6088 6089 if (!sched_core_enabled(rq)) 6090 return __pick_next_task(rq, prev, rf); 6091 6092 cpu = cpu_of(rq); 6093 6094 /* Stopper task is switching into idle, no need core-wide selection. */ 6095 if (cpu_is_offline(cpu)) { 6096 /* 6097 * Reset core_pick so that we don't enter the fastpath when 6098 * coming online. core_pick would already be migrated to 6099 * another cpu during offline. 6100 */ 6101 rq->core_pick = NULL; 6102 return __pick_next_task(rq, prev, rf); 6103 } 6104 6105 /* 6106 * If there were no {en,de}queues since we picked (IOW, the task 6107 * pointers are all still valid), and we haven't scheduled the last 6108 * pick yet, do so now. 6109 * 6110 * rq->core_pick can be NULL if no selection was made for a CPU because 6111 * it was either offline or went offline during a sibling's core-wide 6112 * selection. In this case, do a core-wide selection. 6113 */ 6114 if (rq->core->core_pick_seq == rq->core->core_task_seq && 6115 rq->core->core_pick_seq != rq->core_sched_seq && 6116 rq->core_pick) { 6117 WRITE_ONCE(rq->core_sched_seq, rq->core->core_pick_seq); 6118 6119 next = rq->core_pick; 6120 if (next != prev) { 6121 put_prev_task(rq, prev); 6122 set_next_task(rq, next); 6123 } 6124 6125 rq->core_pick = NULL; 6126 goto out; 6127 } 6128 6129 put_prev_task_balance(rq, prev, rf); 6130 6131 smt_mask = cpu_smt_mask(cpu); 6132 need_sync = !!rq->core->core_cookie; 6133 6134 /* reset state */ 6135 rq->core->core_cookie = 0UL; 6136 if (rq->core->core_forceidle_count) { 6137 if (!core_clock_updated) { 6138 update_rq_clock(rq->core); 6139 core_clock_updated = true; 6140 } 6141 sched_core_account_forceidle(rq); 6142 /* reset after accounting force idle */ 6143 rq->core->core_forceidle_start = 0; 6144 rq->core->core_forceidle_count = 0; 6145 rq->core->core_forceidle_occupation = 0; 6146 need_sync = true; 6147 fi_before = true; 6148 } 6149 6150 /* 6151 * core->core_task_seq, core->core_pick_seq, rq->core_sched_seq 6152 * 6153 * @task_seq guards the task state ({en,de}queues) 6154 * @pick_seq is the @task_seq we did a selection on 6155 * @sched_seq is the @pick_seq we scheduled 6156 * 6157 * However, preemptions can cause multiple picks on the same task set. 6158 * 'Fix' this by also increasing @task_seq for every pick. 6159 */ 6160 rq->core->core_task_seq++; 6161 6162 /* 6163 * Optimize for common case where this CPU has no cookies 6164 * and there are no cookied tasks running on siblings. 6165 */ 6166 if (!need_sync) { 6167 next = pick_task(rq); 6168 if (!next->core_cookie) { 6169 rq->core_pick = NULL; 6170 /* 6171 * For robustness, update the min_vruntime_fi for 6172 * unconstrained picks as well. 6173 */ 6174 WARN_ON_ONCE(fi_before); 6175 task_vruntime_update(rq, next, false); 6176 goto out_set_next; 6177 } 6178 } 6179 6180 /* 6181 * For each thread: do the regular task pick and find the max prio task 6182 * amongst them. 6183 * 6184 * Tie-break prio towards the current CPU 6185 */ 6186 for_each_cpu_wrap(i, smt_mask, cpu) { 6187 rq_i = cpu_rq(i); 6188 6189 /* 6190 * Current cpu always has its clock updated on entrance to 6191 * pick_next_task(). If the current cpu is not the core, 6192 * the core may also have been updated above. 6193 */ 6194 if (i != cpu && (rq_i != rq->core || !core_clock_updated)) 6195 update_rq_clock(rq_i); 6196 6197 p = rq_i->core_pick = pick_task(rq_i); 6198 if (!max || prio_less(max, p, fi_before)) 6199 max = p; 6200 } 6201 6202 cookie = rq->core->core_cookie = max->core_cookie; 6203 6204 /* 6205 * For each thread: try and find a runnable task that matches @max or 6206 * force idle. 6207 */ 6208 for_each_cpu(i, smt_mask) { 6209 rq_i = cpu_rq(i); 6210 p = rq_i->core_pick; 6211 6212 if (!cookie_equals(p, cookie)) { 6213 p = NULL; 6214 if (cookie) 6215 p = sched_core_find(rq_i, cookie); 6216 if (!p) 6217 p = idle_sched_class.pick_task(rq_i); 6218 } 6219 6220 rq_i->core_pick = p; 6221 6222 if (p == rq_i->idle) { 6223 if (rq_i->nr_running) { 6224 rq->core->core_forceidle_count++; 6225 if (!fi_before) 6226 rq->core->core_forceidle_seq++; 6227 } 6228 } else { 6229 occ++; 6230 } 6231 } 6232 6233 if (schedstat_enabled() && rq->core->core_forceidle_count) { 6234 rq->core->core_forceidle_start = rq_clock(rq->core); 6235 rq->core->core_forceidle_occupation = occ; 6236 } 6237 6238 rq->core->core_pick_seq = rq->core->core_task_seq; 6239 next = rq->core_pick; 6240 rq->core_sched_seq = rq->core->core_pick_seq; 6241 6242 /* Something should have been selected for current CPU */ 6243 WARN_ON_ONCE(!next); 6244 6245 /* 6246 * Reschedule siblings 6247 * 6248 * NOTE: L1TF -- at this point we're no longer running the old task and 6249 * sending an IPI (below) ensures the sibling will no longer be running 6250 * their task. This ensures there is no inter-sibling overlap between 6251 * non-matching user state. 6252 */ 6253 for_each_cpu(i, smt_mask) { 6254 rq_i = cpu_rq(i); 6255 6256 /* 6257 * An online sibling might have gone offline before a task 6258 * could be picked for it, or it might be offline but later 6259 * happen to come online, but its too late and nothing was 6260 * picked for it. That's Ok - it will pick tasks for itself, 6261 * so ignore it. 6262 */ 6263 if (!rq_i->core_pick) 6264 continue; 6265 6266 /* 6267 * Update for new !FI->FI transitions, or if continuing to be in !FI: 6268 * fi_before fi update? 6269 * 0 0 1 6270 * 0 1 1 6271 * 1 0 1 6272 * 1 1 0 6273 */ 6274 if (!(fi_before && rq->core->core_forceidle_count)) 6275 task_vruntime_update(rq_i, rq_i->core_pick, !!rq->core->core_forceidle_count); 6276 6277 rq_i->core_pick->core_occupation = occ; 6278 6279 if (i == cpu) { 6280 rq_i->core_pick = NULL; 6281 continue; 6282 } 6283 6284 /* Did we break L1TF mitigation requirements? */ 6285 WARN_ON_ONCE(!cookie_match(next, rq_i->core_pick)); 6286 6287 if (rq_i->curr == rq_i->core_pick) { 6288 rq_i->core_pick = NULL; 6289 continue; 6290 } 6291 6292 resched_curr(rq_i); 6293 } 6294 6295 out_set_next: 6296 set_next_task(rq, next); 6297 out: 6298 if (rq->core->core_forceidle_count && next == rq->idle) 6299 queue_core_balance(rq); 6300 6301 return next; 6302 } 6303 6304 static bool try_steal_cookie(int this, int that) 6305 { 6306 struct rq *dst = cpu_rq(this), *src = cpu_rq(that); 6307 struct task_struct *p; 6308 unsigned long cookie; 6309 bool success = false; 6310 6311 local_irq_disable(); 6312 double_rq_lock(dst, src); 6313 6314 cookie = dst->core->core_cookie; 6315 if (!cookie) 6316 goto unlock; 6317 6318 if (dst->curr != dst->idle) 6319 goto unlock; 6320 6321 p = sched_core_find(src, cookie); 6322 if (!p) 6323 goto unlock; 6324 6325 do { 6326 if (p == src->core_pick || p == src->curr) 6327 goto next; 6328 6329 if (!is_cpu_allowed(p, this)) 6330 goto next; 6331 6332 if (p->core_occupation > dst->idle->core_occupation) 6333 goto next; 6334 /* 6335 * sched_core_find() and sched_core_next() will ensure that task @p 6336 * is not throttled now, we also need to check whether the runqueue 6337 * of the destination CPU is being throttled. 6338 */ 6339 if (sched_task_is_throttled(p, this)) 6340 goto next; 6341 6342 deactivate_task(src, p, 0); 6343 set_task_cpu(p, this); 6344 activate_task(dst, p, 0); 6345 6346 resched_curr(dst); 6347 6348 success = true; 6349 break; 6350 6351 next: 6352 p = sched_core_next(p, cookie); 6353 } while (p); 6354 6355 unlock: 6356 double_rq_unlock(dst, src); 6357 local_irq_enable(); 6358 6359 return success; 6360 } 6361 6362 static bool steal_cookie_task(int cpu, struct sched_domain *sd) 6363 { 6364 int i; 6365 6366 for_each_cpu_wrap(i, sched_domain_span(sd), cpu + 1) { 6367 if (i == cpu) 6368 continue; 6369 6370 if (need_resched()) 6371 break; 6372 6373 if (try_steal_cookie(cpu, i)) 6374 return true; 6375 } 6376 6377 return false; 6378 } 6379 6380 static void sched_core_balance(struct rq *rq) 6381 { 6382 struct sched_domain *sd; 6383 int cpu = cpu_of(rq); 6384 6385 preempt_disable(); 6386 rcu_read_lock(); 6387 raw_spin_rq_unlock_irq(rq); 6388 for_each_domain(cpu, sd) { 6389 if (need_resched()) 6390 break; 6391 6392 if (steal_cookie_task(cpu, sd)) 6393 break; 6394 } 6395 raw_spin_rq_lock_irq(rq); 6396 rcu_read_unlock(); 6397 preempt_enable(); 6398 } 6399 6400 static DEFINE_PER_CPU(struct balance_callback, core_balance_head); 6401 6402 static void queue_core_balance(struct rq *rq) 6403 { 6404 if (!sched_core_enabled(rq)) 6405 return; 6406 6407 if (!rq->core->core_cookie) 6408 return; 6409 6410 if (!rq->nr_running) /* not forced idle */ 6411 return; 6412 6413 queue_balance_callback(rq, &per_cpu(core_balance_head, rq->cpu), sched_core_balance); 6414 } 6415 6416 static void sched_core_cpu_starting(unsigned int cpu) 6417 { 6418 const struct cpumask *smt_mask = cpu_smt_mask(cpu); 6419 struct rq *rq = cpu_rq(cpu), *core_rq = NULL; 6420 unsigned long flags; 6421 int t; 6422 6423 sched_core_lock(cpu, &flags); 6424 6425 WARN_ON_ONCE(rq->core != rq); 6426 6427 /* if we're the first, we'll be our own leader */ 6428 if (cpumask_weight(smt_mask) == 1) 6429 goto unlock; 6430 6431 /* find the leader */ 6432 for_each_cpu(t, smt_mask) { 6433 if (t == cpu) 6434 continue; 6435 rq = cpu_rq(t); 6436 if (rq->core == rq) { 6437 core_rq = rq; 6438 break; 6439 } 6440 } 6441 6442 if (WARN_ON_ONCE(!core_rq)) /* whoopsie */ 6443 goto unlock; 6444 6445 /* install and validate core_rq */ 6446 for_each_cpu(t, smt_mask) { 6447 rq = cpu_rq(t); 6448 6449 if (t == cpu) 6450 rq->core = core_rq; 6451 6452 WARN_ON_ONCE(rq->core != core_rq); 6453 } 6454 6455 unlock: 6456 sched_core_unlock(cpu, &flags); 6457 } 6458 6459 static void sched_core_cpu_deactivate(unsigned int cpu) 6460 { 6461 const struct cpumask *smt_mask = cpu_smt_mask(cpu); 6462 struct rq *rq = cpu_rq(cpu), *core_rq = NULL; 6463 unsigned long flags; 6464 int t; 6465 6466 sched_core_lock(cpu, &flags); 6467 6468 /* if we're the last man standing, nothing to do */ 6469 if (cpumask_weight(smt_mask) == 1) { 6470 WARN_ON_ONCE(rq->core != rq); 6471 goto unlock; 6472 } 6473 6474 /* if we're not the leader, nothing to do */ 6475 if (rq->core != rq) 6476 goto unlock; 6477 6478 /* find a new leader */ 6479 for_each_cpu(t, smt_mask) { 6480 if (t == cpu) 6481 continue; 6482 core_rq = cpu_rq(t); 6483 break; 6484 } 6485 6486 if (WARN_ON_ONCE(!core_rq)) /* impossible */ 6487 goto unlock; 6488 6489 /* copy the shared state to the new leader */ 6490 core_rq->core_task_seq = rq->core_task_seq; 6491 core_rq->core_pick_seq = rq->core_pick_seq; 6492 core_rq->core_cookie = rq->core_cookie; 6493 core_rq->core_forceidle_count = rq->core_forceidle_count; 6494 core_rq->core_forceidle_seq = rq->core_forceidle_seq; 6495 core_rq->core_forceidle_occupation = rq->core_forceidle_occupation; 6496 6497 /* 6498 * Accounting edge for forced idle is handled in pick_next_task(). 6499 * Don't need another one here, since the hotplug thread shouldn't 6500 * have a cookie. 6501 */ 6502 core_rq->core_forceidle_start = 0; 6503 6504 /* install new leader */ 6505 for_each_cpu(t, smt_mask) { 6506 rq = cpu_rq(t); 6507 rq->core = core_rq; 6508 } 6509 6510 unlock: 6511 sched_core_unlock(cpu, &flags); 6512 } 6513 6514 static inline void sched_core_cpu_dying(unsigned int cpu) 6515 { 6516 struct rq *rq = cpu_rq(cpu); 6517 6518 if (rq->core != rq) 6519 rq->core = rq; 6520 } 6521 6522 #else /* !CONFIG_SCHED_CORE */ 6523 6524 static inline void sched_core_cpu_starting(unsigned int cpu) {} 6525 static inline void sched_core_cpu_deactivate(unsigned int cpu) {} 6526 static inline void sched_core_cpu_dying(unsigned int cpu) {} 6527 6528 static struct task_struct * 6529 pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 6530 { 6531 return __pick_next_task(rq, prev, rf); 6532 } 6533 6534 #endif /* CONFIG_SCHED_CORE */ 6535 6536 /* 6537 * Constants for the sched_mode argument of __schedule(). 6538 * 6539 * The mode argument allows RT enabled kernels to differentiate a 6540 * preemption from blocking on an 'sleeping' spin/rwlock. Note that 6541 * SM_MASK_PREEMPT for !RT has all bits set, which allows the compiler to 6542 * optimize the AND operation out and just check for zero. 6543 */ 6544 #define SM_NONE 0x0 6545 #define SM_PREEMPT 0x1 6546 #define SM_RTLOCK_WAIT 0x2 6547 6548 #ifndef CONFIG_PREEMPT_RT 6549 # define SM_MASK_PREEMPT (~0U) 6550 #else 6551 # define SM_MASK_PREEMPT SM_PREEMPT 6552 #endif 6553 6554 /* 6555 * __schedule() is the main scheduler function. 6556 * 6557 * The main means of driving the scheduler and thus entering this function are: 6558 * 6559 * 1. Explicit blocking: mutex, semaphore, waitqueue, etc. 6560 * 6561 * 2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return 6562 * paths. For example, see arch/x86/entry_64.S. 6563 * 6564 * To drive preemption between tasks, the scheduler sets the flag in timer 6565 * interrupt handler scheduler_tick(). 6566 * 6567 * 3. Wakeups don't really cause entry into schedule(). They add a 6568 * task to the run-queue and that's it. 6569 * 6570 * Now, if the new task added to the run-queue preempts the current 6571 * task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets 6572 * called on the nearest possible occasion: 6573 * 6574 * - If the kernel is preemptible (CONFIG_PREEMPTION=y): 6575 * 6576 * - in syscall or exception context, at the next outmost 6577 * preempt_enable(). (this might be as soon as the wake_up()'s 6578 * spin_unlock()!) 6579 * 6580 * - in IRQ context, return from interrupt-handler to 6581 * preemptible context 6582 * 6583 * - If the kernel is not preemptible (CONFIG_PREEMPTION is not set) 6584 * then at the next: 6585 * 6586 * - cond_resched() call 6587 * - explicit schedule() call 6588 * - return from syscall or exception to user-space 6589 * - return from interrupt-handler to user-space 6590 * 6591 * WARNING: must be called with preemption disabled! 6592 */ 6593 static void __sched notrace __schedule(unsigned int sched_mode) 6594 { 6595 struct task_struct *prev, *next; 6596 unsigned long *switch_count; 6597 unsigned long prev_state; 6598 struct rq_flags rf; 6599 struct rq *rq; 6600 int cpu; 6601 6602 cpu = smp_processor_id(); 6603 rq = cpu_rq(cpu); 6604 prev = rq->curr; 6605 6606 schedule_debug(prev, !!sched_mode); 6607 6608 if (sched_feat(HRTICK) || sched_feat(HRTICK_DL)) 6609 hrtick_clear(rq); 6610 6611 local_irq_disable(); 6612 rcu_note_context_switch(!!sched_mode); 6613 6614 /* 6615 * Make sure that signal_pending_state()->signal_pending() below 6616 * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE) 6617 * done by the caller to avoid the race with signal_wake_up(): 6618 * 6619 * __set_current_state(@state) signal_wake_up() 6620 * schedule() set_tsk_thread_flag(p, TIF_SIGPENDING) 6621 * wake_up_state(p, state) 6622 * LOCK rq->lock LOCK p->pi_state 6623 * smp_mb__after_spinlock() smp_mb__after_spinlock() 6624 * if (signal_pending_state()) if (p->state & @state) 6625 * 6626 * Also, the membarrier system call requires a full memory barrier 6627 * after coming from user-space, before storing to rq->curr. 6628 */ 6629 rq_lock(rq, &rf); 6630 smp_mb__after_spinlock(); 6631 6632 /* Promote REQ to ACT */ 6633 rq->clock_update_flags <<= 1; 6634 update_rq_clock(rq); 6635 6636 switch_count = &prev->nivcsw; 6637 6638 /* 6639 * We must load prev->state once (task_struct::state is volatile), such 6640 * that we form a control dependency vs deactivate_task() below. 6641 */ 6642 prev_state = READ_ONCE(prev->__state); 6643 if (!(sched_mode & SM_MASK_PREEMPT) && prev_state) { 6644 if (signal_pending_state(prev_state, prev)) { 6645 WRITE_ONCE(prev->__state, TASK_RUNNING); 6646 } else { 6647 prev->sched_contributes_to_load = 6648 (prev_state & TASK_UNINTERRUPTIBLE) && 6649 !(prev_state & TASK_NOLOAD) && 6650 !(prev_state & TASK_FROZEN); 6651 6652 if (prev->sched_contributes_to_load) 6653 rq->nr_uninterruptible++; 6654 6655 /* 6656 * __schedule() ttwu() 6657 * prev_state = prev->state; if (p->on_rq && ...) 6658 * if (prev_state) goto out; 6659 * p->on_rq = 0; smp_acquire__after_ctrl_dep(); 6660 * p->state = TASK_WAKING 6661 * 6662 * Where __schedule() and ttwu() have matching control dependencies. 6663 * 6664 * After this, schedule() must not care about p->state any more. 6665 */ 6666 deactivate_task(rq, prev, DEQUEUE_SLEEP | DEQUEUE_NOCLOCK); 6667 6668 if (prev->in_iowait) { 6669 atomic_inc(&rq->nr_iowait); 6670 delayacct_blkio_start(); 6671 } 6672 } 6673 switch_count = &prev->nvcsw; 6674 } 6675 6676 next = pick_next_task(rq, prev, &rf); 6677 clear_tsk_need_resched(prev); 6678 clear_preempt_need_resched(); 6679 #ifdef CONFIG_SCHED_DEBUG 6680 rq->last_seen_need_resched_ns = 0; 6681 #endif 6682 6683 if (likely(prev != next)) { 6684 rq->nr_switches++; 6685 /* 6686 * RCU users of rcu_dereference(rq->curr) may not see 6687 * changes to task_struct made by pick_next_task(). 6688 */ 6689 RCU_INIT_POINTER(rq->curr, next); 6690 /* 6691 * The membarrier system call requires each architecture 6692 * to have a full memory barrier after updating 6693 * rq->curr, before returning to user-space. 6694 * 6695 * Here are the schemes providing that barrier on the 6696 * various architectures: 6697 * - mm ? switch_mm() : mmdrop() for x86, s390, sparc, PowerPC. 6698 * switch_mm() rely on membarrier_arch_switch_mm() on PowerPC. 6699 * - finish_lock_switch() for weakly-ordered 6700 * architectures where spin_unlock is a full barrier, 6701 * - switch_to() for arm64 (weakly-ordered, spin_unlock 6702 * is a RELEASE barrier), 6703 */ 6704 ++*switch_count; 6705 6706 migrate_disable_switch(rq, prev); 6707 psi_sched_switch(prev, next, !task_on_rq_queued(prev)); 6708 6709 trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); 6710 6711 /* Also unlocks the rq: */ 6712 rq = context_switch(rq, prev, next, &rf); 6713 } else { 6714 rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP); 6715 6716 rq_unpin_lock(rq, &rf); 6717 __balance_callbacks(rq); 6718 raw_spin_rq_unlock_irq(rq); 6719 } 6720 } 6721 6722 void __noreturn do_task_dead(void) 6723 { 6724 /* Causes final put_task_struct in finish_task_switch(): */ 6725 set_special_state(TASK_DEAD); 6726 6727 /* Tell freezer to ignore us: */ 6728 current->flags |= PF_NOFREEZE; 6729 6730 __schedule(SM_NONE); 6731 BUG(); 6732 6733 /* Avoid "noreturn function does return" - but don't continue if BUG() is a NOP: */ 6734 for (;;) 6735 cpu_relax(); 6736 } 6737 6738 static inline void sched_submit_work(struct task_struct *tsk) 6739 { 6740 unsigned int task_flags; 6741 6742 if (task_is_running(tsk)) 6743 return; 6744 6745 task_flags = tsk->flags; 6746 /* 6747 * If a worker goes to sleep, notify and ask workqueue whether it 6748 * wants to wake up a task to maintain concurrency. 6749 */ 6750 if (task_flags & (PF_WQ_WORKER | PF_IO_WORKER)) { 6751 if (task_flags & PF_WQ_WORKER) 6752 wq_worker_sleeping(tsk); 6753 else 6754 io_wq_worker_sleeping(tsk); 6755 } 6756 6757 /* 6758 * spinlock and rwlock must not flush block requests. This will 6759 * deadlock if the callback attempts to acquire a lock which is 6760 * already acquired. 6761 */ 6762 SCHED_WARN_ON(current->__state & TASK_RTLOCK_WAIT); 6763 6764 /* 6765 * If we are going to sleep and we have plugged IO queued, 6766 * make sure to submit it to avoid deadlocks. 6767 */ 6768 blk_flush_plug(tsk->plug, true); 6769 } 6770 6771 static void sched_update_worker(struct task_struct *tsk) 6772 { 6773 if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER)) { 6774 if (tsk->flags & PF_WQ_WORKER) 6775 wq_worker_running(tsk); 6776 else 6777 io_wq_worker_running(tsk); 6778 } 6779 } 6780 6781 asmlinkage __visible void __sched schedule(void) 6782 { 6783 struct task_struct *tsk = current; 6784 6785 sched_submit_work(tsk); 6786 do { 6787 preempt_disable(); 6788 __schedule(SM_NONE); 6789 sched_preempt_enable_no_resched(); 6790 } while (need_resched()); 6791 sched_update_worker(tsk); 6792 } 6793 EXPORT_SYMBOL(schedule); 6794 6795 /* 6796 * synchronize_rcu_tasks() makes sure that no task is stuck in preempted 6797 * state (have scheduled out non-voluntarily) by making sure that all 6798 * tasks have either left the run queue or have gone into user space. 6799 * As idle tasks do not do either, they must not ever be preempted 6800 * (schedule out non-voluntarily). 6801 * 6802 * schedule_idle() is similar to schedule_preempt_disable() except that it 6803 * never enables preemption because it does not call sched_submit_work(). 6804 */ 6805 void __sched schedule_idle(void) 6806 { 6807 /* 6808 * As this skips calling sched_submit_work(), which the idle task does 6809 * regardless because that function is a nop when the task is in a 6810 * TASK_RUNNING state, make sure this isn't used someplace that the 6811 * current task can be in any other state. Note, idle is always in the 6812 * TASK_RUNNING state. 6813 */ 6814 WARN_ON_ONCE(current->__state); 6815 do { 6816 __schedule(SM_NONE); 6817 } while (need_resched()); 6818 } 6819 6820 #if defined(CONFIG_CONTEXT_TRACKING_USER) && !defined(CONFIG_HAVE_CONTEXT_TRACKING_USER_OFFSTACK) 6821 asmlinkage __visible void __sched schedule_user(void) 6822 { 6823 /* 6824 * If we come here after a random call to set_need_resched(), 6825 * or we have been woken up remotely but the IPI has not yet arrived, 6826 * we haven't yet exited the RCU idle mode. Do it here manually until 6827 * we find a better solution. 6828 * 6829 * NB: There are buggy callers of this function. Ideally we 6830 * should warn if prev_state != CONTEXT_USER, but that will trigger 6831 * too frequently to make sense yet. 6832 */ 6833 enum ctx_state prev_state = exception_enter(); 6834 schedule(); 6835 exception_exit(prev_state); 6836 } 6837 #endif 6838 6839 /** 6840 * schedule_preempt_disabled - called with preemption disabled 6841 * 6842 * Returns with preemption disabled. Note: preempt_count must be 1 6843 */ 6844 void __sched schedule_preempt_disabled(void) 6845 { 6846 sched_preempt_enable_no_resched(); 6847 schedule(); 6848 preempt_disable(); 6849 } 6850 6851 #ifdef CONFIG_PREEMPT_RT 6852 void __sched notrace schedule_rtlock(void) 6853 { 6854 do { 6855 preempt_disable(); 6856 __schedule(SM_RTLOCK_WAIT); 6857 sched_preempt_enable_no_resched(); 6858 } while (need_resched()); 6859 } 6860 NOKPROBE_SYMBOL(schedule_rtlock); 6861 #endif 6862 6863 static void __sched notrace preempt_schedule_common(void) 6864 { 6865 do { 6866 /* 6867 * Because the function tracer can trace preempt_count_sub() 6868 * and it also uses preempt_enable/disable_notrace(), if 6869 * NEED_RESCHED is set, the preempt_enable_notrace() called 6870 * by the function tracer will call this function again and 6871 * cause infinite recursion. 6872 * 6873 * Preemption must be disabled here before the function 6874 * tracer can trace. Break up preempt_disable() into two 6875 * calls. One to disable preemption without fear of being 6876 * traced. The other to still record the preemption latency, 6877 * which can also be traced by the function tracer. 6878 */ 6879 preempt_disable_notrace(); 6880 preempt_latency_start(1); 6881 __schedule(SM_PREEMPT); 6882 preempt_latency_stop(1); 6883 preempt_enable_no_resched_notrace(); 6884 6885 /* 6886 * Check again in case we missed a preemption opportunity 6887 * between schedule and now. 6888 */ 6889 } while (need_resched()); 6890 } 6891 6892 #ifdef CONFIG_PREEMPTION 6893 /* 6894 * This is the entry point to schedule() from in-kernel preemption 6895 * off of preempt_enable. 6896 */ 6897 asmlinkage __visible void __sched notrace preempt_schedule(void) 6898 { 6899 /* 6900 * If there is a non-zero preempt_count or interrupts are disabled, 6901 * we do not want to preempt the current task. Just return.. 6902 */ 6903 if (likely(!preemptible())) 6904 return; 6905 preempt_schedule_common(); 6906 } 6907 NOKPROBE_SYMBOL(preempt_schedule); 6908 EXPORT_SYMBOL(preempt_schedule); 6909 6910 #ifdef CONFIG_PREEMPT_DYNAMIC 6911 #if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 6912 #ifndef preempt_schedule_dynamic_enabled 6913 #define preempt_schedule_dynamic_enabled preempt_schedule 6914 #define preempt_schedule_dynamic_disabled NULL 6915 #endif 6916 DEFINE_STATIC_CALL(preempt_schedule, preempt_schedule_dynamic_enabled); 6917 EXPORT_STATIC_CALL_TRAMP(preempt_schedule); 6918 #elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 6919 static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule); 6920 void __sched notrace dynamic_preempt_schedule(void) 6921 { 6922 if (!static_branch_unlikely(&sk_dynamic_preempt_schedule)) 6923 return; 6924 preempt_schedule(); 6925 } 6926 NOKPROBE_SYMBOL(dynamic_preempt_schedule); 6927 EXPORT_SYMBOL(dynamic_preempt_schedule); 6928 #endif 6929 #endif 6930 6931 /** 6932 * preempt_schedule_notrace - preempt_schedule called by tracing 6933 * 6934 * The tracing infrastructure uses preempt_enable_notrace to prevent 6935 * recursion and tracing preempt enabling caused by the tracing 6936 * infrastructure itself. But as tracing can happen in areas coming 6937 * from userspace or just about to enter userspace, a preempt enable 6938 * can occur before user_exit() is called. This will cause the scheduler 6939 * to be called when the system is still in usermode. 6940 * 6941 * To prevent this, the preempt_enable_notrace will use this function 6942 * instead of preempt_schedule() to exit user context if needed before 6943 * calling the scheduler. 6944 */ 6945 asmlinkage __visible void __sched notrace preempt_schedule_notrace(void) 6946 { 6947 enum ctx_state prev_ctx; 6948 6949 if (likely(!preemptible())) 6950 return; 6951 6952 do { 6953 /* 6954 * Because the function tracer can trace preempt_count_sub() 6955 * and it also uses preempt_enable/disable_notrace(), if 6956 * NEED_RESCHED is set, the preempt_enable_notrace() called 6957 * by the function tracer will call this function again and 6958 * cause infinite recursion. 6959 * 6960 * Preemption must be disabled here before the function 6961 * tracer can trace. Break up preempt_disable() into two 6962 * calls. One to disable preemption without fear of being 6963 * traced. The other to still record the preemption latency, 6964 * which can also be traced by the function tracer. 6965 */ 6966 preempt_disable_notrace(); 6967 preempt_latency_start(1); 6968 /* 6969 * Needs preempt disabled in case user_exit() is traced 6970 * and the tracer calls preempt_enable_notrace() causing 6971 * an infinite recursion. 6972 */ 6973 prev_ctx = exception_enter(); 6974 __schedule(SM_PREEMPT); 6975 exception_exit(prev_ctx); 6976 6977 preempt_latency_stop(1); 6978 preempt_enable_no_resched_notrace(); 6979 } while (need_resched()); 6980 } 6981 EXPORT_SYMBOL_GPL(preempt_schedule_notrace); 6982 6983 #ifdef CONFIG_PREEMPT_DYNAMIC 6984 #if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 6985 #ifndef preempt_schedule_notrace_dynamic_enabled 6986 #define preempt_schedule_notrace_dynamic_enabled preempt_schedule_notrace 6987 #define preempt_schedule_notrace_dynamic_disabled NULL 6988 #endif 6989 DEFINE_STATIC_CALL(preempt_schedule_notrace, preempt_schedule_notrace_dynamic_enabled); 6990 EXPORT_STATIC_CALL_TRAMP(preempt_schedule_notrace); 6991 #elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 6992 static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule_notrace); 6993 void __sched notrace dynamic_preempt_schedule_notrace(void) 6994 { 6995 if (!static_branch_unlikely(&sk_dynamic_preempt_schedule_notrace)) 6996 return; 6997 preempt_schedule_notrace(); 6998 } 6999 NOKPROBE_SYMBOL(dynamic_preempt_schedule_notrace); 7000 EXPORT_SYMBOL(dynamic_preempt_schedule_notrace); 7001 #endif 7002 #endif 7003 7004 #endif /* CONFIG_PREEMPTION */ 7005 7006 /* 7007 * This is the entry point to schedule() from kernel preemption 7008 * off of irq context. 7009 * Note, that this is called and return with irqs disabled. This will 7010 * protect us against recursive calling from irq. 7011 */ 7012 asmlinkage __visible void __sched preempt_schedule_irq(void) 7013 { 7014 enum ctx_state prev_state; 7015 7016 /* Catch callers which need to be fixed */ 7017 BUG_ON(preempt_count() || !irqs_disabled()); 7018 7019 prev_state = exception_enter(); 7020 7021 do { 7022 preempt_disable(); 7023 local_irq_enable(); 7024 __schedule(SM_PREEMPT); 7025 local_irq_disable(); 7026 sched_preempt_enable_no_resched(); 7027 } while (need_resched()); 7028 7029 exception_exit(prev_state); 7030 } 7031 7032 int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flags, 7033 void *key) 7034 { 7035 WARN_ON_ONCE(IS_ENABLED(CONFIG_SCHED_DEBUG) && wake_flags & ~WF_SYNC); 7036 return try_to_wake_up(curr->private, mode, wake_flags); 7037 } 7038 EXPORT_SYMBOL(default_wake_function); 7039 7040 static void __setscheduler_prio(struct task_struct *p, int prio) 7041 { 7042 if (dl_prio(prio)) 7043 p->sched_class = &dl_sched_class; 7044 else if (rt_prio(prio)) 7045 p->sched_class = &rt_sched_class; 7046 else 7047 p->sched_class = &fair_sched_class; 7048 7049 p->prio = prio; 7050 } 7051 7052 #ifdef CONFIG_RT_MUTEXES 7053 7054 static inline int __rt_effective_prio(struct task_struct *pi_task, int prio) 7055 { 7056 if (pi_task) 7057 prio = min(prio, pi_task->prio); 7058 7059 return prio; 7060 } 7061 7062 static inline int rt_effective_prio(struct task_struct *p, int prio) 7063 { 7064 struct task_struct *pi_task = rt_mutex_get_top_task(p); 7065 7066 return __rt_effective_prio(pi_task, prio); 7067 } 7068 7069 /* 7070 * rt_mutex_setprio - set the current priority of a task 7071 * @p: task to boost 7072 * @pi_task: donor task 7073 * 7074 * This function changes the 'effective' priority of a task. It does 7075 * not touch ->normal_prio like __setscheduler(). 7076 * 7077 * Used by the rt_mutex code to implement priority inheritance 7078 * logic. Call site only calls if the priority of the task changed. 7079 */ 7080 void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) 7081 { 7082 int prio, oldprio, queued, running, queue_flag = 7083 DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; 7084 const struct sched_class *prev_class; 7085 struct rq_flags rf; 7086 struct rq *rq; 7087 7088 /* XXX used to be waiter->prio, not waiter->task->prio */ 7089 prio = __rt_effective_prio(pi_task, p->normal_prio); 7090 7091 /* 7092 * If nothing changed; bail early. 7093 */ 7094 if (p->pi_top_task == pi_task && prio == p->prio && !dl_prio(prio)) 7095 return; 7096 7097 rq = __task_rq_lock(p, &rf); 7098 update_rq_clock(rq); 7099 /* 7100 * Set under pi_lock && rq->lock, such that the value can be used under 7101 * either lock. 7102 * 7103 * Note that there is loads of tricky to make this pointer cache work 7104 * right. rt_mutex_slowunlock()+rt_mutex_postunlock() work together to 7105 * ensure a task is de-boosted (pi_task is set to NULL) before the 7106 * task is allowed to run again (and can exit). This ensures the pointer 7107 * points to a blocked task -- which guarantees the task is present. 7108 */ 7109 p->pi_top_task = pi_task; 7110 7111 /* 7112 * For FIFO/RR we only need to set prio, if that matches we're done. 7113 */ 7114 if (prio == p->prio && !dl_prio(prio)) 7115 goto out_unlock; 7116 7117 /* 7118 * Idle task boosting is a nono in general. There is one 7119 * exception, when PREEMPT_RT and NOHZ is active: 7120 * 7121 * The idle task calls get_next_timer_interrupt() and holds 7122 * the timer wheel base->lock on the CPU and another CPU wants 7123 * to access the timer (probably to cancel it). We can safely 7124 * ignore the boosting request, as the idle CPU runs this code 7125 * with interrupts disabled and will complete the lock 7126 * protected section without being interrupted. So there is no 7127 * real need to boost. 7128 */ 7129 if (unlikely(p == rq->idle)) { 7130 WARN_ON(p != rq->curr); 7131 WARN_ON(p->pi_blocked_on); 7132 goto out_unlock; 7133 } 7134 7135 trace_sched_pi_setprio(p, pi_task); 7136 oldprio = p->prio; 7137 7138 if (oldprio == prio) 7139 queue_flag &= ~DEQUEUE_MOVE; 7140 7141 prev_class = p->sched_class; 7142 queued = task_on_rq_queued(p); 7143 running = task_current(rq, p); 7144 if (queued) 7145 dequeue_task(rq, p, queue_flag); 7146 if (running) 7147 put_prev_task(rq, p); 7148 7149 /* 7150 * Boosting condition are: 7151 * 1. -rt task is running and holds mutex A 7152 * --> -dl task blocks on mutex A 7153 * 7154 * 2. -dl task is running and holds mutex A 7155 * --> -dl task blocks on mutex A and could preempt the 7156 * running task 7157 */ 7158 if (dl_prio(prio)) { 7159 if (!dl_prio(p->normal_prio) || 7160 (pi_task && dl_prio(pi_task->prio) && 7161 dl_entity_preempt(&pi_task->dl, &p->dl))) { 7162 p->dl.pi_se = pi_task->dl.pi_se; 7163 queue_flag |= ENQUEUE_REPLENISH; 7164 } else { 7165 p->dl.pi_se = &p->dl; 7166 } 7167 } else if (rt_prio(prio)) { 7168 if (dl_prio(oldprio)) 7169 p->dl.pi_se = &p->dl; 7170 if (oldprio < prio) 7171 queue_flag |= ENQUEUE_HEAD; 7172 } else { 7173 if (dl_prio(oldprio)) 7174 p->dl.pi_se = &p->dl; 7175 if (rt_prio(oldprio)) 7176 p->rt.timeout = 0; 7177 } 7178 7179 __setscheduler_prio(p, prio); 7180 7181 if (queued) 7182 enqueue_task(rq, p, queue_flag); 7183 if (running) 7184 set_next_task(rq, p); 7185 7186 check_class_changed(rq, p, prev_class, oldprio); 7187 out_unlock: 7188 /* Avoid rq from going away on us: */ 7189 preempt_disable(); 7190 7191 rq_unpin_lock(rq, &rf); 7192 __balance_callbacks(rq); 7193 raw_spin_rq_unlock(rq); 7194 7195 preempt_enable(); 7196 } 7197 #else 7198 static inline int rt_effective_prio(struct task_struct *p, int prio) 7199 { 7200 return prio; 7201 } 7202 #endif 7203 7204 void set_user_nice(struct task_struct *p, long nice) 7205 { 7206 bool queued, running; 7207 int old_prio; 7208 struct rq_flags rf; 7209 struct rq *rq; 7210 7211 if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE) 7212 return; 7213 /* 7214 * We have to be careful, if called from sys_setpriority(), 7215 * the task might be in the middle of scheduling on another CPU. 7216 */ 7217 rq = task_rq_lock(p, &rf); 7218 update_rq_clock(rq); 7219 7220 /* 7221 * The RT priorities are set via sched_setscheduler(), but we still 7222 * allow the 'normal' nice value to be set - but as expected 7223 * it won't have any effect on scheduling until the task is 7224 * SCHED_DEADLINE, SCHED_FIFO or SCHED_RR: 7225 */ 7226 if (task_has_dl_policy(p) || task_has_rt_policy(p)) { 7227 p->static_prio = NICE_TO_PRIO(nice); 7228 goto out_unlock; 7229 } 7230 queued = task_on_rq_queued(p); 7231 running = task_current(rq, p); 7232 if (queued) 7233 dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); 7234 if (running) 7235 put_prev_task(rq, p); 7236 7237 p->static_prio = NICE_TO_PRIO(nice); 7238 set_load_weight(p, true); 7239 old_prio = p->prio; 7240 p->prio = effective_prio(p); 7241 7242 if (queued) 7243 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); 7244 if (running) 7245 set_next_task(rq, p); 7246 7247 /* 7248 * If the task increased its priority or is running and 7249 * lowered its priority, then reschedule its CPU: 7250 */ 7251 p->sched_class->prio_changed(rq, p, old_prio); 7252 7253 out_unlock: 7254 task_rq_unlock(rq, p, &rf); 7255 } 7256 EXPORT_SYMBOL(set_user_nice); 7257 7258 /* 7259 * is_nice_reduction - check if nice value is an actual reduction 7260 * 7261 * Similar to can_nice() but does not perform a capability check. 7262 * 7263 * @p: task 7264 * @nice: nice value 7265 */ 7266 static bool is_nice_reduction(const struct task_struct *p, const int nice) 7267 { 7268 /* Convert nice value [19,-20] to rlimit style value [1,40]: */ 7269 int nice_rlim = nice_to_rlimit(nice); 7270 7271 return (nice_rlim <= task_rlimit(p, RLIMIT_NICE)); 7272 } 7273 7274 /* 7275 * can_nice - check if a task can reduce its nice value 7276 * @p: task 7277 * @nice: nice value 7278 */ 7279 int can_nice(const struct task_struct *p, const int nice) 7280 { 7281 return is_nice_reduction(p, nice) || capable(CAP_SYS_NICE); 7282 } 7283 7284 #ifdef __ARCH_WANT_SYS_NICE 7285 7286 /* 7287 * sys_nice - change the priority of the current process. 7288 * @increment: priority increment 7289 * 7290 * sys_setpriority is a more generic, but much slower function that 7291 * does similar things. 7292 */ 7293 SYSCALL_DEFINE1(nice, int, increment) 7294 { 7295 long nice, retval; 7296 7297 /* 7298 * Setpriority might change our priority at the same moment. 7299 * We don't have to worry. Conceptually one call occurs first 7300 * and we have a single winner. 7301 */ 7302 increment = clamp(increment, -NICE_WIDTH, NICE_WIDTH); 7303 nice = task_nice(current) + increment; 7304 7305 nice = clamp_val(nice, MIN_NICE, MAX_NICE); 7306 if (increment < 0 && !can_nice(current, nice)) 7307 return -EPERM; 7308 7309 retval = security_task_setnice(current, nice); 7310 if (retval) 7311 return retval; 7312 7313 set_user_nice(current, nice); 7314 return 0; 7315 } 7316 7317 #endif 7318 7319 /** 7320 * task_prio - return the priority value of a given task. 7321 * @p: the task in question. 7322 * 7323 * Return: The priority value as seen by users in /proc. 7324 * 7325 * sched policy return value kernel prio user prio/nice 7326 * 7327 * normal, batch, idle [0 ... 39] [100 ... 139] 0/[-20 ... 19] 7328 * fifo, rr [-2 ... -100] [98 ... 0] [1 ... 99] 7329 * deadline -101 -1 0 7330 */ 7331 int task_prio(const struct task_struct *p) 7332 { 7333 return p->prio - MAX_RT_PRIO; 7334 } 7335 7336 /** 7337 * idle_cpu - is a given CPU idle currently? 7338 * @cpu: the processor in question. 7339 * 7340 * Return: 1 if the CPU is currently idle. 0 otherwise. 7341 */ 7342 int idle_cpu(int cpu) 7343 { 7344 struct rq *rq = cpu_rq(cpu); 7345 7346 if (rq->curr != rq->idle) 7347 return 0; 7348 7349 if (rq->nr_running) 7350 return 0; 7351 7352 #ifdef CONFIG_SMP 7353 if (rq->ttwu_pending) 7354 return 0; 7355 #endif 7356 7357 return 1; 7358 } 7359 7360 /** 7361 * available_idle_cpu - is a given CPU idle for enqueuing work. 7362 * @cpu: the CPU in question. 7363 * 7364 * Return: 1 if the CPU is currently idle. 0 otherwise. 7365 */ 7366 int available_idle_cpu(int cpu) 7367 { 7368 if (!idle_cpu(cpu)) 7369 return 0; 7370 7371 if (vcpu_is_preempted(cpu)) 7372 return 0; 7373 7374 return 1; 7375 } 7376 7377 /** 7378 * idle_task - return the idle task for a given CPU. 7379 * @cpu: the processor in question. 7380 * 7381 * Return: The idle task for the CPU @cpu. 7382 */ 7383 struct task_struct *idle_task(int cpu) 7384 { 7385 return cpu_rq(cpu)->idle; 7386 } 7387 7388 #ifdef CONFIG_SCHED_CORE 7389 int sched_core_idle_cpu(int cpu) 7390 { 7391 struct rq *rq = cpu_rq(cpu); 7392 7393 if (sched_core_enabled(rq) && rq->curr == rq->idle) 7394 return 1; 7395 7396 return idle_cpu(cpu); 7397 } 7398 7399 #endif 7400 7401 #ifdef CONFIG_SMP 7402 /* 7403 * This function computes an effective utilization for the given CPU, to be 7404 * used for frequency selection given the linear relation: f = u * f_max. 7405 * 7406 * The scheduler tracks the following metrics: 7407 * 7408 * cpu_util_{cfs,rt,dl,irq}() 7409 * cpu_bw_dl() 7410 * 7411 * Where the cfs,rt and dl util numbers are tracked with the same metric and 7412 * synchronized windows and are thus directly comparable. 7413 * 7414 * The cfs,rt,dl utilization are the running times measured with rq->clock_task 7415 * which excludes things like IRQ and steal-time. These latter are then accrued 7416 * in the irq utilization. 7417 * 7418 * The DL bandwidth number otoh is not a measured metric but a value computed 7419 * based on the task model parameters and gives the minimal utilization 7420 * required to meet deadlines. 7421 */ 7422 unsigned long effective_cpu_util(int cpu, unsigned long util_cfs, 7423 enum cpu_util_type type, 7424 struct task_struct *p) 7425 { 7426 unsigned long dl_util, util, irq, max; 7427 struct rq *rq = cpu_rq(cpu); 7428 7429 max = arch_scale_cpu_capacity(cpu); 7430 7431 if (!uclamp_is_used() && 7432 type == FREQUENCY_UTIL && rt_rq_is_runnable(&rq->rt)) { 7433 return max; 7434 } 7435 7436 /* 7437 * Early check to see if IRQ/steal time saturates the CPU, can be 7438 * because of inaccuracies in how we track these -- see 7439 * update_irq_load_avg(). 7440 */ 7441 irq = cpu_util_irq(rq); 7442 if (unlikely(irq >= max)) 7443 return max; 7444 7445 /* 7446 * Because the time spend on RT/DL tasks is visible as 'lost' time to 7447 * CFS tasks and we use the same metric to track the effective 7448 * utilization (PELT windows are synchronized) we can directly add them 7449 * to obtain the CPU's actual utilization. 7450 * 7451 * CFS and RT utilization can be boosted or capped, depending on 7452 * utilization clamp constraints requested by currently RUNNABLE 7453 * tasks. 7454 * When there are no CFS RUNNABLE tasks, clamps are released and 7455 * frequency will be gracefully reduced with the utilization decay. 7456 */ 7457 util = util_cfs + cpu_util_rt(rq); 7458 if (type == FREQUENCY_UTIL) 7459 util = uclamp_rq_util_with(rq, util, p); 7460 7461 dl_util = cpu_util_dl(rq); 7462 7463 /* 7464 * For frequency selection we do not make cpu_util_dl() a permanent part 7465 * of this sum because we want to use cpu_bw_dl() later on, but we need 7466 * to check if the CFS+RT+DL sum is saturated (ie. no idle time) such 7467 * that we select f_max when there is no idle time. 7468 * 7469 * NOTE: numerical errors or stop class might cause us to not quite hit 7470 * saturation when we should -- something for later. 7471 */ 7472 if (util + dl_util >= max) 7473 return max; 7474 7475 /* 7476 * OTOH, for energy computation we need the estimated running time, so 7477 * include util_dl and ignore dl_bw. 7478 */ 7479 if (type == ENERGY_UTIL) 7480 util += dl_util; 7481 7482 /* 7483 * There is still idle time; further improve the number by using the 7484 * irq metric. Because IRQ/steal time is hidden from the task clock we 7485 * need to scale the task numbers: 7486 * 7487 * max - irq 7488 * U' = irq + --------- * U 7489 * max 7490 */ 7491 util = scale_irq_capacity(util, irq, max); 7492 util += irq; 7493 7494 /* 7495 * Bandwidth required by DEADLINE must always be granted while, for 7496 * FAIR and RT, we use blocked utilization of IDLE CPUs as a mechanism 7497 * to gracefully reduce the frequency when no tasks show up for longer 7498 * periods of time. 7499 * 7500 * Ideally we would like to set bw_dl as min/guaranteed freq and util + 7501 * bw_dl as requested freq. However, cpufreq is not yet ready for such 7502 * an interface. So, we only do the latter for now. 7503 */ 7504 if (type == FREQUENCY_UTIL) 7505 util += cpu_bw_dl(rq); 7506 7507 return min(max, util); 7508 } 7509 7510 unsigned long sched_cpu_util(int cpu) 7511 { 7512 return effective_cpu_util(cpu, cpu_util_cfs(cpu), ENERGY_UTIL, NULL); 7513 } 7514 #endif /* CONFIG_SMP */ 7515 7516 /** 7517 * find_process_by_pid - find a process with a matching PID value. 7518 * @pid: the pid in question. 7519 * 7520 * The task of @pid, if found. %NULL otherwise. 7521 */ 7522 static struct task_struct *find_process_by_pid(pid_t pid) 7523 { 7524 return pid ? find_task_by_vpid(pid) : current; 7525 } 7526 7527 /* 7528 * sched_setparam() passes in -1 for its policy, to let the functions 7529 * it calls know not to change it. 7530 */ 7531 #define SETPARAM_POLICY -1 7532 7533 static void __setscheduler_params(struct task_struct *p, 7534 const struct sched_attr *attr) 7535 { 7536 int policy = attr->sched_policy; 7537 7538 if (policy == SETPARAM_POLICY) 7539 policy = p->policy; 7540 7541 p->policy = policy; 7542 7543 if (dl_policy(policy)) 7544 __setparam_dl(p, attr); 7545 else if (fair_policy(policy)) 7546 p->static_prio = NICE_TO_PRIO(attr->sched_nice); 7547 7548 /* 7549 * __sched_setscheduler() ensures attr->sched_priority == 0 when 7550 * !rt_policy. Always setting this ensures that things like 7551 * getparam()/getattr() don't report silly values for !rt tasks. 7552 */ 7553 p->rt_priority = attr->sched_priority; 7554 p->normal_prio = normal_prio(p); 7555 set_load_weight(p, true); 7556 } 7557 7558 /* 7559 * Check the target process has a UID that matches the current process's: 7560 */ 7561 static bool check_same_owner(struct task_struct *p) 7562 { 7563 const struct cred *cred = current_cred(), *pcred; 7564 bool match; 7565 7566 rcu_read_lock(); 7567 pcred = __task_cred(p); 7568 match = (uid_eq(cred->euid, pcred->euid) || 7569 uid_eq(cred->euid, pcred->uid)); 7570 rcu_read_unlock(); 7571 return match; 7572 } 7573 7574 /* 7575 * Allow unprivileged RT tasks to decrease priority. 7576 * Only issue a capable test if needed and only once to avoid an audit 7577 * event on permitted non-privileged operations: 7578 */ 7579 static int user_check_sched_setscheduler(struct task_struct *p, 7580 const struct sched_attr *attr, 7581 int policy, int reset_on_fork) 7582 { 7583 if (fair_policy(policy)) { 7584 if (attr->sched_nice < task_nice(p) && 7585 !is_nice_reduction(p, attr->sched_nice)) 7586 goto req_priv; 7587 } 7588 7589 if (rt_policy(policy)) { 7590 unsigned long rlim_rtprio = task_rlimit(p, RLIMIT_RTPRIO); 7591 7592 /* Can't set/change the rt policy: */ 7593 if (policy != p->policy && !rlim_rtprio) 7594 goto req_priv; 7595 7596 /* Can't increase priority: */ 7597 if (attr->sched_priority > p->rt_priority && 7598 attr->sched_priority > rlim_rtprio) 7599 goto req_priv; 7600 } 7601 7602 /* 7603 * Can't set/change SCHED_DEADLINE policy at all for now 7604 * (safest behavior); in the future we would like to allow 7605 * unprivileged DL tasks to increase their relative deadline 7606 * or reduce their runtime (both ways reducing utilization) 7607 */ 7608 if (dl_policy(policy)) 7609 goto req_priv; 7610 7611 /* 7612 * Treat SCHED_IDLE as nice 20. Only allow a switch to 7613 * SCHED_NORMAL if the RLIMIT_NICE would normally permit it. 7614 */ 7615 if (task_has_idle_policy(p) && !idle_policy(policy)) { 7616 if (!is_nice_reduction(p, task_nice(p))) 7617 goto req_priv; 7618 } 7619 7620 /* Can't change other user's priorities: */ 7621 if (!check_same_owner(p)) 7622 goto req_priv; 7623 7624 /* Normal users shall not reset the sched_reset_on_fork flag: */ 7625 if (p->sched_reset_on_fork && !reset_on_fork) 7626 goto req_priv; 7627 7628 return 0; 7629 7630 req_priv: 7631 if (!capable(CAP_SYS_NICE)) 7632 return -EPERM; 7633 7634 return 0; 7635 } 7636 7637 static int __sched_setscheduler(struct task_struct *p, 7638 const struct sched_attr *attr, 7639 bool user, bool pi) 7640 { 7641 int oldpolicy = -1, policy = attr->sched_policy; 7642 int retval, oldprio, newprio, queued, running; 7643 const struct sched_class *prev_class; 7644 struct balance_callback *head; 7645 struct rq_flags rf; 7646 int reset_on_fork; 7647 int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; 7648 struct rq *rq; 7649 bool cpuset_locked = false; 7650 7651 /* The pi code expects interrupts enabled */ 7652 BUG_ON(pi && in_interrupt()); 7653 recheck: 7654 /* Double check policy once rq lock held: */ 7655 if (policy < 0) { 7656 reset_on_fork = p->sched_reset_on_fork; 7657 policy = oldpolicy = p->policy; 7658 } else { 7659 reset_on_fork = !!(attr->sched_flags & SCHED_FLAG_RESET_ON_FORK); 7660 7661 if (!valid_policy(policy)) 7662 return -EINVAL; 7663 } 7664 7665 if (attr->sched_flags & ~(SCHED_FLAG_ALL | SCHED_FLAG_SUGOV)) 7666 return -EINVAL; 7667 7668 /* 7669 * Valid priorities for SCHED_FIFO and SCHED_RR are 7670 * 1..MAX_RT_PRIO-1, valid priority for SCHED_NORMAL, 7671 * SCHED_BATCH and SCHED_IDLE is 0. 7672 */ 7673 if (attr->sched_priority > MAX_RT_PRIO-1) 7674 return -EINVAL; 7675 if ((dl_policy(policy) && !__checkparam_dl(attr)) || 7676 (rt_policy(policy) != (attr->sched_priority != 0))) 7677 return -EINVAL; 7678 7679 if (user) { 7680 retval = user_check_sched_setscheduler(p, attr, policy, reset_on_fork); 7681 if (retval) 7682 return retval; 7683 7684 if (attr->sched_flags & SCHED_FLAG_SUGOV) 7685 return -EINVAL; 7686 7687 retval = security_task_setscheduler(p); 7688 if (retval) 7689 return retval; 7690 } 7691 7692 /* Update task specific "requested" clamps */ 7693 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP) { 7694 retval = uclamp_validate(p, attr); 7695 if (retval) 7696 return retval; 7697 } 7698 7699 /* 7700 * SCHED_DEADLINE bandwidth accounting relies on stable cpusets 7701 * information. 7702 */ 7703 if (dl_policy(policy) || dl_policy(p->policy)) { 7704 cpuset_locked = true; 7705 cpuset_lock(); 7706 } 7707 7708 /* 7709 * Make sure no PI-waiters arrive (or leave) while we are 7710 * changing the priority of the task: 7711 * 7712 * To be able to change p->policy safely, the appropriate 7713 * runqueue lock must be held. 7714 */ 7715 rq = task_rq_lock(p, &rf); 7716 update_rq_clock(rq); 7717 7718 /* 7719 * Changing the policy of the stop threads its a very bad idea: 7720 */ 7721 if (p == rq->stop) { 7722 retval = -EINVAL; 7723 goto unlock; 7724 } 7725 7726 /* 7727 * If not changing anything there's no need to proceed further, 7728 * but store a possible modification of reset_on_fork. 7729 */ 7730 if (unlikely(policy == p->policy)) { 7731 if (fair_policy(policy) && attr->sched_nice != task_nice(p)) 7732 goto change; 7733 if (rt_policy(policy) && attr->sched_priority != p->rt_priority) 7734 goto change; 7735 if (dl_policy(policy) && dl_param_changed(p, attr)) 7736 goto change; 7737 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP) 7738 goto change; 7739 7740 p->sched_reset_on_fork = reset_on_fork; 7741 retval = 0; 7742 goto unlock; 7743 } 7744 change: 7745 7746 if (user) { 7747 #ifdef CONFIG_RT_GROUP_SCHED 7748 /* 7749 * Do not allow realtime tasks into groups that have no runtime 7750 * assigned. 7751 */ 7752 if (rt_bandwidth_enabled() && rt_policy(policy) && 7753 task_group(p)->rt_bandwidth.rt_runtime == 0 && 7754 !task_group_is_autogroup(task_group(p))) { 7755 retval = -EPERM; 7756 goto unlock; 7757 } 7758 #endif 7759 #ifdef CONFIG_SMP 7760 if (dl_bandwidth_enabled() && dl_policy(policy) && 7761 !(attr->sched_flags & SCHED_FLAG_SUGOV)) { 7762 cpumask_t *span = rq->rd->span; 7763 7764 /* 7765 * Don't allow tasks with an affinity mask smaller than 7766 * the entire root_domain to become SCHED_DEADLINE. We 7767 * will also fail if there's no bandwidth available. 7768 */ 7769 if (!cpumask_subset(span, p->cpus_ptr) || 7770 rq->rd->dl_bw.bw == 0) { 7771 retval = -EPERM; 7772 goto unlock; 7773 } 7774 } 7775 #endif 7776 } 7777 7778 /* Re-check policy now with rq lock held: */ 7779 if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { 7780 policy = oldpolicy = -1; 7781 task_rq_unlock(rq, p, &rf); 7782 if (cpuset_locked) 7783 cpuset_unlock(); 7784 goto recheck; 7785 } 7786 7787 /* 7788 * If setscheduling to SCHED_DEADLINE (or changing the parameters 7789 * of a SCHED_DEADLINE task) we need to check if enough bandwidth 7790 * is available. 7791 */ 7792 if ((dl_policy(policy) || dl_task(p)) && sched_dl_overflow(p, policy, attr)) { 7793 retval = -EBUSY; 7794 goto unlock; 7795 } 7796 7797 p->sched_reset_on_fork = reset_on_fork; 7798 oldprio = p->prio; 7799 7800 newprio = __normal_prio(policy, attr->sched_priority, attr->sched_nice); 7801 if (pi) { 7802 /* 7803 * Take priority boosted tasks into account. If the new 7804 * effective priority is unchanged, we just store the new 7805 * normal parameters and do not touch the scheduler class and 7806 * the runqueue. This will be done when the task deboost 7807 * itself. 7808 */ 7809 newprio = rt_effective_prio(p, newprio); 7810 if (newprio == oldprio) 7811 queue_flags &= ~DEQUEUE_MOVE; 7812 } 7813 7814 queued = task_on_rq_queued(p); 7815 running = task_current(rq, p); 7816 if (queued) 7817 dequeue_task(rq, p, queue_flags); 7818 if (running) 7819 put_prev_task(rq, p); 7820 7821 prev_class = p->sched_class; 7822 7823 if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { 7824 __setscheduler_params(p, attr); 7825 __setscheduler_prio(p, newprio); 7826 } 7827 __setscheduler_uclamp(p, attr); 7828 7829 if (queued) { 7830 /* 7831 * We enqueue to tail when the priority of a task is 7832 * increased (user space view). 7833 */ 7834 if (oldprio < p->prio) 7835 queue_flags |= ENQUEUE_HEAD; 7836 7837 enqueue_task(rq, p, queue_flags); 7838 } 7839 if (running) 7840 set_next_task(rq, p); 7841 7842 check_class_changed(rq, p, prev_class, oldprio); 7843 7844 /* Avoid rq from going away on us: */ 7845 preempt_disable(); 7846 head = splice_balance_callbacks(rq); 7847 task_rq_unlock(rq, p, &rf); 7848 7849 if (pi) { 7850 if (cpuset_locked) 7851 cpuset_unlock(); 7852 rt_mutex_adjust_pi(p); 7853 } 7854 7855 /* Run balance callbacks after we've adjusted the PI chain: */ 7856 balance_callbacks(rq, head); 7857 preempt_enable(); 7858 7859 return 0; 7860 7861 unlock: 7862 task_rq_unlock(rq, p, &rf); 7863 if (cpuset_locked) 7864 cpuset_unlock(); 7865 return retval; 7866 } 7867 7868 static int _sched_setscheduler(struct task_struct *p, int policy, 7869 const struct sched_param *param, bool check) 7870 { 7871 struct sched_attr attr = { 7872 .sched_policy = policy, 7873 .sched_priority = param->sched_priority, 7874 .sched_nice = PRIO_TO_NICE(p->static_prio), 7875 }; 7876 7877 /* Fixup the legacy SCHED_RESET_ON_FORK hack. */ 7878 if ((policy != SETPARAM_POLICY) && (policy & SCHED_RESET_ON_FORK)) { 7879 attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK; 7880 policy &= ~SCHED_RESET_ON_FORK; 7881 attr.sched_policy = policy; 7882 } 7883 7884 return __sched_setscheduler(p, &attr, check, true); 7885 } 7886 /** 7887 * sched_setscheduler - change the scheduling policy and/or RT priority of a thread. 7888 * @p: the task in question. 7889 * @policy: new policy. 7890 * @param: structure containing the new RT priority. 7891 * 7892 * Use sched_set_fifo(), read its comment. 7893 * 7894 * Return: 0 on success. An error code otherwise. 7895 * 7896 * NOTE that the task may be already dead. 7897 */ 7898 int sched_setscheduler(struct task_struct *p, int policy, 7899 const struct sched_param *param) 7900 { 7901 return _sched_setscheduler(p, policy, param, true); 7902 } 7903 7904 int sched_setattr(struct task_struct *p, const struct sched_attr *attr) 7905 { 7906 return __sched_setscheduler(p, attr, true, true); 7907 } 7908 7909 int sched_setattr_nocheck(struct task_struct *p, const struct sched_attr *attr) 7910 { 7911 return __sched_setscheduler(p, attr, false, true); 7912 } 7913 EXPORT_SYMBOL_GPL(sched_setattr_nocheck); 7914 7915 /** 7916 * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace. 7917 * @p: the task in question. 7918 * @policy: new policy. 7919 * @param: structure containing the new RT priority. 7920 * 7921 * Just like sched_setscheduler, only don't bother checking if the 7922 * current context has permission. For example, this is needed in 7923 * stop_machine(): we create temporary high priority worker threads, 7924 * but our caller might not have that capability. 7925 * 7926 * Return: 0 on success. An error code otherwise. 7927 */ 7928 int sched_setscheduler_nocheck(struct task_struct *p, int policy, 7929 const struct sched_param *param) 7930 { 7931 return _sched_setscheduler(p, policy, param, false); 7932 } 7933 7934 /* 7935 * SCHED_FIFO is a broken scheduler model; that is, it is fundamentally 7936 * incapable of resource management, which is the one thing an OS really should 7937 * be doing. 7938 * 7939 * This is of course the reason it is limited to privileged users only. 7940 * 7941 * Worse still; it is fundamentally impossible to compose static priority 7942 * workloads. You cannot take two correctly working static prio workloads 7943 * and smash them together and still expect them to work. 7944 * 7945 * For this reason 'all' FIFO tasks the kernel creates are basically at: 7946 * 7947 * MAX_RT_PRIO / 2 7948 * 7949 * The administrator _MUST_ configure the system, the kernel simply doesn't 7950 * know enough information to make a sensible choice. 7951 */ 7952 void sched_set_fifo(struct task_struct *p) 7953 { 7954 struct sched_param sp = { .sched_priority = MAX_RT_PRIO / 2 }; 7955 WARN_ON_ONCE(sched_setscheduler_nocheck(p, SCHED_FIFO, &sp) != 0); 7956 } 7957 EXPORT_SYMBOL_GPL(sched_set_fifo); 7958 7959 /* 7960 * For when you don't much care about FIFO, but want to be above SCHED_NORMAL. 7961 */ 7962 void sched_set_fifo_low(struct task_struct *p) 7963 { 7964 struct sched_param sp = { .sched_priority = 1 }; 7965 WARN_ON_ONCE(sched_setscheduler_nocheck(p, SCHED_FIFO, &sp) != 0); 7966 } 7967 EXPORT_SYMBOL_GPL(sched_set_fifo_low); 7968 7969 void sched_set_normal(struct task_struct *p, int nice) 7970 { 7971 struct sched_attr attr = { 7972 .sched_policy = SCHED_NORMAL, 7973 .sched_nice = nice, 7974 }; 7975 WARN_ON_ONCE(sched_setattr_nocheck(p, &attr) != 0); 7976 } 7977 EXPORT_SYMBOL_GPL(sched_set_normal); 7978 7979 static int 7980 do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param) 7981 { 7982 struct sched_param lparam; 7983 struct task_struct *p; 7984 int retval; 7985 7986 if (!param || pid < 0) 7987 return -EINVAL; 7988 if (copy_from_user(&lparam, param, sizeof(struct sched_param))) 7989 return -EFAULT; 7990 7991 rcu_read_lock(); 7992 retval = -ESRCH; 7993 p = find_process_by_pid(pid); 7994 if (likely(p)) 7995 get_task_struct(p); 7996 rcu_read_unlock(); 7997 7998 if (likely(p)) { 7999 retval = sched_setscheduler(p, policy, &lparam); 8000 put_task_struct(p); 8001 } 8002 8003 return retval; 8004 } 8005 8006 /* 8007 * Mimics kernel/events/core.c perf_copy_attr(). 8008 */ 8009 static int sched_copy_attr(struct sched_attr __user *uattr, struct sched_attr *attr) 8010 { 8011 u32 size; 8012 int ret; 8013 8014 /* Zero the full structure, so that a short copy will be nice: */ 8015 memset(attr, 0, sizeof(*attr)); 8016 8017 ret = get_user(size, &uattr->size); 8018 if (ret) 8019 return ret; 8020 8021 /* ABI compatibility quirk: */ 8022 if (!size) 8023 size = SCHED_ATTR_SIZE_VER0; 8024 if (size < SCHED_ATTR_SIZE_VER0 || size > PAGE_SIZE) 8025 goto err_size; 8026 8027 ret = copy_struct_from_user(attr, sizeof(*attr), uattr, size); 8028 if (ret) { 8029 if (ret == -E2BIG) 8030 goto err_size; 8031 return ret; 8032 } 8033 8034 if ((attr->sched_flags & SCHED_FLAG_UTIL_CLAMP) && 8035 size < SCHED_ATTR_SIZE_VER1) 8036 return -EINVAL; 8037 8038 /* 8039 * XXX: Do we want to be lenient like existing syscalls; or do we want 8040 * to be strict and return an error on out-of-bounds values? 8041 */ 8042 attr->sched_nice = clamp(attr->sched_nice, MIN_NICE, MAX_NICE); 8043 8044 return 0; 8045 8046 err_size: 8047 put_user(sizeof(*attr), &uattr->size); 8048 return -E2BIG; 8049 } 8050 8051 static void get_params(struct task_struct *p, struct sched_attr *attr) 8052 { 8053 if (task_has_dl_policy(p)) 8054 __getparam_dl(p, attr); 8055 else if (task_has_rt_policy(p)) 8056 attr->sched_priority = p->rt_priority; 8057 else 8058 attr->sched_nice = task_nice(p); 8059 } 8060 8061 /** 8062 * sys_sched_setscheduler - set/change the scheduler policy and RT priority 8063 * @pid: the pid in question. 8064 * @policy: new policy. 8065 * @param: structure containing the new RT priority. 8066 * 8067 * Return: 0 on success. An error code otherwise. 8068 */ 8069 SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy, struct sched_param __user *, param) 8070 { 8071 if (policy < 0) 8072 return -EINVAL; 8073 8074 return do_sched_setscheduler(pid, policy, param); 8075 } 8076 8077 /** 8078 * sys_sched_setparam - set/change the RT priority of a thread 8079 * @pid: the pid in question. 8080 * @param: structure containing the new RT priority. 8081 * 8082 * Return: 0 on success. An error code otherwise. 8083 */ 8084 SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param) 8085 { 8086 return do_sched_setscheduler(pid, SETPARAM_POLICY, param); 8087 } 8088 8089 /** 8090 * sys_sched_setattr - same as above, but with extended sched_attr 8091 * @pid: the pid in question. 8092 * @uattr: structure containing the extended parameters. 8093 * @flags: for future extension. 8094 */ 8095 SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr, 8096 unsigned int, flags) 8097 { 8098 struct sched_attr attr; 8099 struct task_struct *p; 8100 int retval; 8101 8102 if (!uattr || pid < 0 || flags) 8103 return -EINVAL; 8104 8105 retval = sched_copy_attr(uattr, &attr); 8106 if (retval) 8107 return retval; 8108 8109 if ((int)attr.sched_policy < 0) 8110 return -EINVAL; 8111 if (attr.sched_flags & SCHED_FLAG_KEEP_POLICY) 8112 attr.sched_policy = SETPARAM_POLICY; 8113 8114 rcu_read_lock(); 8115 retval = -ESRCH; 8116 p = find_process_by_pid(pid); 8117 if (likely(p)) 8118 get_task_struct(p); 8119 rcu_read_unlock(); 8120 8121 if (likely(p)) { 8122 if (attr.sched_flags & SCHED_FLAG_KEEP_PARAMS) 8123 get_params(p, &attr); 8124 retval = sched_setattr(p, &attr); 8125 put_task_struct(p); 8126 } 8127 8128 return retval; 8129 } 8130 8131 /** 8132 * sys_sched_getscheduler - get the policy (scheduling class) of a thread 8133 * @pid: the pid in question. 8134 * 8135 * Return: On success, the policy of the thread. Otherwise, a negative error 8136 * code. 8137 */ 8138 SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid) 8139 { 8140 struct task_struct *p; 8141 int retval; 8142 8143 if (pid < 0) 8144 return -EINVAL; 8145 8146 retval = -ESRCH; 8147 rcu_read_lock(); 8148 p = find_process_by_pid(pid); 8149 if (p) { 8150 retval = security_task_getscheduler(p); 8151 if (!retval) 8152 retval = p->policy 8153 | (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0); 8154 } 8155 rcu_read_unlock(); 8156 return retval; 8157 } 8158 8159 /** 8160 * sys_sched_getparam - get the RT priority of a thread 8161 * @pid: the pid in question. 8162 * @param: structure containing the RT priority. 8163 * 8164 * Return: On success, 0 and the RT priority is in @param. Otherwise, an error 8165 * code. 8166 */ 8167 SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param) 8168 { 8169 struct sched_param lp = { .sched_priority = 0 }; 8170 struct task_struct *p; 8171 int retval; 8172 8173 if (!param || pid < 0) 8174 return -EINVAL; 8175 8176 rcu_read_lock(); 8177 p = find_process_by_pid(pid); 8178 retval = -ESRCH; 8179 if (!p) 8180 goto out_unlock; 8181 8182 retval = security_task_getscheduler(p); 8183 if (retval) 8184 goto out_unlock; 8185 8186 if (task_has_rt_policy(p)) 8187 lp.sched_priority = p->rt_priority; 8188 rcu_read_unlock(); 8189 8190 /* 8191 * This one might sleep, we cannot do it with a spinlock held ... 8192 */ 8193 retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0; 8194 8195 return retval; 8196 8197 out_unlock: 8198 rcu_read_unlock(); 8199 return retval; 8200 } 8201 8202 /* 8203 * Copy the kernel size attribute structure (which might be larger 8204 * than what user-space knows about) to user-space. 8205 * 8206 * Note that all cases are valid: user-space buffer can be larger or 8207 * smaller than the kernel-space buffer. The usual case is that both 8208 * have the same size. 8209 */ 8210 static int 8211 sched_attr_copy_to_user(struct sched_attr __user *uattr, 8212 struct sched_attr *kattr, 8213 unsigned int usize) 8214 { 8215 unsigned int ksize = sizeof(*kattr); 8216 8217 if (!access_ok(uattr, usize)) 8218 return -EFAULT; 8219 8220 /* 8221 * sched_getattr() ABI forwards and backwards compatibility: 8222 * 8223 * If usize == ksize then we just copy everything to user-space and all is good. 8224 * 8225 * If usize < ksize then we only copy as much as user-space has space for, 8226 * this keeps ABI compatibility as well. We skip the rest. 8227 * 8228 * If usize > ksize then user-space is using a newer version of the ABI, 8229 * which part the kernel doesn't know about. Just ignore it - tooling can 8230 * detect the kernel's knowledge of attributes from the attr->size value 8231 * which is set to ksize in this case. 8232 */ 8233 kattr->size = min(usize, ksize); 8234 8235 if (copy_to_user(uattr, kattr, kattr->size)) 8236 return -EFAULT; 8237 8238 return 0; 8239 } 8240 8241 /** 8242 * sys_sched_getattr - similar to sched_getparam, but with sched_attr 8243 * @pid: the pid in question. 8244 * @uattr: structure containing the extended parameters. 8245 * @usize: sizeof(attr) for fwd/bwd comp. 8246 * @flags: for future extension. 8247 */ 8248 SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr, 8249 unsigned int, usize, unsigned int, flags) 8250 { 8251 struct sched_attr kattr = { }; 8252 struct task_struct *p; 8253 int retval; 8254 8255 if (!uattr || pid < 0 || usize > PAGE_SIZE || 8256 usize < SCHED_ATTR_SIZE_VER0 || flags) 8257 return -EINVAL; 8258 8259 rcu_read_lock(); 8260 p = find_process_by_pid(pid); 8261 retval = -ESRCH; 8262 if (!p) 8263 goto out_unlock; 8264 8265 retval = security_task_getscheduler(p); 8266 if (retval) 8267 goto out_unlock; 8268 8269 kattr.sched_policy = p->policy; 8270 if (p->sched_reset_on_fork) 8271 kattr.sched_flags |= SCHED_FLAG_RESET_ON_FORK; 8272 get_params(p, &kattr); 8273 kattr.sched_flags &= SCHED_FLAG_ALL; 8274 8275 #ifdef CONFIG_UCLAMP_TASK 8276 /* 8277 * This could race with another potential updater, but this is fine 8278 * because it'll correctly read the old or the new value. We don't need 8279 * to guarantee who wins the race as long as it doesn't return garbage. 8280 */ 8281 kattr.sched_util_min = p->uclamp_req[UCLAMP_MIN].value; 8282 kattr.sched_util_max = p->uclamp_req[UCLAMP_MAX].value; 8283 #endif 8284 8285 rcu_read_unlock(); 8286 8287 return sched_attr_copy_to_user(uattr, &kattr, usize); 8288 8289 out_unlock: 8290 rcu_read_unlock(); 8291 return retval; 8292 } 8293 8294 #ifdef CONFIG_SMP 8295 int dl_task_check_affinity(struct task_struct *p, const struct cpumask *mask) 8296 { 8297 int ret = 0; 8298 8299 /* 8300 * If the task isn't a deadline task or admission control is 8301 * disabled then we don't care about affinity changes. 8302 */ 8303 if (!task_has_dl_policy(p) || !dl_bandwidth_enabled()) 8304 return 0; 8305 8306 /* 8307 * Since bandwidth control happens on root_domain basis, 8308 * if admission test is enabled, we only admit -deadline 8309 * tasks allowed to run on all the CPUs in the task's 8310 * root_domain. 8311 */ 8312 rcu_read_lock(); 8313 if (!cpumask_subset(task_rq(p)->rd->span, mask)) 8314 ret = -EBUSY; 8315 rcu_read_unlock(); 8316 return ret; 8317 } 8318 #endif 8319 8320 static int 8321 __sched_setaffinity(struct task_struct *p, struct affinity_context *ctx) 8322 { 8323 int retval; 8324 cpumask_var_t cpus_allowed, new_mask; 8325 8326 if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) 8327 return -ENOMEM; 8328 8329 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) { 8330 retval = -ENOMEM; 8331 goto out_free_cpus_allowed; 8332 } 8333 8334 cpuset_cpus_allowed(p, cpus_allowed); 8335 cpumask_and(new_mask, ctx->new_mask, cpus_allowed); 8336 8337 ctx->new_mask = new_mask; 8338 ctx->flags |= SCA_CHECK; 8339 8340 retval = dl_task_check_affinity(p, new_mask); 8341 if (retval) 8342 goto out_free_new_mask; 8343 8344 retval = __set_cpus_allowed_ptr(p, ctx); 8345 if (retval) 8346 goto out_free_new_mask; 8347 8348 cpuset_cpus_allowed(p, cpus_allowed); 8349 if (!cpumask_subset(new_mask, cpus_allowed)) { 8350 /* 8351 * We must have raced with a concurrent cpuset update. 8352 * Just reset the cpumask to the cpuset's cpus_allowed. 8353 */ 8354 cpumask_copy(new_mask, cpus_allowed); 8355 8356 /* 8357 * If SCA_USER is set, a 2nd call to __set_cpus_allowed_ptr() 8358 * will restore the previous user_cpus_ptr value. 8359 * 8360 * In the unlikely event a previous user_cpus_ptr exists, 8361 * we need to further restrict the mask to what is allowed 8362 * by that old user_cpus_ptr. 8363 */ 8364 if (unlikely((ctx->flags & SCA_USER) && ctx->user_mask)) { 8365 bool empty = !cpumask_and(new_mask, new_mask, 8366 ctx->user_mask); 8367 8368 if (WARN_ON_ONCE(empty)) 8369 cpumask_copy(new_mask, cpus_allowed); 8370 } 8371 __set_cpus_allowed_ptr(p, ctx); 8372 retval = -EINVAL; 8373 } 8374 8375 out_free_new_mask: 8376 free_cpumask_var(new_mask); 8377 out_free_cpus_allowed: 8378 free_cpumask_var(cpus_allowed); 8379 return retval; 8380 } 8381 8382 long sched_setaffinity(pid_t pid, const struct cpumask *in_mask) 8383 { 8384 struct affinity_context ac; 8385 struct cpumask *user_mask; 8386 struct task_struct *p; 8387 int retval; 8388 8389 rcu_read_lock(); 8390 8391 p = find_process_by_pid(pid); 8392 if (!p) { 8393 rcu_read_unlock(); 8394 return -ESRCH; 8395 } 8396 8397 /* Prevent p going away */ 8398 get_task_struct(p); 8399 rcu_read_unlock(); 8400 8401 if (p->flags & PF_NO_SETAFFINITY) { 8402 retval = -EINVAL; 8403 goto out_put_task; 8404 } 8405 8406 if (!check_same_owner(p)) { 8407 rcu_read_lock(); 8408 if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) { 8409 rcu_read_unlock(); 8410 retval = -EPERM; 8411 goto out_put_task; 8412 } 8413 rcu_read_unlock(); 8414 } 8415 8416 retval = security_task_setscheduler(p); 8417 if (retval) 8418 goto out_put_task; 8419 8420 /* 8421 * With non-SMP configs, user_cpus_ptr/user_mask isn't used and 8422 * alloc_user_cpus_ptr() returns NULL. 8423 */ 8424 user_mask = alloc_user_cpus_ptr(NUMA_NO_NODE); 8425 if (user_mask) { 8426 cpumask_copy(user_mask, in_mask); 8427 } else if (IS_ENABLED(CONFIG_SMP)) { 8428 retval = -ENOMEM; 8429 goto out_put_task; 8430 } 8431 8432 ac = (struct affinity_context){ 8433 .new_mask = in_mask, 8434 .user_mask = user_mask, 8435 .flags = SCA_USER, 8436 }; 8437 8438 retval = __sched_setaffinity(p, &ac); 8439 kfree(ac.user_mask); 8440 8441 out_put_task: 8442 put_task_struct(p); 8443 return retval; 8444 } 8445 8446 static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len, 8447 struct cpumask *new_mask) 8448 { 8449 if (len < cpumask_size()) 8450 cpumask_clear(new_mask); 8451 else if (len > cpumask_size()) 8452 len = cpumask_size(); 8453 8454 return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0; 8455 } 8456 8457 /** 8458 * sys_sched_setaffinity - set the CPU affinity of a process 8459 * @pid: pid of the process 8460 * @len: length in bytes of the bitmask pointed to by user_mask_ptr 8461 * @user_mask_ptr: user-space pointer to the new CPU mask 8462 * 8463 * Return: 0 on success. An error code otherwise. 8464 */ 8465 SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len, 8466 unsigned long __user *, user_mask_ptr) 8467 { 8468 cpumask_var_t new_mask; 8469 int retval; 8470 8471 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) 8472 return -ENOMEM; 8473 8474 retval = get_user_cpu_mask(user_mask_ptr, len, new_mask); 8475 if (retval == 0) 8476 retval = sched_setaffinity(pid, new_mask); 8477 free_cpumask_var(new_mask); 8478 return retval; 8479 } 8480 8481 long sched_getaffinity(pid_t pid, struct cpumask *mask) 8482 { 8483 struct task_struct *p; 8484 unsigned long flags; 8485 int retval; 8486 8487 rcu_read_lock(); 8488 8489 retval = -ESRCH; 8490 p = find_process_by_pid(pid); 8491 if (!p) 8492 goto out_unlock; 8493 8494 retval = security_task_getscheduler(p); 8495 if (retval) 8496 goto out_unlock; 8497 8498 raw_spin_lock_irqsave(&p->pi_lock, flags); 8499 cpumask_and(mask, &p->cpus_mask, cpu_active_mask); 8500 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 8501 8502 out_unlock: 8503 rcu_read_unlock(); 8504 8505 return retval; 8506 } 8507 8508 /** 8509 * sys_sched_getaffinity - get the CPU affinity of a process 8510 * @pid: pid of the process 8511 * @len: length in bytes of the bitmask pointed to by user_mask_ptr 8512 * @user_mask_ptr: user-space pointer to hold the current CPU mask 8513 * 8514 * Return: size of CPU mask copied to user_mask_ptr on success. An 8515 * error code otherwise. 8516 */ 8517 SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len, 8518 unsigned long __user *, user_mask_ptr) 8519 { 8520 int ret; 8521 cpumask_var_t mask; 8522 8523 if ((len * BITS_PER_BYTE) < nr_cpu_ids) 8524 return -EINVAL; 8525 if (len & (sizeof(unsigned long)-1)) 8526 return -EINVAL; 8527 8528 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) 8529 return -ENOMEM; 8530 8531 ret = sched_getaffinity(pid, mask); 8532 if (ret == 0) { 8533 unsigned int retlen = min(len, cpumask_size()); 8534 8535 if (copy_to_user(user_mask_ptr, cpumask_bits(mask), retlen)) 8536 ret = -EFAULT; 8537 else 8538 ret = retlen; 8539 } 8540 free_cpumask_var(mask); 8541 8542 return ret; 8543 } 8544 8545 static void do_sched_yield(void) 8546 { 8547 struct rq_flags rf; 8548 struct rq *rq; 8549 8550 rq = this_rq_lock_irq(&rf); 8551 8552 schedstat_inc(rq->yld_count); 8553 current->sched_class->yield_task(rq); 8554 8555 preempt_disable(); 8556 rq_unlock_irq(rq, &rf); 8557 sched_preempt_enable_no_resched(); 8558 8559 schedule(); 8560 } 8561 8562 /** 8563 * sys_sched_yield - yield the current processor to other threads. 8564 * 8565 * This function yields the current CPU to other tasks. If there are no 8566 * other threads running on this CPU then this function will return. 8567 * 8568 * Return: 0. 8569 */ 8570 SYSCALL_DEFINE0(sched_yield) 8571 { 8572 do_sched_yield(); 8573 return 0; 8574 } 8575 8576 #if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC) 8577 int __sched __cond_resched(void) 8578 { 8579 if (should_resched(0)) { 8580 preempt_schedule_common(); 8581 return 1; 8582 } 8583 /* 8584 * In preemptible kernels, ->rcu_read_lock_nesting tells the tick 8585 * whether the current CPU is in an RCU read-side critical section, 8586 * so the tick can report quiescent states even for CPUs looping 8587 * in kernel context. In contrast, in non-preemptible kernels, 8588 * RCU readers leave no in-memory hints, which means that CPU-bound 8589 * processes executing in kernel context might never report an 8590 * RCU quiescent state. Therefore, the following code causes 8591 * cond_resched() to report a quiescent state, but only when RCU 8592 * is in urgent need of one. 8593 */ 8594 #ifndef CONFIG_PREEMPT_RCU 8595 rcu_all_qs(); 8596 #endif 8597 return 0; 8598 } 8599 EXPORT_SYMBOL(__cond_resched); 8600 #endif 8601 8602 #ifdef CONFIG_PREEMPT_DYNAMIC 8603 #if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 8604 #define cond_resched_dynamic_enabled __cond_resched 8605 #define cond_resched_dynamic_disabled ((void *)&__static_call_return0) 8606 DEFINE_STATIC_CALL_RET0(cond_resched, __cond_resched); 8607 EXPORT_STATIC_CALL_TRAMP(cond_resched); 8608 8609 #define might_resched_dynamic_enabled __cond_resched 8610 #define might_resched_dynamic_disabled ((void *)&__static_call_return0) 8611 DEFINE_STATIC_CALL_RET0(might_resched, __cond_resched); 8612 EXPORT_STATIC_CALL_TRAMP(might_resched); 8613 #elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 8614 static DEFINE_STATIC_KEY_FALSE(sk_dynamic_cond_resched); 8615 int __sched dynamic_cond_resched(void) 8616 { 8617 klp_sched_try_switch(); 8618 if (!static_branch_unlikely(&sk_dynamic_cond_resched)) 8619 return 0; 8620 return __cond_resched(); 8621 } 8622 EXPORT_SYMBOL(dynamic_cond_resched); 8623 8624 static DEFINE_STATIC_KEY_FALSE(sk_dynamic_might_resched); 8625 int __sched dynamic_might_resched(void) 8626 { 8627 if (!static_branch_unlikely(&sk_dynamic_might_resched)) 8628 return 0; 8629 return __cond_resched(); 8630 } 8631 EXPORT_SYMBOL(dynamic_might_resched); 8632 #endif 8633 #endif 8634 8635 /* 8636 * __cond_resched_lock() - if a reschedule is pending, drop the given lock, 8637 * call schedule, and on return reacquire the lock. 8638 * 8639 * This works OK both with and without CONFIG_PREEMPTION. We do strange low-level 8640 * operations here to prevent schedule() from being called twice (once via 8641 * spin_unlock(), once by hand). 8642 */ 8643 int __cond_resched_lock(spinlock_t *lock) 8644 { 8645 int resched = should_resched(PREEMPT_LOCK_OFFSET); 8646 int ret = 0; 8647 8648 lockdep_assert_held(lock); 8649 8650 if (spin_needbreak(lock) || resched) { 8651 spin_unlock(lock); 8652 if (!_cond_resched()) 8653 cpu_relax(); 8654 ret = 1; 8655 spin_lock(lock); 8656 } 8657 return ret; 8658 } 8659 EXPORT_SYMBOL(__cond_resched_lock); 8660 8661 int __cond_resched_rwlock_read(rwlock_t *lock) 8662 { 8663 int resched = should_resched(PREEMPT_LOCK_OFFSET); 8664 int ret = 0; 8665 8666 lockdep_assert_held_read(lock); 8667 8668 if (rwlock_needbreak(lock) || resched) { 8669 read_unlock(lock); 8670 if (!_cond_resched()) 8671 cpu_relax(); 8672 ret = 1; 8673 read_lock(lock); 8674 } 8675 return ret; 8676 } 8677 EXPORT_SYMBOL(__cond_resched_rwlock_read); 8678 8679 int __cond_resched_rwlock_write(rwlock_t *lock) 8680 { 8681 int resched = should_resched(PREEMPT_LOCK_OFFSET); 8682 int ret = 0; 8683 8684 lockdep_assert_held_write(lock); 8685 8686 if (rwlock_needbreak(lock) || resched) { 8687 write_unlock(lock); 8688 if (!_cond_resched()) 8689 cpu_relax(); 8690 ret = 1; 8691 write_lock(lock); 8692 } 8693 return ret; 8694 } 8695 EXPORT_SYMBOL(__cond_resched_rwlock_write); 8696 8697 #ifdef CONFIG_PREEMPT_DYNAMIC 8698 8699 #ifdef CONFIG_GENERIC_ENTRY 8700 #include <linux/entry-common.h> 8701 #endif 8702 8703 /* 8704 * SC:cond_resched 8705 * SC:might_resched 8706 * SC:preempt_schedule 8707 * SC:preempt_schedule_notrace 8708 * SC:irqentry_exit_cond_resched 8709 * 8710 * 8711 * NONE: 8712 * cond_resched <- __cond_resched 8713 * might_resched <- RET0 8714 * preempt_schedule <- NOP 8715 * preempt_schedule_notrace <- NOP 8716 * irqentry_exit_cond_resched <- NOP 8717 * 8718 * VOLUNTARY: 8719 * cond_resched <- __cond_resched 8720 * might_resched <- __cond_resched 8721 * preempt_schedule <- NOP 8722 * preempt_schedule_notrace <- NOP 8723 * irqentry_exit_cond_resched <- NOP 8724 * 8725 * FULL: 8726 * cond_resched <- RET0 8727 * might_resched <- RET0 8728 * preempt_schedule <- preempt_schedule 8729 * preempt_schedule_notrace <- preempt_schedule_notrace 8730 * irqentry_exit_cond_resched <- irqentry_exit_cond_resched 8731 */ 8732 8733 enum { 8734 preempt_dynamic_undefined = -1, 8735 preempt_dynamic_none, 8736 preempt_dynamic_voluntary, 8737 preempt_dynamic_full, 8738 }; 8739 8740 int preempt_dynamic_mode = preempt_dynamic_undefined; 8741 8742 int sched_dynamic_mode(const char *str) 8743 { 8744 if (!strcmp(str, "none")) 8745 return preempt_dynamic_none; 8746 8747 if (!strcmp(str, "voluntary")) 8748 return preempt_dynamic_voluntary; 8749 8750 if (!strcmp(str, "full")) 8751 return preempt_dynamic_full; 8752 8753 return -EINVAL; 8754 } 8755 8756 #if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 8757 #define preempt_dynamic_enable(f) static_call_update(f, f##_dynamic_enabled) 8758 #define preempt_dynamic_disable(f) static_call_update(f, f##_dynamic_disabled) 8759 #elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 8760 #define preempt_dynamic_enable(f) static_key_enable(&sk_dynamic_##f.key) 8761 #define preempt_dynamic_disable(f) static_key_disable(&sk_dynamic_##f.key) 8762 #else 8763 #error "Unsupported PREEMPT_DYNAMIC mechanism" 8764 #endif 8765 8766 static DEFINE_MUTEX(sched_dynamic_mutex); 8767 static bool klp_override; 8768 8769 static void __sched_dynamic_update(int mode) 8770 { 8771 /* 8772 * Avoid {NONE,VOLUNTARY} -> FULL transitions from ever ending up in 8773 * the ZERO state, which is invalid. 8774 */ 8775 if (!klp_override) 8776 preempt_dynamic_enable(cond_resched); 8777 preempt_dynamic_enable(might_resched); 8778 preempt_dynamic_enable(preempt_schedule); 8779 preempt_dynamic_enable(preempt_schedule_notrace); 8780 preempt_dynamic_enable(irqentry_exit_cond_resched); 8781 8782 switch (mode) { 8783 case preempt_dynamic_none: 8784 if (!klp_override) 8785 preempt_dynamic_enable(cond_resched); 8786 preempt_dynamic_disable(might_resched); 8787 preempt_dynamic_disable(preempt_schedule); 8788 preempt_dynamic_disable(preempt_schedule_notrace); 8789 preempt_dynamic_disable(irqentry_exit_cond_resched); 8790 if (mode != preempt_dynamic_mode) 8791 pr_info("Dynamic Preempt: none\n"); 8792 break; 8793 8794 case preempt_dynamic_voluntary: 8795 if (!klp_override) 8796 preempt_dynamic_enable(cond_resched); 8797 preempt_dynamic_enable(might_resched); 8798 preempt_dynamic_disable(preempt_schedule); 8799 preempt_dynamic_disable(preempt_schedule_notrace); 8800 preempt_dynamic_disable(irqentry_exit_cond_resched); 8801 if (mode != preempt_dynamic_mode) 8802 pr_info("Dynamic Preempt: voluntary\n"); 8803 break; 8804 8805 case preempt_dynamic_full: 8806 if (!klp_override) 8807 preempt_dynamic_disable(cond_resched); 8808 preempt_dynamic_disable(might_resched); 8809 preempt_dynamic_enable(preempt_schedule); 8810 preempt_dynamic_enable(preempt_schedule_notrace); 8811 preempt_dynamic_enable(irqentry_exit_cond_resched); 8812 if (mode != preempt_dynamic_mode) 8813 pr_info("Dynamic Preempt: full\n"); 8814 break; 8815 } 8816 8817 preempt_dynamic_mode = mode; 8818 } 8819 8820 void sched_dynamic_update(int mode) 8821 { 8822 mutex_lock(&sched_dynamic_mutex); 8823 __sched_dynamic_update(mode); 8824 mutex_unlock(&sched_dynamic_mutex); 8825 } 8826 8827 #ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL 8828 8829 static int klp_cond_resched(void) 8830 { 8831 __klp_sched_try_switch(); 8832 return __cond_resched(); 8833 } 8834 8835 void sched_dynamic_klp_enable(void) 8836 { 8837 mutex_lock(&sched_dynamic_mutex); 8838 8839 klp_override = true; 8840 static_call_update(cond_resched, klp_cond_resched); 8841 8842 mutex_unlock(&sched_dynamic_mutex); 8843 } 8844 8845 void sched_dynamic_klp_disable(void) 8846 { 8847 mutex_lock(&sched_dynamic_mutex); 8848 8849 klp_override = false; 8850 __sched_dynamic_update(preempt_dynamic_mode); 8851 8852 mutex_unlock(&sched_dynamic_mutex); 8853 } 8854 8855 #endif /* CONFIG_HAVE_PREEMPT_DYNAMIC_CALL */ 8856 8857 static int __init setup_preempt_mode(char *str) 8858 { 8859 int mode = sched_dynamic_mode(str); 8860 if (mode < 0) { 8861 pr_warn("Dynamic Preempt: unsupported mode: %s\n", str); 8862 return 0; 8863 } 8864 8865 sched_dynamic_update(mode); 8866 return 1; 8867 } 8868 __setup("preempt=", setup_preempt_mode); 8869 8870 static void __init preempt_dynamic_init(void) 8871 { 8872 if (preempt_dynamic_mode == preempt_dynamic_undefined) { 8873 if (IS_ENABLED(CONFIG_PREEMPT_NONE)) { 8874 sched_dynamic_update(preempt_dynamic_none); 8875 } else if (IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY)) { 8876 sched_dynamic_update(preempt_dynamic_voluntary); 8877 } else { 8878 /* Default static call setting, nothing to do */ 8879 WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT)); 8880 preempt_dynamic_mode = preempt_dynamic_full; 8881 pr_info("Dynamic Preempt: full\n"); 8882 } 8883 } 8884 } 8885 8886 #define PREEMPT_MODEL_ACCESSOR(mode) \ 8887 bool preempt_model_##mode(void) \ 8888 { \ 8889 WARN_ON_ONCE(preempt_dynamic_mode == preempt_dynamic_undefined); \ 8890 return preempt_dynamic_mode == preempt_dynamic_##mode; \ 8891 } \ 8892 EXPORT_SYMBOL_GPL(preempt_model_##mode) 8893 8894 PREEMPT_MODEL_ACCESSOR(none); 8895 PREEMPT_MODEL_ACCESSOR(voluntary); 8896 PREEMPT_MODEL_ACCESSOR(full); 8897 8898 #else /* !CONFIG_PREEMPT_DYNAMIC */ 8899 8900 static inline void preempt_dynamic_init(void) { } 8901 8902 #endif /* #ifdef CONFIG_PREEMPT_DYNAMIC */ 8903 8904 /** 8905 * yield - yield the current processor to other threads. 8906 * 8907 * Do not ever use this function, there's a 99% chance you're doing it wrong. 8908 * 8909 * The scheduler is at all times free to pick the calling task as the most 8910 * eligible task to run, if removing the yield() call from your code breaks 8911 * it, it's already broken. 8912 * 8913 * Typical broken usage is: 8914 * 8915 * while (!event) 8916 * yield(); 8917 * 8918 * where one assumes that yield() will let 'the other' process run that will 8919 * make event true. If the current task is a SCHED_FIFO task that will never 8920 * happen. Never use yield() as a progress guarantee!! 8921 * 8922 * If you want to use yield() to wait for something, use wait_event(). 8923 * If you want to use yield() to be 'nice' for others, use cond_resched(). 8924 * If you still want to use yield(), do not! 8925 */ 8926 void __sched yield(void) 8927 { 8928 set_current_state(TASK_RUNNING); 8929 do_sched_yield(); 8930 } 8931 EXPORT_SYMBOL(yield); 8932 8933 /** 8934 * yield_to - yield the current processor to another thread in 8935 * your thread group, or accelerate that thread toward the 8936 * processor it's on. 8937 * @p: target task 8938 * @preempt: whether task preemption is allowed or not 8939 * 8940 * It's the caller's job to ensure that the target task struct 8941 * can't go away on us before we can do any checks. 8942 * 8943 * Return: 8944 * true (>0) if we indeed boosted the target task. 8945 * false (0) if we failed to boost the target. 8946 * -ESRCH if there's no task to yield to. 8947 */ 8948 int __sched yield_to(struct task_struct *p, bool preempt) 8949 { 8950 struct task_struct *curr = current; 8951 struct rq *rq, *p_rq; 8952 unsigned long flags; 8953 int yielded = 0; 8954 8955 local_irq_save(flags); 8956 rq = this_rq(); 8957 8958 again: 8959 p_rq = task_rq(p); 8960 /* 8961 * If we're the only runnable task on the rq and target rq also 8962 * has only one task, there's absolutely no point in yielding. 8963 */ 8964 if (rq->nr_running == 1 && p_rq->nr_running == 1) { 8965 yielded = -ESRCH; 8966 goto out_irq; 8967 } 8968 8969 double_rq_lock(rq, p_rq); 8970 if (task_rq(p) != p_rq) { 8971 double_rq_unlock(rq, p_rq); 8972 goto again; 8973 } 8974 8975 if (!curr->sched_class->yield_to_task) 8976 goto out_unlock; 8977 8978 if (curr->sched_class != p->sched_class) 8979 goto out_unlock; 8980 8981 if (task_on_cpu(p_rq, p) || !task_is_running(p)) 8982 goto out_unlock; 8983 8984 yielded = curr->sched_class->yield_to_task(rq, p); 8985 if (yielded) { 8986 schedstat_inc(rq->yld_count); 8987 /* 8988 * Make p's CPU reschedule; pick_next_entity takes care of 8989 * fairness. 8990 */ 8991 if (preempt && rq != p_rq) 8992 resched_curr(p_rq); 8993 } 8994 8995 out_unlock: 8996 double_rq_unlock(rq, p_rq); 8997 out_irq: 8998 local_irq_restore(flags); 8999 9000 if (yielded > 0) 9001 schedule(); 9002 9003 return yielded; 9004 } 9005 EXPORT_SYMBOL_GPL(yield_to); 9006 9007 int io_schedule_prepare(void) 9008 { 9009 int old_iowait = current->in_iowait; 9010 9011 current->in_iowait = 1; 9012 blk_flush_plug(current->plug, true); 9013 return old_iowait; 9014 } 9015 9016 void io_schedule_finish(int token) 9017 { 9018 current->in_iowait = token; 9019 } 9020 9021 /* 9022 * This task is about to go to sleep on IO. Increment rq->nr_iowait so 9023 * that process accounting knows that this is a task in IO wait state. 9024 */ 9025 long __sched io_schedule_timeout(long timeout) 9026 { 9027 int token; 9028 long ret; 9029 9030 token = io_schedule_prepare(); 9031 ret = schedule_timeout(timeout); 9032 io_schedule_finish(token); 9033 9034 return ret; 9035 } 9036 EXPORT_SYMBOL(io_schedule_timeout); 9037 9038 void __sched io_schedule(void) 9039 { 9040 int token; 9041 9042 token = io_schedule_prepare(); 9043 schedule(); 9044 io_schedule_finish(token); 9045 } 9046 EXPORT_SYMBOL(io_schedule); 9047 9048 /** 9049 * sys_sched_get_priority_max - return maximum RT priority. 9050 * @policy: scheduling class. 9051 * 9052 * Return: On success, this syscall returns the maximum 9053 * rt_priority that can be used by a given scheduling class. 9054 * On failure, a negative error code is returned. 9055 */ 9056 SYSCALL_DEFINE1(sched_get_priority_max, int, policy) 9057 { 9058 int ret = -EINVAL; 9059 9060 switch (policy) { 9061 case SCHED_FIFO: 9062 case SCHED_RR: 9063 ret = MAX_RT_PRIO-1; 9064 break; 9065 case SCHED_DEADLINE: 9066 case SCHED_NORMAL: 9067 case SCHED_BATCH: 9068 case SCHED_IDLE: 9069 ret = 0; 9070 break; 9071 } 9072 return ret; 9073 } 9074 9075 /** 9076 * sys_sched_get_priority_min - return minimum RT priority. 9077 * @policy: scheduling class. 9078 * 9079 * Return: On success, this syscall returns the minimum 9080 * rt_priority that can be used by a given scheduling class. 9081 * On failure, a negative error code is returned. 9082 */ 9083 SYSCALL_DEFINE1(sched_get_priority_min, int, policy) 9084 { 9085 int ret = -EINVAL; 9086 9087 switch (policy) { 9088 case SCHED_FIFO: 9089 case SCHED_RR: 9090 ret = 1; 9091 break; 9092 case SCHED_DEADLINE: 9093 case SCHED_NORMAL: 9094 case SCHED_BATCH: 9095 case SCHED_IDLE: 9096 ret = 0; 9097 } 9098 return ret; 9099 } 9100 9101 static int sched_rr_get_interval(pid_t pid, struct timespec64 *t) 9102 { 9103 struct task_struct *p; 9104 unsigned int time_slice; 9105 struct rq_flags rf; 9106 struct rq *rq; 9107 int retval; 9108 9109 if (pid < 0) 9110 return -EINVAL; 9111 9112 retval = -ESRCH; 9113 rcu_read_lock(); 9114 p = find_process_by_pid(pid); 9115 if (!p) 9116 goto out_unlock; 9117 9118 retval = security_task_getscheduler(p); 9119 if (retval) 9120 goto out_unlock; 9121 9122 rq = task_rq_lock(p, &rf); 9123 time_slice = 0; 9124 if (p->sched_class->get_rr_interval) 9125 time_slice = p->sched_class->get_rr_interval(rq, p); 9126 task_rq_unlock(rq, p, &rf); 9127 9128 rcu_read_unlock(); 9129 jiffies_to_timespec64(time_slice, t); 9130 return 0; 9131 9132 out_unlock: 9133 rcu_read_unlock(); 9134 return retval; 9135 } 9136 9137 /** 9138 * sys_sched_rr_get_interval - return the default timeslice of a process. 9139 * @pid: pid of the process. 9140 * @interval: userspace pointer to the timeslice value. 9141 * 9142 * this syscall writes the default timeslice value of a given process 9143 * into the user-space timespec buffer. A value of '0' means infinity. 9144 * 9145 * Return: On success, 0 and the timeslice is in @interval. Otherwise, 9146 * an error code. 9147 */ 9148 SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid, 9149 struct __kernel_timespec __user *, interval) 9150 { 9151 struct timespec64 t; 9152 int retval = sched_rr_get_interval(pid, &t); 9153 9154 if (retval == 0) 9155 retval = put_timespec64(&t, interval); 9156 9157 return retval; 9158 } 9159 9160 #ifdef CONFIG_COMPAT_32BIT_TIME 9161 SYSCALL_DEFINE2(sched_rr_get_interval_time32, pid_t, pid, 9162 struct old_timespec32 __user *, interval) 9163 { 9164 struct timespec64 t; 9165 int retval = sched_rr_get_interval(pid, &t); 9166 9167 if (retval == 0) 9168 retval = put_old_timespec32(&t, interval); 9169 return retval; 9170 } 9171 #endif 9172 9173 void sched_show_task(struct task_struct *p) 9174 { 9175 unsigned long free = 0; 9176 int ppid; 9177 9178 if (!try_get_task_stack(p)) 9179 return; 9180 9181 pr_info("task:%-15.15s state:%c", p->comm, task_state_to_char(p)); 9182 9183 if (task_is_running(p)) 9184 pr_cont(" running task "); 9185 #ifdef CONFIG_DEBUG_STACK_USAGE 9186 free = stack_not_used(p); 9187 #endif 9188 ppid = 0; 9189 rcu_read_lock(); 9190 if (pid_alive(p)) 9191 ppid = task_pid_nr(rcu_dereference(p->real_parent)); 9192 rcu_read_unlock(); 9193 pr_cont(" stack:%-5lu pid:%-5d ppid:%-6d flags:0x%08lx\n", 9194 free, task_pid_nr(p), ppid, 9195 read_task_thread_flags(p)); 9196 9197 print_worker_info(KERN_INFO, p); 9198 print_stop_info(KERN_INFO, p); 9199 show_stack(p, NULL, KERN_INFO); 9200 put_task_stack(p); 9201 } 9202 EXPORT_SYMBOL_GPL(sched_show_task); 9203 9204 static inline bool 9205 state_filter_match(unsigned long state_filter, struct task_struct *p) 9206 { 9207 unsigned int state = READ_ONCE(p->__state); 9208 9209 /* no filter, everything matches */ 9210 if (!state_filter) 9211 return true; 9212 9213 /* filter, but doesn't match */ 9214 if (!(state & state_filter)) 9215 return false; 9216 9217 /* 9218 * When looking for TASK_UNINTERRUPTIBLE skip TASK_IDLE (allows 9219 * TASK_KILLABLE). 9220 */ 9221 if (state_filter == TASK_UNINTERRUPTIBLE && (state & TASK_NOLOAD)) 9222 return false; 9223 9224 return true; 9225 } 9226 9227 9228 void show_state_filter(unsigned int state_filter) 9229 { 9230 struct task_struct *g, *p; 9231 9232 rcu_read_lock(); 9233 for_each_process_thread(g, p) { 9234 /* 9235 * reset the NMI-timeout, listing all files on a slow 9236 * console might take a lot of time: 9237 * Also, reset softlockup watchdogs on all CPUs, because 9238 * another CPU might be blocked waiting for us to process 9239 * an IPI. 9240 */ 9241 touch_nmi_watchdog(); 9242 touch_all_softlockup_watchdogs(); 9243 if (state_filter_match(state_filter, p)) 9244 sched_show_task(p); 9245 } 9246 9247 #ifdef CONFIG_SCHED_DEBUG 9248 if (!state_filter) 9249 sysrq_sched_debug_show(); 9250 #endif 9251 rcu_read_unlock(); 9252 /* 9253 * Only show locks if all tasks are dumped: 9254 */ 9255 if (!state_filter) 9256 debug_show_all_locks(); 9257 } 9258 9259 /** 9260 * init_idle - set up an idle thread for a given CPU 9261 * @idle: task in question 9262 * @cpu: CPU the idle task belongs to 9263 * 9264 * NOTE: this function does not set the idle thread's NEED_RESCHED 9265 * flag, to make booting more robust. 9266 */ 9267 void __init init_idle(struct task_struct *idle, int cpu) 9268 { 9269 #ifdef CONFIG_SMP 9270 struct affinity_context ac = (struct affinity_context) { 9271 .new_mask = cpumask_of(cpu), 9272 .flags = 0, 9273 }; 9274 #endif 9275 struct rq *rq = cpu_rq(cpu); 9276 unsigned long flags; 9277 9278 __sched_fork(0, idle); 9279 9280 raw_spin_lock_irqsave(&idle->pi_lock, flags); 9281 raw_spin_rq_lock(rq); 9282 9283 idle->__state = TASK_RUNNING; 9284 idle->se.exec_start = sched_clock(); 9285 /* 9286 * PF_KTHREAD should already be set at this point; regardless, make it 9287 * look like a proper per-CPU kthread. 9288 */ 9289 idle->flags |= PF_IDLE | PF_KTHREAD | PF_NO_SETAFFINITY; 9290 kthread_set_per_cpu(idle, cpu); 9291 9292 #ifdef CONFIG_SMP 9293 /* 9294 * It's possible that init_idle() gets called multiple times on a task, 9295 * in that case do_set_cpus_allowed() will not do the right thing. 9296 * 9297 * And since this is boot we can forgo the serialization. 9298 */ 9299 set_cpus_allowed_common(idle, &ac); 9300 #endif 9301 /* 9302 * We're having a chicken and egg problem, even though we are 9303 * holding rq->lock, the CPU isn't yet set to this CPU so the 9304 * lockdep check in task_group() will fail. 9305 * 9306 * Similar case to sched_fork(). / Alternatively we could 9307 * use task_rq_lock() here and obtain the other rq->lock. 9308 * 9309 * Silence PROVE_RCU 9310 */ 9311 rcu_read_lock(); 9312 __set_task_cpu(idle, cpu); 9313 rcu_read_unlock(); 9314 9315 rq->idle = idle; 9316 rcu_assign_pointer(rq->curr, idle); 9317 idle->on_rq = TASK_ON_RQ_QUEUED; 9318 #ifdef CONFIG_SMP 9319 idle->on_cpu = 1; 9320 #endif 9321 raw_spin_rq_unlock(rq); 9322 raw_spin_unlock_irqrestore(&idle->pi_lock, flags); 9323 9324 /* Set the preempt count _outside_ the spinlocks! */ 9325 init_idle_preempt_count(idle, cpu); 9326 9327 /* 9328 * The idle tasks have their own, simple scheduling class: 9329 */ 9330 idle->sched_class = &idle_sched_class; 9331 ftrace_graph_init_idle_task(idle, cpu); 9332 vtime_init_idle(idle, cpu); 9333 #ifdef CONFIG_SMP 9334 sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu); 9335 #endif 9336 } 9337 9338 #ifdef CONFIG_SMP 9339 9340 int cpuset_cpumask_can_shrink(const struct cpumask *cur, 9341 const struct cpumask *trial) 9342 { 9343 int ret = 1; 9344 9345 if (cpumask_empty(cur)) 9346 return ret; 9347 9348 ret = dl_cpuset_cpumask_can_shrink(cur, trial); 9349 9350 return ret; 9351 } 9352 9353 int task_can_attach(struct task_struct *p) 9354 { 9355 int ret = 0; 9356 9357 /* 9358 * Kthreads which disallow setaffinity shouldn't be moved 9359 * to a new cpuset; we don't want to change their CPU 9360 * affinity and isolating such threads by their set of 9361 * allowed nodes is unnecessary. Thus, cpusets are not 9362 * applicable for such threads. This prevents checking for 9363 * success of set_cpus_allowed_ptr() on all attached tasks 9364 * before cpus_mask may be changed. 9365 */ 9366 if (p->flags & PF_NO_SETAFFINITY) 9367 ret = -EINVAL; 9368 9369 return ret; 9370 } 9371 9372 bool sched_smp_initialized __read_mostly; 9373 9374 #ifdef CONFIG_NUMA_BALANCING 9375 /* Migrate current task p to target_cpu */ 9376 int migrate_task_to(struct task_struct *p, int target_cpu) 9377 { 9378 struct migration_arg arg = { p, target_cpu }; 9379 int curr_cpu = task_cpu(p); 9380 9381 if (curr_cpu == target_cpu) 9382 return 0; 9383 9384 if (!cpumask_test_cpu(target_cpu, p->cpus_ptr)) 9385 return -EINVAL; 9386 9387 /* TODO: This is not properly updating schedstats */ 9388 9389 trace_sched_move_numa(p, curr_cpu, target_cpu); 9390 return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg); 9391 } 9392 9393 /* 9394 * Requeue a task on a given node and accurately track the number of NUMA 9395 * tasks on the runqueues 9396 */ 9397 void sched_setnuma(struct task_struct *p, int nid) 9398 { 9399 bool queued, running; 9400 struct rq_flags rf; 9401 struct rq *rq; 9402 9403 rq = task_rq_lock(p, &rf); 9404 queued = task_on_rq_queued(p); 9405 running = task_current(rq, p); 9406 9407 if (queued) 9408 dequeue_task(rq, p, DEQUEUE_SAVE); 9409 if (running) 9410 put_prev_task(rq, p); 9411 9412 p->numa_preferred_nid = nid; 9413 9414 if (queued) 9415 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); 9416 if (running) 9417 set_next_task(rq, p); 9418 task_rq_unlock(rq, p, &rf); 9419 } 9420 #endif /* CONFIG_NUMA_BALANCING */ 9421 9422 #ifdef CONFIG_HOTPLUG_CPU 9423 /* 9424 * Ensure that the idle task is using init_mm right before its CPU goes 9425 * offline. 9426 */ 9427 void idle_task_exit(void) 9428 { 9429 struct mm_struct *mm = current->active_mm; 9430 9431 BUG_ON(cpu_online(smp_processor_id())); 9432 BUG_ON(current != this_rq()->idle); 9433 9434 if (mm != &init_mm) { 9435 switch_mm(mm, &init_mm, current); 9436 finish_arch_post_lock_switch(); 9437 } 9438 9439 /* finish_cpu(), as ran on the BP, will clean up the active_mm state */ 9440 } 9441 9442 static int __balance_push_cpu_stop(void *arg) 9443 { 9444 struct task_struct *p = arg; 9445 struct rq *rq = this_rq(); 9446 struct rq_flags rf; 9447 int cpu; 9448 9449 raw_spin_lock_irq(&p->pi_lock); 9450 rq_lock(rq, &rf); 9451 9452 update_rq_clock(rq); 9453 9454 if (task_rq(p) == rq && task_on_rq_queued(p)) { 9455 cpu = select_fallback_rq(rq->cpu, p); 9456 rq = __migrate_task(rq, &rf, p, cpu); 9457 } 9458 9459 rq_unlock(rq, &rf); 9460 raw_spin_unlock_irq(&p->pi_lock); 9461 9462 put_task_struct(p); 9463 9464 return 0; 9465 } 9466 9467 static DEFINE_PER_CPU(struct cpu_stop_work, push_work); 9468 9469 /* 9470 * Ensure we only run per-cpu kthreads once the CPU goes !active. 9471 * 9472 * This is enabled below SCHED_AP_ACTIVE; when !cpu_active(), but only 9473 * effective when the hotplug motion is down. 9474 */ 9475 static void balance_push(struct rq *rq) 9476 { 9477 struct task_struct *push_task = rq->curr; 9478 9479 lockdep_assert_rq_held(rq); 9480 9481 /* 9482 * Ensure the thing is persistent until balance_push_set(.on = false); 9483 */ 9484 rq->balance_callback = &balance_push_callback; 9485 9486 /* 9487 * Only active while going offline and when invoked on the outgoing 9488 * CPU. 9489 */ 9490 if (!cpu_dying(rq->cpu) || rq != this_rq()) 9491 return; 9492 9493 /* 9494 * Both the cpu-hotplug and stop task are in this case and are 9495 * required to complete the hotplug process. 9496 */ 9497 if (kthread_is_per_cpu(push_task) || 9498 is_migration_disabled(push_task)) { 9499 9500 /* 9501 * If this is the idle task on the outgoing CPU try to wake 9502 * up the hotplug control thread which might wait for the 9503 * last task to vanish. The rcuwait_active() check is 9504 * accurate here because the waiter is pinned on this CPU 9505 * and can't obviously be running in parallel. 9506 * 9507 * On RT kernels this also has to check whether there are 9508 * pinned and scheduled out tasks on the runqueue. They 9509 * need to leave the migrate disabled section first. 9510 */ 9511 if (!rq->nr_running && !rq_has_pinned_tasks(rq) && 9512 rcuwait_active(&rq->hotplug_wait)) { 9513 raw_spin_rq_unlock(rq); 9514 rcuwait_wake_up(&rq->hotplug_wait); 9515 raw_spin_rq_lock(rq); 9516 } 9517 return; 9518 } 9519 9520 get_task_struct(push_task); 9521 /* 9522 * Temporarily drop rq->lock such that we can wake-up the stop task. 9523 * Both preemption and IRQs are still disabled. 9524 */ 9525 raw_spin_rq_unlock(rq); 9526 stop_one_cpu_nowait(rq->cpu, __balance_push_cpu_stop, push_task, 9527 this_cpu_ptr(&push_work)); 9528 /* 9529 * At this point need_resched() is true and we'll take the loop in 9530 * schedule(). The next pick is obviously going to be the stop task 9531 * which kthread_is_per_cpu() and will push this task away. 9532 */ 9533 raw_spin_rq_lock(rq); 9534 } 9535 9536 static void balance_push_set(int cpu, bool on) 9537 { 9538 struct rq *rq = cpu_rq(cpu); 9539 struct rq_flags rf; 9540 9541 rq_lock_irqsave(rq, &rf); 9542 if (on) { 9543 WARN_ON_ONCE(rq->balance_callback); 9544 rq->balance_callback = &balance_push_callback; 9545 } else if (rq->balance_callback == &balance_push_callback) { 9546 rq->balance_callback = NULL; 9547 } 9548 rq_unlock_irqrestore(rq, &rf); 9549 } 9550 9551 /* 9552 * Invoked from a CPUs hotplug control thread after the CPU has been marked 9553 * inactive. All tasks which are not per CPU kernel threads are either 9554 * pushed off this CPU now via balance_push() or placed on a different CPU 9555 * during wakeup. Wait until the CPU is quiescent. 9556 */ 9557 static void balance_hotplug_wait(void) 9558 { 9559 struct rq *rq = this_rq(); 9560 9561 rcuwait_wait_event(&rq->hotplug_wait, 9562 rq->nr_running == 1 && !rq_has_pinned_tasks(rq), 9563 TASK_UNINTERRUPTIBLE); 9564 } 9565 9566 #else 9567 9568 static inline void balance_push(struct rq *rq) 9569 { 9570 } 9571 9572 static inline void balance_push_set(int cpu, bool on) 9573 { 9574 } 9575 9576 static inline void balance_hotplug_wait(void) 9577 { 9578 } 9579 9580 #endif /* CONFIG_HOTPLUG_CPU */ 9581 9582 void set_rq_online(struct rq *rq) 9583 { 9584 if (!rq->online) { 9585 const struct sched_class *class; 9586 9587 cpumask_set_cpu(rq->cpu, rq->rd->online); 9588 rq->online = 1; 9589 9590 for_each_class(class) { 9591 if (class->rq_online) 9592 class->rq_online(rq); 9593 } 9594 } 9595 } 9596 9597 void set_rq_offline(struct rq *rq) 9598 { 9599 if (rq->online) { 9600 const struct sched_class *class; 9601 9602 update_rq_clock(rq); 9603 for_each_class(class) { 9604 if (class->rq_offline) 9605 class->rq_offline(rq); 9606 } 9607 9608 cpumask_clear_cpu(rq->cpu, rq->rd->online); 9609 rq->online = 0; 9610 } 9611 } 9612 9613 /* 9614 * used to mark begin/end of suspend/resume: 9615 */ 9616 static int num_cpus_frozen; 9617 9618 /* 9619 * Update cpusets according to cpu_active mask. If cpusets are 9620 * disabled, cpuset_update_active_cpus() becomes a simple wrapper 9621 * around partition_sched_domains(). 9622 * 9623 * If we come here as part of a suspend/resume, don't touch cpusets because we 9624 * want to restore it back to its original state upon resume anyway. 9625 */ 9626 static void cpuset_cpu_active(void) 9627 { 9628 if (cpuhp_tasks_frozen) { 9629 /* 9630 * num_cpus_frozen tracks how many CPUs are involved in suspend 9631 * resume sequence. As long as this is not the last online 9632 * operation in the resume sequence, just build a single sched 9633 * domain, ignoring cpusets. 9634 */ 9635 partition_sched_domains(1, NULL, NULL); 9636 if (--num_cpus_frozen) 9637 return; 9638 /* 9639 * This is the last CPU online operation. So fall through and 9640 * restore the original sched domains by considering the 9641 * cpuset configurations. 9642 */ 9643 cpuset_force_rebuild(); 9644 } 9645 cpuset_update_active_cpus(); 9646 } 9647 9648 static int cpuset_cpu_inactive(unsigned int cpu) 9649 { 9650 if (!cpuhp_tasks_frozen) { 9651 int ret = dl_bw_check_overflow(cpu); 9652 9653 if (ret) 9654 return ret; 9655 cpuset_update_active_cpus(); 9656 } else { 9657 num_cpus_frozen++; 9658 partition_sched_domains(1, NULL, NULL); 9659 } 9660 return 0; 9661 } 9662 9663 int sched_cpu_activate(unsigned int cpu) 9664 { 9665 struct rq *rq = cpu_rq(cpu); 9666 struct rq_flags rf; 9667 9668 /* 9669 * Clear the balance_push callback and prepare to schedule 9670 * regular tasks. 9671 */ 9672 balance_push_set(cpu, false); 9673 9674 #ifdef CONFIG_SCHED_SMT 9675 /* 9676 * When going up, increment the number of cores with SMT present. 9677 */ 9678 if (cpumask_weight(cpu_smt_mask(cpu)) == 2) 9679 static_branch_inc_cpuslocked(&sched_smt_present); 9680 #endif 9681 set_cpu_active(cpu, true); 9682 9683 if (sched_smp_initialized) { 9684 sched_update_numa(cpu, true); 9685 sched_domains_numa_masks_set(cpu); 9686 cpuset_cpu_active(); 9687 } 9688 9689 /* 9690 * Put the rq online, if not already. This happens: 9691 * 9692 * 1) In the early boot process, because we build the real domains 9693 * after all CPUs have been brought up. 9694 * 9695 * 2) At runtime, if cpuset_cpu_active() fails to rebuild the 9696 * domains. 9697 */ 9698 rq_lock_irqsave(rq, &rf); 9699 if (rq->rd) { 9700 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); 9701 set_rq_online(rq); 9702 } 9703 rq_unlock_irqrestore(rq, &rf); 9704 9705 return 0; 9706 } 9707 9708 int sched_cpu_deactivate(unsigned int cpu) 9709 { 9710 struct rq *rq = cpu_rq(cpu); 9711 struct rq_flags rf; 9712 int ret; 9713 9714 /* 9715 * Remove CPU from nohz.idle_cpus_mask to prevent participating in 9716 * load balancing when not active 9717 */ 9718 nohz_balance_exit_idle(rq); 9719 9720 set_cpu_active(cpu, false); 9721 9722 /* 9723 * From this point forward, this CPU will refuse to run any task that 9724 * is not: migrate_disable() or KTHREAD_IS_PER_CPU, and will actively 9725 * push those tasks away until this gets cleared, see 9726 * sched_cpu_dying(). 9727 */ 9728 balance_push_set(cpu, true); 9729 9730 /* 9731 * We've cleared cpu_active_mask / set balance_push, wait for all 9732 * preempt-disabled and RCU users of this state to go away such that 9733 * all new such users will observe it. 9734 * 9735 * Specifically, we rely on ttwu to no longer target this CPU, see 9736 * ttwu_queue_cond() and is_cpu_allowed(). 9737 * 9738 * Do sync before park smpboot threads to take care the rcu boost case. 9739 */ 9740 synchronize_rcu(); 9741 9742 rq_lock_irqsave(rq, &rf); 9743 if (rq->rd) { 9744 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); 9745 set_rq_offline(rq); 9746 } 9747 rq_unlock_irqrestore(rq, &rf); 9748 9749 #ifdef CONFIG_SCHED_SMT 9750 /* 9751 * When going down, decrement the number of cores with SMT present. 9752 */ 9753 if (cpumask_weight(cpu_smt_mask(cpu)) == 2) 9754 static_branch_dec_cpuslocked(&sched_smt_present); 9755 9756 sched_core_cpu_deactivate(cpu); 9757 #endif 9758 9759 if (!sched_smp_initialized) 9760 return 0; 9761 9762 sched_update_numa(cpu, false); 9763 ret = cpuset_cpu_inactive(cpu); 9764 if (ret) { 9765 balance_push_set(cpu, false); 9766 set_cpu_active(cpu, true); 9767 sched_update_numa(cpu, true); 9768 return ret; 9769 } 9770 sched_domains_numa_masks_clear(cpu); 9771 return 0; 9772 } 9773 9774 static void sched_rq_cpu_starting(unsigned int cpu) 9775 { 9776 struct rq *rq = cpu_rq(cpu); 9777 9778 rq->calc_load_update = calc_load_update; 9779 update_max_interval(); 9780 } 9781 9782 int sched_cpu_starting(unsigned int cpu) 9783 { 9784 sched_core_cpu_starting(cpu); 9785 sched_rq_cpu_starting(cpu); 9786 sched_tick_start(cpu); 9787 return 0; 9788 } 9789 9790 #ifdef CONFIG_HOTPLUG_CPU 9791 9792 /* 9793 * Invoked immediately before the stopper thread is invoked to bring the 9794 * CPU down completely. At this point all per CPU kthreads except the 9795 * hotplug thread (current) and the stopper thread (inactive) have been 9796 * either parked or have been unbound from the outgoing CPU. Ensure that 9797 * any of those which might be on the way out are gone. 9798 * 9799 * If after this point a bound task is being woken on this CPU then the 9800 * responsible hotplug callback has failed to do it's job. 9801 * sched_cpu_dying() will catch it with the appropriate fireworks. 9802 */ 9803 int sched_cpu_wait_empty(unsigned int cpu) 9804 { 9805 balance_hotplug_wait(); 9806 return 0; 9807 } 9808 9809 /* 9810 * Since this CPU is going 'away' for a while, fold any nr_active delta we 9811 * might have. Called from the CPU stopper task after ensuring that the 9812 * stopper is the last running task on the CPU, so nr_active count is 9813 * stable. We need to take the teardown thread which is calling this into 9814 * account, so we hand in adjust = 1 to the load calculation. 9815 * 9816 * Also see the comment "Global load-average calculations". 9817 */ 9818 static void calc_load_migrate(struct rq *rq) 9819 { 9820 long delta = calc_load_fold_active(rq, 1); 9821 9822 if (delta) 9823 atomic_long_add(delta, &calc_load_tasks); 9824 } 9825 9826 static void dump_rq_tasks(struct rq *rq, const char *loglvl) 9827 { 9828 struct task_struct *g, *p; 9829 int cpu = cpu_of(rq); 9830 9831 lockdep_assert_rq_held(rq); 9832 9833 printk("%sCPU%d enqueued tasks (%u total):\n", loglvl, cpu, rq->nr_running); 9834 for_each_process_thread(g, p) { 9835 if (task_cpu(p) != cpu) 9836 continue; 9837 9838 if (!task_on_rq_queued(p)) 9839 continue; 9840 9841 printk("%s\tpid: %d, name: %s\n", loglvl, p->pid, p->comm); 9842 } 9843 } 9844 9845 int sched_cpu_dying(unsigned int cpu) 9846 { 9847 struct rq *rq = cpu_rq(cpu); 9848 struct rq_flags rf; 9849 9850 /* Handle pending wakeups and then migrate everything off */ 9851 sched_tick_stop(cpu); 9852 9853 rq_lock_irqsave(rq, &rf); 9854 if (rq->nr_running != 1 || rq_has_pinned_tasks(rq)) { 9855 WARN(true, "Dying CPU not properly vacated!"); 9856 dump_rq_tasks(rq, KERN_WARNING); 9857 } 9858 rq_unlock_irqrestore(rq, &rf); 9859 9860 calc_load_migrate(rq); 9861 update_max_interval(); 9862 hrtick_clear(rq); 9863 sched_core_cpu_dying(cpu); 9864 return 0; 9865 } 9866 #endif 9867 9868 void __init sched_init_smp(void) 9869 { 9870 sched_init_numa(NUMA_NO_NODE); 9871 9872 /* 9873 * There's no userspace yet to cause hotplug operations; hence all the 9874 * CPU masks are stable and all blatant races in the below code cannot 9875 * happen. 9876 */ 9877 mutex_lock(&sched_domains_mutex); 9878 sched_init_domains(cpu_active_mask); 9879 mutex_unlock(&sched_domains_mutex); 9880 9881 /* Move init over to a non-isolated CPU */ 9882 if (set_cpus_allowed_ptr(current, housekeeping_cpumask(HK_TYPE_DOMAIN)) < 0) 9883 BUG(); 9884 current->flags &= ~PF_NO_SETAFFINITY; 9885 sched_init_granularity(); 9886 9887 init_sched_rt_class(); 9888 init_sched_dl_class(); 9889 9890 sched_smp_initialized = true; 9891 } 9892 9893 static int __init migration_init(void) 9894 { 9895 sched_cpu_starting(smp_processor_id()); 9896 return 0; 9897 } 9898 early_initcall(migration_init); 9899 9900 #else 9901 void __init sched_init_smp(void) 9902 { 9903 sched_init_granularity(); 9904 } 9905 #endif /* CONFIG_SMP */ 9906 9907 int in_sched_functions(unsigned long addr) 9908 { 9909 return in_lock_functions(addr) || 9910 (addr >= (unsigned long)__sched_text_start 9911 && addr < (unsigned long)__sched_text_end); 9912 } 9913 9914 #ifdef CONFIG_CGROUP_SCHED 9915 /* 9916 * Default task group. 9917 * Every task in system belongs to this group at bootup. 9918 */ 9919 struct task_group root_task_group; 9920 LIST_HEAD(task_groups); 9921 9922 /* Cacheline aligned slab cache for task_group */ 9923 static struct kmem_cache *task_group_cache __read_mostly; 9924 #endif 9925 9926 void __init sched_init(void) 9927 { 9928 unsigned long ptr = 0; 9929 int i; 9930 9931 /* Make sure the linker didn't screw up */ 9932 BUG_ON(&idle_sched_class != &fair_sched_class + 1 || 9933 &fair_sched_class != &rt_sched_class + 1 || 9934 &rt_sched_class != &dl_sched_class + 1); 9935 #ifdef CONFIG_SMP 9936 BUG_ON(&dl_sched_class != &stop_sched_class + 1); 9937 #endif 9938 9939 wait_bit_init(); 9940 9941 #ifdef CONFIG_FAIR_GROUP_SCHED 9942 ptr += 2 * nr_cpu_ids * sizeof(void **); 9943 #endif 9944 #ifdef CONFIG_RT_GROUP_SCHED 9945 ptr += 2 * nr_cpu_ids * sizeof(void **); 9946 #endif 9947 if (ptr) { 9948 ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT); 9949 9950 #ifdef CONFIG_FAIR_GROUP_SCHED 9951 root_task_group.se = (struct sched_entity **)ptr; 9952 ptr += nr_cpu_ids * sizeof(void **); 9953 9954 root_task_group.cfs_rq = (struct cfs_rq **)ptr; 9955 ptr += nr_cpu_ids * sizeof(void **); 9956 9957 root_task_group.shares = ROOT_TASK_GROUP_LOAD; 9958 init_cfs_bandwidth(&root_task_group.cfs_bandwidth); 9959 #endif /* CONFIG_FAIR_GROUP_SCHED */ 9960 #ifdef CONFIG_RT_GROUP_SCHED 9961 root_task_group.rt_se = (struct sched_rt_entity **)ptr; 9962 ptr += nr_cpu_ids * sizeof(void **); 9963 9964 root_task_group.rt_rq = (struct rt_rq **)ptr; 9965 ptr += nr_cpu_ids * sizeof(void **); 9966 9967 #endif /* CONFIG_RT_GROUP_SCHED */ 9968 } 9969 9970 init_rt_bandwidth(&def_rt_bandwidth, global_rt_period(), global_rt_runtime()); 9971 9972 #ifdef CONFIG_SMP 9973 init_defrootdomain(); 9974 #endif 9975 9976 #ifdef CONFIG_RT_GROUP_SCHED 9977 init_rt_bandwidth(&root_task_group.rt_bandwidth, 9978 global_rt_period(), global_rt_runtime()); 9979 #endif /* CONFIG_RT_GROUP_SCHED */ 9980 9981 #ifdef CONFIG_CGROUP_SCHED 9982 task_group_cache = KMEM_CACHE(task_group, 0); 9983 9984 list_add(&root_task_group.list, &task_groups); 9985 INIT_LIST_HEAD(&root_task_group.children); 9986 INIT_LIST_HEAD(&root_task_group.siblings); 9987 autogroup_init(&init_task); 9988 #endif /* CONFIG_CGROUP_SCHED */ 9989 9990 for_each_possible_cpu(i) { 9991 struct rq *rq; 9992 9993 rq = cpu_rq(i); 9994 raw_spin_lock_init(&rq->__lock); 9995 rq->nr_running = 0; 9996 rq->calc_load_active = 0; 9997 rq->calc_load_update = jiffies + LOAD_FREQ; 9998 init_cfs_rq(&rq->cfs); 9999 init_rt_rq(&rq->rt); 10000 init_dl_rq(&rq->dl); 10001 #ifdef CONFIG_FAIR_GROUP_SCHED 10002 INIT_LIST_HEAD(&rq->leaf_cfs_rq_list); 10003 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 10004 /* 10005 * How much CPU bandwidth does root_task_group get? 10006 * 10007 * In case of task-groups formed thr' the cgroup filesystem, it 10008 * gets 100% of the CPU resources in the system. This overall 10009 * system CPU resource is divided among the tasks of 10010 * root_task_group and its child task-groups in a fair manner, 10011 * based on each entity's (task or task-group's) weight 10012 * (se->load.weight). 10013 * 10014 * In other words, if root_task_group has 10 tasks of weight 10015 * 1024) and two child groups A0 and A1 (of weight 1024 each), 10016 * then A0's share of the CPU resource is: 10017 * 10018 * A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33% 10019 * 10020 * We achieve this by letting root_task_group's tasks sit 10021 * directly in rq->cfs (i.e root_task_group->se[] = NULL). 10022 */ 10023 init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL); 10024 #endif /* CONFIG_FAIR_GROUP_SCHED */ 10025 10026 rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime; 10027 #ifdef CONFIG_RT_GROUP_SCHED 10028 init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL); 10029 #endif 10030 #ifdef CONFIG_SMP 10031 rq->sd = NULL; 10032 rq->rd = NULL; 10033 rq->cpu_capacity = rq->cpu_capacity_orig = SCHED_CAPACITY_SCALE; 10034 rq->balance_callback = &balance_push_callback; 10035 rq->active_balance = 0; 10036 rq->next_balance = jiffies; 10037 rq->push_cpu = 0; 10038 rq->cpu = i; 10039 rq->online = 0; 10040 rq->idle_stamp = 0; 10041 rq->avg_idle = 2*sysctl_sched_migration_cost; 10042 rq->wake_stamp = jiffies; 10043 rq->wake_avg_idle = rq->avg_idle; 10044 rq->max_idle_balance_cost = sysctl_sched_migration_cost; 10045 10046 INIT_LIST_HEAD(&rq->cfs_tasks); 10047 10048 rq_attach_root(rq, &def_root_domain); 10049 #ifdef CONFIG_NO_HZ_COMMON 10050 rq->last_blocked_load_update_tick = jiffies; 10051 atomic_set(&rq->nohz_flags, 0); 10052 10053 INIT_CSD(&rq->nohz_csd, nohz_csd_func, rq); 10054 #endif 10055 #ifdef CONFIG_HOTPLUG_CPU 10056 rcuwait_init(&rq->hotplug_wait); 10057 #endif 10058 #endif /* CONFIG_SMP */ 10059 hrtick_rq_init(rq); 10060 atomic_set(&rq->nr_iowait, 0); 10061 10062 #ifdef CONFIG_SCHED_CORE 10063 rq->core = rq; 10064 rq->core_pick = NULL; 10065 rq->core_enabled = 0; 10066 rq->core_tree = RB_ROOT; 10067 rq->core_forceidle_count = 0; 10068 rq->core_forceidle_occupation = 0; 10069 rq->core_forceidle_start = 0; 10070 10071 rq->core_cookie = 0UL; 10072 #endif 10073 zalloc_cpumask_var_node(&rq->scratch_mask, GFP_KERNEL, cpu_to_node(i)); 10074 } 10075 10076 set_load_weight(&init_task, false); 10077 10078 /* 10079 * The boot idle thread does lazy MMU switching as well: 10080 */ 10081 mmgrab_lazy_tlb(&init_mm); 10082 enter_lazy_tlb(&init_mm, current); 10083 10084 /* 10085 * The idle task doesn't need the kthread struct to function, but it 10086 * is dressed up as a per-CPU kthread and thus needs to play the part 10087 * if we want to avoid special-casing it in code that deals with per-CPU 10088 * kthreads. 10089 */ 10090 WARN_ON(!set_kthread_struct(current)); 10091 10092 /* 10093 * Make us the idle thread. Technically, schedule() should not be 10094 * called from this thread, however somewhere below it might be, 10095 * but because we are the idle thread, we just pick up running again 10096 * when this runqueue becomes "idle". 10097 */ 10098 init_idle(current, smp_processor_id()); 10099 10100 calc_load_update = jiffies + LOAD_FREQ; 10101 10102 #ifdef CONFIG_SMP 10103 idle_thread_set_boot_cpu(); 10104 balance_push_set(smp_processor_id(), false); 10105 #endif 10106 init_sched_fair_class(); 10107 10108 psi_init(); 10109 10110 init_uclamp(); 10111 10112 preempt_dynamic_init(); 10113 10114 scheduler_running = 1; 10115 } 10116 10117 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP 10118 10119 void __might_sleep(const char *file, int line) 10120 { 10121 unsigned int state = get_current_state(); 10122 /* 10123 * Blocking primitives will set (and therefore destroy) current->state, 10124 * since we will exit with TASK_RUNNING make sure we enter with it, 10125 * otherwise we will destroy state. 10126 */ 10127 WARN_ONCE(state != TASK_RUNNING && current->task_state_change, 10128 "do not call blocking ops when !TASK_RUNNING; " 10129 "state=%x set at [<%p>] %pS\n", state, 10130 (void *)current->task_state_change, 10131 (void *)current->task_state_change); 10132 10133 __might_resched(file, line, 0); 10134 } 10135 EXPORT_SYMBOL(__might_sleep); 10136 10137 static void print_preempt_disable_ip(int preempt_offset, unsigned long ip) 10138 { 10139 if (!IS_ENABLED(CONFIG_DEBUG_PREEMPT)) 10140 return; 10141 10142 if (preempt_count() == preempt_offset) 10143 return; 10144 10145 pr_err("Preemption disabled at:"); 10146 print_ip_sym(KERN_ERR, ip); 10147 } 10148 10149 static inline bool resched_offsets_ok(unsigned int offsets) 10150 { 10151 unsigned int nested = preempt_count(); 10152 10153 nested += rcu_preempt_depth() << MIGHT_RESCHED_RCU_SHIFT; 10154 10155 return nested == offsets; 10156 } 10157 10158 void __might_resched(const char *file, int line, unsigned int offsets) 10159 { 10160 /* Ratelimiting timestamp: */ 10161 static unsigned long prev_jiffy; 10162 10163 unsigned long preempt_disable_ip; 10164 10165 /* WARN_ON_ONCE() by default, no rate limit required: */ 10166 rcu_sleep_check(); 10167 10168 if ((resched_offsets_ok(offsets) && !irqs_disabled() && 10169 !is_idle_task(current) && !current->non_block_count) || 10170 system_state == SYSTEM_BOOTING || system_state > SYSTEM_RUNNING || 10171 oops_in_progress) 10172 return; 10173 10174 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) 10175 return; 10176 prev_jiffy = jiffies; 10177 10178 /* Save this before calling printk(), since that will clobber it: */ 10179 preempt_disable_ip = get_preempt_disable_ip(current); 10180 10181 pr_err("BUG: sleeping function called from invalid context at %s:%d\n", 10182 file, line); 10183 pr_err("in_atomic(): %d, irqs_disabled(): %d, non_block: %d, pid: %d, name: %s\n", 10184 in_atomic(), irqs_disabled(), current->non_block_count, 10185 current->pid, current->comm); 10186 pr_err("preempt_count: %x, expected: %x\n", preempt_count(), 10187 offsets & MIGHT_RESCHED_PREEMPT_MASK); 10188 10189 if (IS_ENABLED(CONFIG_PREEMPT_RCU)) { 10190 pr_err("RCU nest depth: %d, expected: %u\n", 10191 rcu_preempt_depth(), offsets >> MIGHT_RESCHED_RCU_SHIFT); 10192 } 10193 10194 if (task_stack_end_corrupted(current)) 10195 pr_emerg("Thread overran stack, or stack corrupted\n"); 10196 10197 debug_show_held_locks(current); 10198 if (irqs_disabled()) 10199 print_irqtrace_events(current); 10200 10201 print_preempt_disable_ip(offsets & MIGHT_RESCHED_PREEMPT_MASK, 10202 preempt_disable_ip); 10203 10204 dump_stack(); 10205 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 10206 } 10207 EXPORT_SYMBOL(__might_resched); 10208 10209 void __cant_sleep(const char *file, int line, int preempt_offset) 10210 { 10211 static unsigned long prev_jiffy; 10212 10213 if (irqs_disabled()) 10214 return; 10215 10216 if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) 10217 return; 10218 10219 if (preempt_count() > preempt_offset) 10220 return; 10221 10222 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) 10223 return; 10224 prev_jiffy = jiffies; 10225 10226 printk(KERN_ERR "BUG: assuming atomic context at %s:%d\n", file, line); 10227 printk(KERN_ERR "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n", 10228 in_atomic(), irqs_disabled(), 10229 current->pid, current->comm); 10230 10231 debug_show_held_locks(current); 10232 dump_stack(); 10233 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 10234 } 10235 EXPORT_SYMBOL_GPL(__cant_sleep); 10236 10237 #ifdef CONFIG_SMP 10238 void __cant_migrate(const char *file, int line) 10239 { 10240 static unsigned long prev_jiffy; 10241 10242 if (irqs_disabled()) 10243 return; 10244 10245 if (is_migration_disabled(current)) 10246 return; 10247 10248 if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) 10249 return; 10250 10251 if (preempt_count() > 0) 10252 return; 10253 10254 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) 10255 return; 10256 prev_jiffy = jiffies; 10257 10258 pr_err("BUG: assuming non migratable context at %s:%d\n", file, line); 10259 pr_err("in_atomic(): %d, irqs_disabled(): %d, migration_disabled() %u pid: %d, name: %s\n", 10260 in_atomic(), irqs_disabled(), is_migration_disabled(current), 10261 current->pid, current->comm); 10262 10263 debug_show_held_locks(current); 10264 dump_stack(); 10265 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 10266 } 10267 EXPORT_SYMBOL_GPL(__cant_migrate); 10268 #endif 10269 #endif 10270 10271 #ifdef CONFIG_MAGIC_SYSRQ 10272 void normalize_rt_tasks(void) 10273 { 10274 struct task_struct *g, *p; 10275 struct sched_attr attr = { 10276 .sched_policy = SCHED_NORMAL, 10277 }; 10278 10279 read_lock(&tasklist_lock); 10280 for_each_process_thread(g, p) { 10281 /* 10282 * Only normalize user tasks: 10283 */ 10284 if (p->flags & PF_KTHREAD) 10285 continue; 10286 10287 p->se.exec_start = 0; 10288 schedstat_set(p->stats.wait_start, 0); 10289 schedstat_set(p->stats.sleep_start, 0); 10290 schedstat_set(p->stats.block_start, 0); 10291 10292 if (!dl_task(p) && !rt_task(p)) { 10293 /* 10294 * Renice negative nice level userspace 10295 * tasks back to 0: 10296 */ 10297 if (task_nice(p) < 0) 10298 set_user_nice(p, 0); 10299 continue; 10300 } 10301 10302 __sched_setscheduler(p, &attr, false, false); 10303 } 10304 read_unlock(&tasklist_lock); 10305 } 10306 10307 #endif /* CONFIG_MAGIC_SYSRQ */ 10308 10309 #if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) 10310 /* 10311 * These functions are only useful for the IA64 MCA handling, or kdb. 10312 * 10313 * They can only be called when the whole system has been 10314 * stopped - every CPU needs to be quiescent, and no scheduling 10315 * activity can take place. Using them for anything else would 10316 * be a serious bug, and as a result, they aren't even visible 10317 * under any other configuration. 10318 */ 10319 10320 /** 10321 * curr_task - return the current task for a given CPU. 10322 * @cpu: the processor in question. 10323 * 10324 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! 10325 * 10326 * Return: The current task for @cpu. 10327 */ 10328 struct task_struct *curr_task(int cpu) 10329 { 10330 return cpu_curr(cpu); 10331 } 10332 10333 #endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */ 10334 10335 #ifdef CONFIG_IA64 10336 /** 10337 * ia64_set_curr_task - set the current task for a given CPU. 10338 * @cpu: the processor in question. 10339 * @p: the task pointer to set. 10340 * 10341 * Description: This function must only be used when non-maskable interrupts 10342 * are serviced on a separate stack. It allows the architecture to switch the 10343 * notion of the current task on a CPU in a non-blocking manner. This function 10344 * must be called with all CPU's synchronized, and interrupts disabled, the 10345 * and caller must save the original value of the current task (see 10346 * curr_task() above) and restore that value before reenabling interrupts and 10347 * re-starting the system. 10348 * 10349 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! 10350 */ 10351 void ia64_set_curr_task(int cpu, struct task_struct *p) 10352 { 10353 cpu_curr(cpu) = p; 10354 } 10355 10356 #endif 10357 10358 #ifdef CONFIG_CGROUP_SCHED 10359 /* task_group_lock serializes the addition/removal of task groups */ 10360 static DEFINE_SPINLOCK(task_group_lock); 10361 10362 static inline void alloc_uclamp_sched_group(struct task_group *tg, 10363 struct task_group *parent) 10364 { 10365 #ifdef CONFIG_UCLAMP_TASK_GROUP 10366 enum uclamp_id clamp_id; 10367 10368 for_each_clamp_id(clamp_id) { 10369 uclamp_se_set(&tg->uclamp_req[clamp_id], 10370 uclamp_none(clamp_id), false); 10371 tg->uclamp[clamp_id] = parent->uclamp[clamp_id]; 10372 } 10373 #endif 10374 } 10375 10376 static void sched_free_group(struct task_group *tg) 10377 { 10378 free_fair_sched_group(tg); 10379 free_rt_sched_group(tg); 10380 autogroup_free(tg); 10381 kmem_cache_free(task_group_cache, tg); 10382 } 10383 10384 static void sched_free_group_rcu(struct rcu_head *rcu) 10385 { 10386 sched_free_group(container_of(rcu, struct task_group, rcu)); 10387 } 10388 10389 static void sched_unregister_group(struct task_group *tg) 10390 { 10391 unregister_fair_sched_group(tg); 10392 unregister_rt_sched_group(tg); 10393 /* 10394 * We have to wait for yet another RCU grace period to expire, as 10395 * print_cfs_stats() might run concurrently. 10396 */ 10397 call_rcu(&tg->rcu, sched_free_group_rcu); 10398 } 10399 10400 /* allocate runqueue etc for a new task group */ 10401 struct task_group *sched_create_group(struct task_group *parent) 10402 { 10403 struct task_group *tg; 10404 10405 tg = kmem_cache_alloc(task_group_cache, GFP_KERNEL | __GFP_ZERO); 10406 if (!tg) 10407 return ERR_PTR(-ENOMEM); 10408 10409 if (!alloc_fair_sched_group(tg, parent)) 10410 goto err; 10411 10412 if (!alloc_rt_sched_group(tg, parent)) 10413 goto err; 10414 10415 alloc_uclamp_sched_group(tg, parent); 10416 10417 return tg; 10418 10419 err: 10420 sched_free_group(tg); 10421 return ERR_PTR(-ENOMEM); 10422 } 10423 10424 void sched_online_group(struct task_group *tg, struct task_group *parent) 10425 { 10426 unsigned long flags; 10427 10428 spin_lock_irqsave(&task_group_lock, flags); 10429 list_add_rcu(&tg->list, &task_groups); 10430 10431 /* Root should already exist: */ 10432 WARN_ON(!parent); 10433 10434 tg->parent = parent; 10435 INIT_LIST_HEAD(&tg->children); 10436 list_add_rcu(&tg->siblings, &parent->children); 10437 spin_unlock_irqrestore(&task_group_lock, flags); 10438 10439 online_fair_sched_group(tg); 10440 } 10441 10442 /* rcu callback to free various structures associated with a task group */ 10443 static void sched_unregister_group_rcu(struct rcu_head *rhp) 10444 { 10445 /* Now it should be safe to free those cfs_rqs: */ 10446 sched_unregister_group(container_of(rhp, struct task_group, rcu)); 10447 } 10448 10449 void sched_destroy_group(struct task_group *tg) 10450 { 10451 /* Wait for possible concurrent references to cfs_rqs complete: */ 10452 call_rcu(&tg->rcu, sched_unregister_group_rcu); 10453 } 10454 10455 void sched_release_group(struct task_group *tg) 10456 { 10457 unsigned long flags; 10458 10459 /* 10460 * Unlink first, to avoid walk_tg_tree_from() from finding us (via 10461 * sched_cfs_period_timer()). 10462 * 10463 * For this to be effective, we have to wait for all pending users of 10464 * this task group to leave their RCU critical section to ensure no new 10465 * user will see our dying task group any more. Specifically ensure 10466 * that tg_unthrottle_up() won't add decayed cfs_rq's to it. 10467 * 10468 * We therefore defer calling unregister_fair_sched_group() to 10469 * sched_unregister_group() which is guarantied to get called only after the 10470 * current RCU grace period has expired. 10471 */ 10472 spin_lock_irqsave(&task_group_lock, flags); 10473 list_del_rcu(&tg->list); 10474 list_del_rcu(&tg->siblings); 10475 spin_unlock_irqrestore(&task_group_lock, flags); 10476 } 10477 10478 static struct task_group *sched_get_task_group(struct task_struct *tsk) 10479 { 10480 struct task_group *tg; 10481 10482 /* 10483 * All callers are synchronized by task_rq_lock(); we do not use RCU 10484 * which is pointless here. Thus, we pass "true" to task_css_check() 10485 * to prevent lockdep warnings. 10486 */ 10487 tg = container_of(task_css_check(tsk, cpu_cgrp_id, true), 10488 struct task_group, css); 10489 tg = autogroup_task_group(tsk, tg); 10490 10491 return tg; 10492 } 10493 10494 static void sched_change_group(struct task_struct *tsk, struct task_group *group) 10495 { 10496 tsk->sched_task_group = group; 10497 10498 #ifdef CONFIG_FAIR_GROUP_SCHED 10499 if (tsk->sched_class->task_change_group) 10500 tsk->sched_class->task_change_group(tsk); 10501 else 10502 #endif 10503 set_task_rq(tsk, task_cpu(tsk)); 10504 } 10505 10506 /* 10507 * Change task's runqueue when it moves between groups. 10508 * 10509 * The caller of this function should have put the task in its new group by 10510 * now. This function just updates tsk->se.cfs_rq and tsk->se.parent to reflect 10511 * its new group. 10512 */ 10513 void sched_move_task(struct task_struct *tsk) 10514 { 10515 int queued, running, queue_flags = 10516 DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; 10517 struct task_group *group; 10518 struct rq_flags rf; 10519 struct rq *rq; 10520 10521 rq = task_rq_lock(tsk, &rf); 10522 /* 10523 * Esp. with SCHED_AUTOGROUP enabled it is possible to get superfluous 10524 * group changes. 10525 */ 10526 group = sched_get_task_group(tsk); 10527 if (group == tsk->sched_task_group) 10528 goto unlock; 10529 10530 update_rq_clock(rq); 10531 10532 running = task_current(rq, tsk); 10533 queued = task_on_rq_queued(tsk); 10534 10535 if (queued) 10536 dequeue_task(rq, tsk, queue_flags); 10537 if (running) 10538 put_prev_task(rq, tsk); 10539 10540 sched_change_group(tsk, group); 10541 10542 if (queued) 10543 enqueue_task(rq, tsk, queue_flags); 10544 if (running) { 10545 set_next_task(rq, tsk); 10546 /* 10547 * After changing group, the running task may have joined a 10548 * throttled one but it's still the running task. Trigger a 10549 * resched to make sure that task can still run. 10550 */ 10551 resched_curr(rq); 10552 } 10553 10554 unlock: 10555 task_rq_unlock(rq, tsk, &rf); 10556 } 10557 10558 static inline struct task_group *css_tg(struct cgroup_subsys_state *css) 10559 { 10560 return css ? container_of(css, struct task_group, css) : NULL; 10561 } 10562 10563 static struct cgroup_subsys_state * 10564 cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) 10565 { 10566 struct task_group *parent = css_tg(parent_css); 10567 struct task_group *tg; 10568 10569 if (!parent) { 10570 /* This is early initialization for the top cgroup */ 10571 return &root_task_group.css; 10572 } 10573 10574 tg = sched_create_group(parent); 10575 if (IS_ERR(tg)) 10576 return ERR_PTR(-ENOMEM); 10577 10578 return &tg->css; 10579 } 10580 10581 /* Expose task group only after completing cgroup initialization */ 10582 static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) 10583 { 10584 struct task_group *tg = css_tg(css); 10585 struct task_group *parent = css_tg(css->parent); 10586 10587 if (parent) 10588 sched_online_group(tg, parent); 10589 10590 #ifdef CONFIG_UCLAMP_TASK_GROUP 10591 /* Propagate the effective uclamp value for the new group */ 10592 mutex_lock(&uclamp_mutex); 10593 rcu_read_lock(); 10594 cpu_util_update_eff(css); 10595 rcu_read_unlock(); 10596 mutex_unlock(&uclamp_mutex); 10597 #endif 10598 10599 return 0; 10600 } 10601 10602 static void cpu_cgroup_css_released(struct cgroup_subsys_state *css) 10603 { 10604 struct task_group *tg = css_tg(css); 10605 10606 sched_release_group(tg); 10607 } 10608 10609 static void cpu_cgroup_css_free(struct cgroup_subsys_state *css) 10610 { 10611 struct task_group *tg = css_tg(css); 10612 10613 /* 10614 * Relies on the RCU grace period between css_released() and this. 10615 */ 10616 sched_unregister_group(tg); 10617 } 10618 10619 #ifdef CONFIG_RT_GROUP_SCHED 10620 static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) 10621 { 10622 struct task_struct *task; 10623 struct cgroup_subsys_state *css; 10624 10625 cgroup_taskset_for_each(task, css, tset) { 10626 if (!sched_rt_can_attach(css_tg(css), task)) 10627 return -EINVAL; 10628 } 10629 return 0; 10630 } 10631 #endif 10632 10633 static void cpu_cgroup_attach(struct cgroup_taskset *tset) 10634 { 10635 struct task_struct *task; 10636 struct cgroup_subsys_state *css; 10637 10638 cgroup_taskset_for_each(task, css, tset) 10639 sched_move_task(task); 10640 } 10641 10642 #ifdef CONFIG_UCLAMP_TASK_GROUP 10643 static void cpu_util_update_eff(struct cgroup_subsys_state *css) 10644 { 10645 struct cgroup_subsys_state *top_css = css; 10646 struct uclamp_se *uc_parent = NULL; 10647 struct uclamp_se *uc_se = NULL; 10648 unsigned int eff[UCLAMP_CNT]; 10649 enum uclamp_id clamp_id; 10650 unsigned int clamps; 10651 10652 lockdep_assert_held(&uclamp_mutex); 10653 SCHED_WARN_ON(!rcu_read_lock_held()); 10654 10655 css_for_each_descendant_pre(css, top_css) { 10656 uc_parent = css_tg(css)->parent 10657 ? css_tg(css)->parent->uclamp : NULL; 10658 10659 for_each_clamp_id(clamp_id) { 10660 /* Assume effective clamps matches requested clamps */ 10661 eff[clamp_id] = css_tg(css)->uclamp_req[clamp_id].value; 10662 /* Cap effective clamps with parent's effective clamps */ 10663 if (uc_parent && 10664 eff[clamp_id] > uc_parent[clamp_id].value) { 10665 eff[clamp_id] = uc_parent[clamp_id].value; 10666 } 10667 } 10668 /* Ensure protection is always capped by limit */ 10669 eff[UCLAMP_MIN] = min(eff[UCLAMP_MIN], eff[UCLAMP_MAX]); 10670 10671 /* Propagate most restrictive effective clamps */ 10672 clamps = 0x0; 10673 uc_se = css_tg(css)->uclamp; 10674 for_each_clamp_id(clamp_id) { 10675 if (eff[clamp_id] == uc_se[clamp_id].value) 10676 continue; 10677 uc_se[clamp_id].value = eff[clamp_id]; 10678 uc_se[clamp_id].bucket_id = uclamp_bucket_id(eff[clamp_id]); 10679 clamps |= (0x1 << clamp_id); 10680 } 10681 if (!clamps) { 10682 css = css_rightmost_descendant(css); 10683 continue; 10684 } 10685 10686 /* Immediately update descendants RUNNABLE tasks */ 10687 uclamp_update_active_tasks(css); 10688 } 10689 } 10690 10691 /* 10692 * Integer 10^N with a given N exponent by casting to integer the literal "1eN" 10693 * C expression. Since there is no way to convert a macro argument (N) into a 10694 * character constant, use two levels of macros. 10695 */ 10696 #define _POW10(exp) ((unsigned int)1e##exp) 10697 #define POW10(exp) _POW10(exp) 10698 10699 struct uclamp_request { 10700 #define UCLAMP_PERCENT_SHIFT 2 10701 #define UCLAMP_PERCENT_SCALE (100 * POW10(UCLAMP_PERCENT_SHIFT)) 10702 s64 percent; 10703 u64 util; 10704 int ret; 10705 }; 10706 10707 static inline struct uclamp_request 10708 capacity_from_percent(char *buf) 10709 { 10710 struct uclamp_request req = { 10711 .percent = UCLAMP_PERCENT_SCALE, 10712 .util = SCHED_CAPACITY_SCALE, 10713 .ret = 0, 10714 }; 10715 10716 buf = strim(buf); 10717 if (strcmp(buf, "max")) { 10718 req.ret = cgroup_parse_float(buf, UCLAMP_PERCENT_SHIFT, 10719 &req.percent); 10720 if (req.ret) 10721 return req; 10722 if ((u64)req.percent > UCLAMP_PERCENT_SCALE) { 10723 req.ret = -ERANGE; 10724 return req; 10725 } 10726 10727 req.util = req.percent << SCHED_CAPACITY_SHIFT; 10728 req.util = DIV_ROUND_CLOSEST_ULL(req.util, UCLAMP_PERCENT_SCALE); 10729 } 10730 10731 return req; 10732 } 10733 10734 static ssize_t cpu_uclamp_write(struct kernfs_open_file *of, char *buf, 10735 size_t nbytes, loff_t off, 10736 enum uclamp_id clamp_id) 10737 { 10738 struct uclamp_request req; 10739 struct task_group *tg; 10740 10741 req = capacity_from_percent(buf); 10742 if (req.ret) 10743 return req.ret; 10744 10745 static_branch_enable(&sched_uclamp_used); 10746 10747 mutex_lock(&uclamp_mutex); 10748 rcu_read_lock(); 10749 10750 tg = css_tg(of_css(of)); 10751 if (tg->uclamp_req[clamp_id].value != req.util) 10752 uclamp_se_set(&tg->uclamp_req[clamp_id], req.util, false); 10753 10754 /* 10755 * Because of not recoverable conversion rounding we keep track of the 10756 * exact requested value 10757 */ 10758 tg->uclamp_pct[clamp_id] = req.percent; 10759 10760 /* Update effective clamps to track the most restrictive value */ 10761 cpu_util_update_eff(of_css(of)); 10762 10763 rcu_read_unlock(); 10764 mutex_unlock(&uclamp_mutex); 10765 10766 return nbytes; 10767 } 10768 10769 static ssize_t cpu_uclamp_min_write(struct kernfs_open_file *of, 10770 char *buf, size_t nbytes, 10771 loff_t off) 10772 { 10773 return cpu_uclamp_write(of, buf, nbytes, off, UCLAMP_MIN); 10774 } 10775 10776 static ssize_t cpu_uclamp_max_write(struct kernfs_open_file *of, 10777 char *buf, size_t nbytes, 10778 loff_t off) 10779 { 10780 return cpu_uclamp_write(of, buf, nbytes, off, UCLAMP_MAX); 10781 } 10782 10783 static inline void cpu_uclamp_print(struct seq_file *sf, 10784 enum uclamp_id clamp_id) 10785 { 10786 struct task_group *tg; 10787 u64 util_clamp; 10788 u64 percent; 10789 u32 rem; 10790 10791 rcu_read_lock(); 10792 tg = css_tg(seq_css(sf)); 10793 util_clamp = tg->uclamp_req[clamp_id].value; 10794 rcu_read_unlock(); 10795 10796 if (util_clamp == SCHED_CAPACITY_SCALE) { 10797 seq_puts(sf, "max\n"); 10798 return; 10799 } 10800 10801 percent = tg->uclamp_pct[clamp_id]; 10802 percent = div_u64_rem(percent, POW10(UCLAMP_PERCENT_SHIFT), &rem); 10803 seq_printf(sf, "%llu.%0*u\n", percent, UCLAMP_PERCENT_SHIFT, rem); 10804 } 10805 10806 static int cpu_uclamp_min_show(struct seq_file *sf, void *v) 10807 { 10808 cpu_uclamp_print(sf, UCLAMP_MIN); 10809 return 0; 10810 } 10811 10812 static int cpu_uclamp_max_show(struct seq_file *sf, void *v) 10813 { 10814 cpu_uclamp_print(sf, UCLAMP_MAX); 10815 return 0; 10816 } 10817 #endif /* CONFIG_UCLAMP_TASK_GROUP */ 10818 10819 #ifdef CONFIG_FAIR_GROUP_SCHED 10820 static int cpu_shares_write_u64(struct cgroup_subsys_state *css, 10821 struct cftype *cftype, u64 shareval) 10822 { 10823 if (shareval > scale_load_down(ULONG_MAX)) 10824 shareval = MAX_SHARES; 10825 return sched_group_set_shares(css_tg(css), scale_load(shareval)); 10826 } 10827 10828 static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css, 10829 struct cftype *cft) 10830 { 10831 struct task_group *tg = css_tg(css); 10832 10833 return (u64) scale_load_down(tg->shares); 10834 } 10835 10836 #ifdef CONFIG_CFS_BANDWIDTH 10837 static DEFINE_MUTEX(cfs_constraints_mutex); 10838 10839 const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */ 10840 static const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */ 10841 /* More than 203 days if BW_SHIFT equals 20. */ 10842 static const u64 max_cfs_runtime = MAX_BW * NSEC_PER_USEC; 10843 10844 static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime); 10845 10846 static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota, 10847 u64 burst) 10848 { 10849 int i, ret = 0, runtime_enabled, runtime_was_enabled; 10850 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 10851 10852 if (tg == &root_task_group) 10853 return -EINVAL; 10854 10855 /* 10856 * Ensure we have at some amount of bandwidth every period. This is 10857 * to prevent reaching a state of large arrears when throttled via 10858 * entity_tick() resulting in prolonged exit starvation. 10859 */ 10860 if (quota < min_cfs_quota_period || period < min_cfs_quota_period) 10861 return -EINVAL; 10862 10863 /* 10864 * Likewise, bound things on the other side by preventing insane quota 10865 * periods. This also allows us to normalize in computing quota 10866 * feasibility. 10867 */ 10868 if (period > max_cfs_quota_period) 10869 return -EINVAL; 10870 10871 /* 10872 * Bound quota to defend quota against overflow during bandwidth shift. 10873 */ 10874 if (quota != RUNTIME_INF && quota > max_cfs_runtime) 10875 return -EINVAL; 10876 10877 if (quota != RUNTIME_INF && (burst > quota || 10878 burst + quota > max_cfs_runtime)) 10879 return -EINVAL; 10880 10881 /* 10882 * Prevent race between setting of cfs_rq->runtime_enabled and 10883 * unthrottle_offline_cfs_rqs(). 10884 */ 10885 cpus_read_lock(); 10886 mutex_lock(&cfs_constraints_mutex); 10887 ret = __cfs_schedulable(tg, period, quota); 10888 if (ret) 10889 goto out_unlock; 10890 10891 runtime_enabled = quota != RUNTIME_INF; 10892 runtime_was_enabled = cfs_b->quota != RUNTIME_INF; 10893 /* 10894 * If we need to toggle cfs_bandwidth_used, off->on must occur 10895 * before making related changes, and on->off must occur afterwards 10896 */ 10897 if (runtime_enabled && !runtime_was_enabled) 10898 cfs_bandwidth_usage_inc(); 10899 raw_spin_lock_irq(&cfs_b->lock); 10900 cfs_b->period = ns_to_ktime(period); 10901 cfs_b->quota = quota; 10902 cfs_b->burst = burst; 10903 10904 __refill_cfs_bandwidth_runtime(cfs_b); 10905 10906 /* Restart the period timer (if active) to handle new period expiry: */ 10907 if (runtime_enabled) 10908 start_cfs_bandwidth(cfs_b); 10909 10910 raw_spin_unlock_irq(&cfs_b->lock); 10911 10912 for_each_online_cpu(i) { 10913 struct cfs_rq *cfs_rq = tg->cfs_rq[i]; 10914 struct rq *rq = cfs_rq->rq; 10915 struct rq_flags rf; 10916 10917 rq_lock_irq(rq, &rf); 10918 cfs_rq->runtime_enabled = runtime_enabled; 10919 cfs_rq->runtime_remaining = 0; 10920 10921 if (cfs_rq->throttled) 10922 unthrottle_cfs_rq(cfs_rq); 10923 rq_unlock_irq(rq, &rf); 10924 } 10925 if (runtime_was_enabled && !runtime_enabled) 10926 cfs_bandwidth_usage_dec(); 10927 out_unlock: 10928 mutex_unlock(&cfs_constraints_mutex); 10929 cpus_read_unlock(); 10930 10931 return ret; 10932 } 10933 10934 static int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us) 10935 { 10936 u64 quota, period, burst; 10937 10938 period = ktime_to_ns(tg->cfs_bandwidth.period); 10939 burst = tg->cfs_bandwidth.burst; 10940 if (cfs_quota_us < 0) 10941 quota = RUNTIME_INF; 10942 else if ((u64)cfs_quota_us <= U64_MAX / NSEC_PER_USEC) 10943 quota = (u64)cfs_quota_us * NSEC_PER_USEC; 10944 else 10945 return -EINVAL; 10946 10947 return tg_set_cfs_bandwidth(tg, period, quota, burst); 10948 } 10949 10950 static long tg_get_cfs_quota(struct task_group *tg) 10951 { 10952 u64 quota_us; 10953 10954 if (tg->cfs_bandwidth.quota == RUNTIME_INF) 10955 return -1; 10956 10957 quota_us = tg->cfs_bandwidth.quota; 10958 do_div(quota_us, NSEC_PER_USEC); 10959 10960 return quota_us; 10961 } 10962 10963 static int tg_set_cfs_period(struct task_group *tg, long cfs_period_us) 10964 { 10965 u64 quota, period, burst; 10966 10967 if ((u64)cfs_period_us > U64_MAX / NSEC_PER_USEC) 10968 return -EINVAL; 10969 10970 period = (u64)cfs_period_us * NSEC_PER_USEC; 10971 quota = tg->cfs_bandwidth.quota; 10972 burst = tg->cfs_bandwidth.burst; 10973 10974 return tg_set_cfs_bandwidth(tg, period, quota, burst); 10975 } 10976 10977 static long tg_get_cfs_period(struct task_group *tg) 10978 { 10979 u64 cfs_period_us; 10980 10981 cfs_period_us = ktime_to_ns(tg->cfs_bandwidth.period); 10982 do_div(cfs_period_us, NSEC_PER_USEC); 10983 10984 return cfs_period_us; 10985 } 10986 10987 static int tg_set_cfs_burst(struct task_group *tg, long cfs_burst_us) 10988 { 10989 u64 quota, period, burst; 10990 10991 if ((u64)cfs_burst_us > U64_MAX / NSEC_PER_USEC) 10992 return -EINVAL; 10993 10994 burst = (u64)cfs_burst_us * NSEC_PER_USEC; 10995 period = ktime_to_ns(tg->cfs_bandwidth.period); 10996 quota = tg->cfs_bandwidth.quota; 10997 10998 return tg_set_cfs_bandwidth(tg, period, quota, burst); 10999 } 11000 11001 static long tg_get_cfs_burst(struct task_group *tg) 11002 { 11003 u64 burst_us; 11004 11005 burst_us = tg->cfs_bandwidth.burst; 11006 do_div(burst_us, NSEC_PER_USEC); 11007 11008 return burst_us; 11009 } 11010 11011 static s64 cpu_cfs_quota_read_s64(struct cgroup_subsys_state *css, 11012 struct cftype *cft) 11013 { 11014 return tg_get_cfs_quota(css_tg(css)); 11015 } 11016 11017 static int cpu_cfs_quota_write_s64(struct cgroup_subsys_state *css, 11018 struct cftype *cftype, s64 cfs_quota_us) 11019 { 11020 return tg_set_cfs_quota(css_tg(css), cfs_quota_us); 11021 } 11022 11023 static u64 cpu_cfs_period_read_u64(struct cgroup_subsys_state *css, 11024 struct cftype *cft) 11025 { 11026 return tg_get_cfs_period(css_tg(css)); 11027 } 11028 11029 static int cpu_cfs_period_write_u64(struct cgroup_subsys_state *css, 11030 struct cftype *cftype, u64 cfs_period_us) 11031 { 11032 return tg_set_cfs_period(css_tg(css), cfs_period_us); 11033 } 11034 11035 static u64 cpu_cfs_burst_read_u64(struct cgroup_subsys_state *css, 11036 struct cftype *cft) 11037 { 11038 return tg_get_cfs_burst(css_tg(css)); 11039 } 11040 11041 static int cpu_cfs_burst_write_u64(struct cgroup_subsys_state *css, 11042 struct cftype *cftype, u64 cfs_burst_us) 11043 { 11044 return tg_set_cfs_burst(css_tg(css), cfs_burst_us); 11045 } 11046 11047 struct cfs_schedulable_data { 11048 struct task_group *tg; 11049 u64 period, quota; 11050 }; 11051 11052 /* 11053 * normalize group quota/period to be quota/max_period 11054 * note: units are usecs 11055 */ 11056 static u64 normalize_cfs_quota(struct task_group *tg, 11057 struct cfs_schedulable_data *d) 11058 { 11059 u64 quota, period; 11060 11061 if (tg == d->tg) { 11062 period = d->period; 11063 quota = d->quota; 11064 } else { 11065 period = tg_get_cfs_period(tg); 11066 quota = tg_get_cfs_quota(tg); 11067 } 11068 11069 /* note: these should typically be equivalent */ 11070 if (quota == RUNTIME_INF || quota == -1) 11071 return RUNTIME_INF; 11072 11073 return to_ratio(period, quota); 11074 } 11075 11076 static int tg_cfs_schedulable_down(struct task_group *tg, void *data) 11077 { 11078 struct cfs_schedulable_data *d = data; 11079 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 11080 s64 quota = 0, parent_quota = -1; 11081 11082 if (!tg->parent) { 11083 quota = RUNTIME_INF; 11084 } else { 11085 struct cfs_bandwidth *parent_b = &tg->parent->cfs_bandwidth; 11086 11087 quota = normalize_cfs_quota(tg, d); 11088 parent_quota = parent_b->hierarchical_quota; 11089 11090 /* 11091 * Ensure max(child_quota) <= parent_quota. On cgroup2, 11092 * always take the min. On cgroup1, only inherit when no 11093 * limit is set: 11094 */ 11095 if (cgroup_subsys_on_dfl(cpu_cgrp_subsys)) { 11096 quota = min(quota, parent_quota); 11097 } else { 11098 if (quota == RUNTIME_INF) 11099 quota = parent_quota; 11100 else if (parent_quota != RUNTIME_INF && quota > parent_quota) 11101 return -EINVAL; 11102 } 11103 } 11104 cfs_b->hierarchical_quota = quota; 11105 11106 return 0; 11107 } 11108 11109 static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota) 11110 { 11111 int ret; 11112 struct cfs_schedulable_data data = { 11113 .tg = tg, 11114 .period = period, 11115 .quota = quota, 11116 }; 11117 11118 if (quota != RUNTIME_INF) { 11119 do_div(data.period, NSEC_PER_USEC); 11120 do_div(data.quota, NSEC_PER_USEC); 11121 } 11122 11123 rcu_read_lock(); 11124 ret = walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data); 11125 rcu_read_unlock(); 11126 11127 return ret; 11128 } 11129 11130 static int cpu_cfs_stat_show(struct seq_file *sf, void *v) 11131 { 11132 struct task_group *tg = css_tg(seq_css(sf)); 11133 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 11134 11135 seq_printf(sf, "nr_periods %d\n", cfs_b->nr_periods); 11136 seq_printf(sf, "nr_throttled %d\n", cfs_b->nr_throttled); 11137 seq_printf(sf, "throttled_time %llu\n", cfs_b->throttled_time); 11138 11139 if (schedstat_enabled() && tg != &root_task_group) { 11140 struct sched_statistics *stats; 11141 u64 ws = 0; 11142 int i; 11143 11144 for_each_possible_cpu(i) { 11145 stats = __schedstats_from_se(tg->se[i]); 11146 ws += schedstat_val(stats->wait_sum); 11147 } 11148 11149 seq_printf(sf, "wait_sum %llu\n", ws); 11150 } 11151 11152 seq_printf(sf, "nr_bursts %d\n", cfs_b->nr_burst); 11153 seq_printf(sf, "burst_time %llu\n", cfs_b->burst_time); 11154 11155 return 0; 11156 } 11157 11158 static u64 throttled_time_self(struct task_group *tg) 11159 { 11160 int i; 11161 u64 total = 0; 11162 11163 for_each_possible_cpu(i) { 11164 total += READ_ONCE(tg->cfs_rq[i]->throttled_clock_self_time); 11165 } 11166 11167 return total; 11168 } 11169 11170 static int cpu_cfs_local_stat_show(struct seq_file *sf, void *v) 11171 { 11172 struct task_group *tg = css_tg(seq_css(sf)); 11173 11174 seq_printf(sf, "throttled_time %llu\n", throttled_time_self(tg)); 11175 11176 return 0; 11177 } 11178 #endif /* CONFIG_CFS_BANDWIDTH */ 11179 #endif /* CONFIG_FAIR_GROUP_SCHED */ 11180 11181 #ifdef CONFIG_RT_GROUP_SCHED 11182 static int cpu_rt_runtime_write(struct cgroup_subsys_state *css, 11183 struct cftype *cft, s64 val) 11184 { 11185 return sched_group_set_rt_runtime(css_tg(css), val); 11186 } 11187 11188 static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css, 11189 struct cftype *cft) 11190 { 11191 return sched_group_rt_runtime(css_tg(css)); 11192 } 11193 11194 static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css, 11195 struct cftype *cftype, u64 rt_period_us) 11196 { 11197 return sched_group_set_rt_period(css_tg(css), rt_period_us); 11198 } 11199 11200 static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css, 11201 struct cftype *cft) 11202 { 11203 return sched_group_rt_period(css_tg(css)); 11204 } 11205 #endif /* CONFIG_RT_GROUP_SCHED */ 11206 11207 #ifdef CONFIG_FAIR_GROUP_SCHED 11208 static s64 cpu_idle_read_s64(struct cgroup_subsys_state *css, 11209 struct cftype *cft) 11210 { 11211 return css_tg(css)->idle; 11212 } 11213 11214 static int cpu_idle_write_s64(struct cgroup_subsys_state *css, 11215 struct cftype *cft, s64 idle) 11216 { 11217 return sched_group_set_idle(css_tg(css), idle); 11218 } 11219 #endif 11220 11221 static struct cftype cpu_legacy_files[] = { 11222 #ifdef CONFIG_FAIR_GROUP_SCHED 11223 { 11224 .name = "shares", 11225 .read_u64 = cpu_shares_read_u64, 11226 .write_u64 = cpu_shares_write_u64, 11227 }, 11228 { 11229 .name = "idle", 11230 .read_s64 = cpu_idle_read_s64, 11231 .write_s64 = cpu_idle_write_s64, 11232 }, 11233 #endif 11234 #ifdef CONFIG_CFS_BANDWIDTH 11235 { 11236 .name = "cfs_quota_us", 11237 .read_s64 = cpu_cfs_quota_read_s64, 11238 .write_s64 = cpu_cfs_quota_write_s64, 11239 }, 11240 { 11241 .name = "cfs_period_us", 11242 .read_u64 = cpu_cfs_period_read_u64, 11243 .write_u64 = cpu_cfs_period_write_u64, 11244 }, 11245 { 11246 .name = "cfs_burst_us", 11247 .read_u64 = cpu_cfs_burst_read_u64, 11248 .write_u64 = cpu_cfs_burst_write_u64, 11249 }, 11250 { 11251 .name = "stat", 11252 .seq_show = cpu_cfs_stat_show, 11253 }, 11254 { 11255 .name = "stat.local", 11256 .seq_show = cpu_cfs_local_stat_show, 11257 }, 11258 #endif 11259 #ifdef CONFIG_RT_GROUP_SCHED 11260 { 11261 .name = "rt_runtime_us", 11262 .read_s64 = cpu_rt_runtime_read, 11263 .write_s64 = cpu_rt_runtime_write, 11264 }, 11265 { 11266 .name = "rt_period_us", 11267 .read_u64 = cpu_rt_period_read_uint, 11268 .write_u64 = cpu_rt_period_write_uint, 11269 }, 11270 #endif 11271 #ifdef CONFIG_UCLAMP_TASK_GROUP 11272 { 11273 .name = "uclamp.min", 11274 .flags = CFTYPE_NOT_ON_ROOT, 11275 .seq_show = cpu_uclamp_min_show, 11276 .write = cpu_uclamp_min_write, 11277 }, 11278 { 11279 .name = "uclamp.max", 11280 .flags = CFTYPE_NOT_ON_ROOT, 11281 .seq_show = cpu_uclamp_max_show, 11282 .write = cpu_uclamp_max_write, 11283 }, 11284 #endif 11285 { } /* Terminate */ 11286 }; 11287 11288 static int cpu_extra_stat_show(struct seq_file *sf, 11289 struct cgroup_subsys_state *css) 11290 { 11291 #ifdef CONFIG_CFS_BANDWIDTH 11292 { 11293 struct task_group *tg = css_tg(css); 11294 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 11295 u64 throttled_usec, burst_usec; 11296 11297 throttled_usec = cfs_b->throttled_time; 11298 do_div(throttled_usec, NSEC_PER_USEC); 11299 burst_usec = cfs_b->burst_time; 11300 do_div(burst_usec, NSEC_PER_USEC); 11301 11302 seq_printf(sf, "nr_periods %d\n" 11303 "nr_throttled %d\n" 11304 "throttled_usec %llu\n" 11305 "nr_bursts %d\n" 11306 "burst_usec %llu\n", 11307 cfs_b->nr_periods, cfs_b->nr_throttled, 11308 throttled_usec, cfs_b->nr_burst, burst_usec); 11309 } 11310 #endif 11311 return 0; 11312 } 11313 11314 static int cpu_local_stat_show(struct seq_file *sf, 11315 struct cgroup_subsys_state *css) 11316 { 11317 #ifdef CONFIG_CFS_BANDWIDTH 11318 { 11319 struct task_group *tg = css_tg(css); 11320 u64 throttled_self_usec; 11321 11322 throttled_self_usec = throttled_time_self(tg); 11323 do_div(throttled_self_usec, NSEC_PER_USEC); 11324 11325 seq_printf(sf, "throttled_usec %llu\n", 11326 throttled_self_usec); 11327 } 11328 #endif 11329 return 0; 11330 } 11331 11332 #ifdef CONFIG_FAIR_GROUP_SCHED 11333 static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css, 11334 struct cftype *cft) 11335 { 11336 struct task_group *tg = css_tg(css); 11337 u64 weight = scale_load_down(tg->shares); 11338 11339 return DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024); 11340 } 11341 11342 static int cpu_weight_write_u64(struct cgroup_subsys_state *css, 11343 struct cftype *cft, u64 weight) 11344 { 11345 /* 11346 * cgroup weight knobs should use the common MIN, DFL and MAX 11347 * values which are 1, 100 and 10000 respectively. While it loses 11348 * a bit of range on both ends, it maps pretty well onto the shares 11349 * value used by scheduler and the round-trip conversions preserve 11350 * the original value over the entire range. 11351 */ 11352 if (weight < CGROUP_WEIGHT_MIN || weight > CGROUP_WEIGHT_MAX) 11353 return -ERANGE; 11354 11355 weight = DIV_ROUND_CLOSEST_ULL(weight * 1024, CGROUP_WEIGHT_DFL); 11356 11357 return sched_group_set_shares(css_tg(css), scale_load(weight)); 11358 } 11359 11360 static s64 cpu_weight_nice_read_s64(struct cgroup_subsys_state *css, 11361 struct cftype *cft) 11362 { 11363 unsigned long weight = scale_load_down(css_tg(css)->shares); 11364 int last_delta = INT_MAX; 11365 int prio, delta; 11366 11367 /* find the closest nice value to the current weight */ 11368 for (prio = 0; prio < ARRAY_SIZE(sched_prio_to_weight); prio++) { 11369 delta = abs(sched_prio_to_weight[prio] - weight); 11370 if (delta >= last_delta) 11371 break; 11372 last_delta = delta; 11373 } 11374 11375 return PRIO_TO_NICE(prio - 1 + MAX_RT_PRIO); 11376 } 11377 11378 static int cpu_weight_nice_write_s64(struct cgroup_subsys_state *css, 11379 struct cftype *cft, s64 nice) 11380 { 11381 unsigned long weight; 11382 int idx; 11383 11384 if (nice < MIN_NICE || nice > MAX_NICE) 11385 return -ERANGE; 11386 11387 idx = NICE_TO_PRIO(nice) - MAX_RT_PRIO; 11388 idx = array_index_nospec(idx, 40); 11389 weight = sched_prio_to_weight[idx]; 11390 11391 return sched_group_set_shares(css_tg(css), scale_load(weight)); 11392 } 11393 #endif 11394 11395 static void __maybe_unused cpu_period_quota_print(struct seq_file *sf, 11396 long period, long quota) 11397 { 11398 if (quota < 0) 11399 seq_puts(sf, "max"); 11400 else 11401 seq_printf(sf, "%ld", quota); 11402 11403 seq_printf(sf, " %ld\n", period); 11404 } 11405 11406 /* caller should put the current value in *@periodp before calling */ 11407 static int __maybe_unused cpu_period_quota_parse(char *buf, 11408 u64 *periodp, u64 *quotap) 11409 { 11410 char tok[21]; /* U64_MAX */ 11411 11412 if (sscanf(buf, "%20s %llu", tok, periodp) < 1) 11413 return -EINVAL; 11414 11415 *periodp *= NSEC_PER_USEC; 11416 11417 if (sscanf(tok, "%llu", quotap)) 11418 *quotap *= NSEC_PER_USEC; 11419 else if (!strcmp(tok, "max")) 11420 *quotap = RUNTIME_INF; 11421 else 11422 return -EINVAL; 11423 11424 return 0; 11425 } 11426 11427 #ifdef CONFIG_CFS_BANDWIDTH 11428 static int cpu_max_show(struct seq_file *sf, void *v) 11429 { 11430 struct task_group *tg = css_tg(seq_css(sf)); 11431 11432 cpu_period_quota_print(sf, tg_get_cfs_period(tg), tg_get_cfs_quota(tg)); 11433 return 0; 11434 } 11435 11436 static ssize_t cpu_max_write(struct kernfs_open_file *of, 11437 char *buf, size_t nbytes, loff_t off) 11438 { 11439 struct task_group *tg = css_tg(of_css(of)); 11440 u64 period = tg_get_cfs_period(tg); 11441 u64 burst = tg_get_cfs_burst(tg); 11442 u64 quota; 11443 int ret; 11444 11445 ret = cpu_period_quota_parse(buf, &period, "a); 11446 if (!ret) 11447 ret = tg_set_cfs_bandwidth(tg, period, quota, burst); 11448 return ret ?: nbytes; 11449 } 11450 #endif 11451 11452 static struct cftype cpu_files[] = { 11453 #ifdef CONFIG_FAIR_GROUP_SCHED 11454 { 11455 .name = "weight", 11456 .flags = CFTYPE_NOT_ON_ROOT, 11457 .read_u64 = cpu_weight_read_u64, 11458 .write_u64 = cpu_weight_write_u64, 11459 }, 11460 { 11461 .name = "weight.nice", 11462 .flags = CFTYPE_NOT_ON_ROOT, 11463 .read_s64 = cpu_weight_nice_read_s64, 11464 .write_s64 = cpu_weight_nice_write_s64, 11465 }, 11466 { 11467 .name = "idle", 11468 .flags = CFTYPE_NOT_ON_ROOT, 11469 .read_s64 = cpu_idle_read_s64, 11470 .write_s64 = cpu_idle_write_s64, 11471 }, 11472 #endif 11473 #ifdef CONFIG_CFS_BANDWIDTH 11474 { 11475 .name = "max", 11476 .flags = CFTYPE_NOT_ON_ROOT, 11477 .seq_show = cpu_max_show, 11478 .write = cpu_max_write, 11479 }, 11480 { 11481 .name = "max.burst", 11482 .flags = CFTYPE_NOT_ON_ROOT, 11483 .read_u64 = cpu_cfs_burst_read_u64, 11484 .write_u64 = cpu_cfs_burst_write_u64, 11485 }, 11486 #endif 11487 #ifdef CONFIG_UCLAMP_TASK_GROUP 11488 { 11489 .name = "uclamp.min", 11490 .flags = CFTYPE_NOT_ON_ROOT, 11491 .seq_show = cpu_uclamp_min_show, 11492 .write = cpu_uclamp_min_write, 11493 }, 11494 { 11495 .name = "uclamp.max", 11496 .flags = CFTYPE_NOT_ON_ROOT, 11497 .seq_show = cpu_uclamp_max_show, 11498 .write = cpu_uclamp_max_write, 11499 }, 11500 #endif 11501 { } /* terminate */ 11502 }; 11503 11504 struct cgroup_subsys cpu_cgrp_subsys = { 11505 .css_alloc = cpu_cgroup_css_alloc, 11506 .css_online = cpu_cgroup_css_online, 11507 .css_released = cpu_cgroup_css_released, 11508 .css_free = cpu_cgroup_css_free, 11509 .css_extra_stat_show = cpu_extra_stat_show, 11510 .css_local_stat_show = cpu_local_stat_show, 11511 #ifdef CONFIG_RT_GROUP_SCHED 11512 .can_attach = cpu_cgroup_can_attach, 11513 #endif 11514 .attach = cpu_cgroup_attach, 11515 .legacy_cftypes = cpu_legacy_files, 11516 .dfl_cftypes = cpu_files, 11517 .early_init = true, 11518 .threaded = true, 11519 }; 11520 11521 #endif /* CONFIG_CGROUP_SCHED */ 11522 11523 void dump_cpu_task(int cpu) 11524 { 11525 if (cpu == smp_processor_id() && in_hardirq()) { 11526 struct pt_regs *regs; 11527 11528 regs = get_irq_regs(); 11529 if (regs) { 11530 show_regs(regs); 11531 return; 11532 } 11533 } 11534 11535 if (trigger_single_cpu_backtrace(cpu)) 11536 return; 11537 11538 pr_info("Task dump for CPU %d:\n", cpu); 11539 sched_show_task(cpu_curr(cpu)); 11540 } 11541 11542 /* 11543 * Nice levels are multiplicative, with a gentle 10% change for every 11544 * nice level changed. I.e. when a CPU-bound task goes from nice 0 to 11545 * nice 1, it will get ~10% less CPU time than another CPU-bound task 11546 * that remained on nice 0. 11547 * 11548 * The "10% effect" is relative and cumulative: from _any_ nice level, 11549 * if you go up 1 level, it's -10% CPU usage, if you go down 1 level 11550 * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25. 11551 * If a task goes up by ~10% and another task goes down by ~10% then 11552 * the relative distance between them is ~25%.) 11553 */ 11554 const int sched_prio_to_weight[40] = { 11555 /* -20 */ 88761, 71755, 56483, 46273, 36291, 11556 /* -15 */ 29154, 23254, 18705, 14949, 11916, 11557 /* -10 */ 9548, 7620, 6100, 4904, 3906, 11558 /* -5 */ 3121, 2501, 1991, 1586, 1277, 11559 /* 0 */ 1024, 820, 655, 526, 423, 11560 /* 5 */ 335, 272, 215, 172, 137, 11561 /* 10 */ 110, 87, 70, 56, 45, 11562 /* 15 */ 36, 29, 23, 18, 15, 11563 }; 11564 11565 /* 11566 * Inverse (2^32/x) values of the sched_prio_to_weight[] array, precalculated. 11567 * 11568 * In cases where the weight does not change often, we can use the 11569 * precalculated inverse to speed up arithmetics by turning divisions 11570 * into multiplications: 11571 */ 11572 const u32 sched_prio_to_wmult[40] = { 11573 /* -20 */ 48388, 59856, 76040, 92818, 118348, 11574 /* -15 */ 147320, 184698, 229616, 287308, 360437, 11575 /* -10 */ 449829, 563644, 704093, 875809, 1099582, 11576 /* -5 */ 1376151, 1717300, 2157191, 2708050, 3363326, 11577 /* 0 */ 4194304, 5237765, 6557202, 8165337, 10153587, 11578 /* 5 */ 12820798, 15790321, 19976592, 24970740, 31350126, 11579 /* 10 */ 39045157, 49367440, 61356676, 76695844, 95443717, 11580 /* 15 */ 119304647, 148102320, 186737708, 238609294, 286331153, 11581 }; 11582 11583 void call_trace_sched_update_nr_running(struct rq *rq, int count) 11584 { 11585 trace_sched_update_nr_running_tp(rq, count); 11586 } 11587 11588 #ifdef CONFIG_SCHED_MM_CID 11589 11590 /* 11591 * @cid_lock: Guarantee forward-progress of cid allocation. 11592 * 11593 * Concurrency ID allocation within a bitmap is mostly lock-free. The cid_lock 11594 * is only used when contention is detected by the lock-free allocation so 11595 * forward progress can be guaranteed. 11596 */ 11597 DEFINE_RAW_SPINLOCK(cid_lock); 11598 11599 /* 11600 * @use_cid_lock: Select cid allocation behavior: lock-free vs spinlock. 11601 * 11602 * When @use_cid_lock is 0, the cid allocation is lock-free. When contention is 11603 * detected, it is set to 1 to ensure that all newly coming allocations are 11604 * serialized by @cid_lock until the allocation which detected contention 11605 * completes and sets @use_cid_lock back to 0. This guarantees forward progress 11606 * of a cid allocation. 11607 */ 11608 int use_cid_lock; 11609 11610 /* 11611 * mm_cid remote-clear implements a lock-free algorithm to clear per-mm/cpu cid 11612 * concurrently with respect to the execution of the source runqueue context 11613 * switch. 11614 * 11615 * There is one basic properties we want to guarantee here: 11616 * 11617 * (1) Remote-clear should _never_ mark a per-cpu cid UNSET when it is actively 11618 * used by a task. That would lead to concurrent allocation of the cid and 11619 * userspace corruption. 11620 * 11621 * Provide this guarantee by introducing a Dekker memory ordering to guarantee 11622 * that a pair of loads observe at least one of a pair of stores, which can be 11623 * shown as: 11624 * 11625 * X = Y = 0 11626 * 11627 * w[X]=1 w[Y]=1 11628 * MB MB 11629 * r[Y]=y r[X]=x 11630 * 11631 * Which guarantees that x==0 && y==0 is impossible. But rather than using 11632 * values 0 and 1, this algorithm cares about specific state transitions of the 11633 * runqueue current task (as updated by the scheduler context switch), and the 11634 * per-mm/cpu cid value. 11635 * 11636 * Let's introduce task (Y) which has task->mm == mm and task (N) which has 11637 * task->mm != mm for the rest of the discussion. There are two scheduler state 11638 * transitions on context switch we care about: 11639 * 11640 * (TSA) Store to rq->curr with transition from (N) to (Y) 11641 * 11642 * (TSB) Store to rq->curr with transition from (Y) to (N) 11643 * 11644 * On the remote-clear side, there is one transition we care about: 11645 * 11646 * (TMA) cmpxchg to *pcpu_cid to set the LAZY flag 11647 * 11648 * There is also a transition to UNSET state which can be performed from all 11649 * sides (scheduler, remote-clear). It is always performed with a cmpxchg which 11650 * guarantees that only a single thread will succeed: 11651 * 11652 * (TMB) cmpxchg to *pcpu_cid to mark UNSET 11653 * 11654 * Just to be clear, what we do _not_ want to happen is a transition to UNSET 11655 * when a thread is actively using the cid (property (1)). 11656 * 11657 * Let's looks at the relevant combinations of TSA/TSB, and TMA transitions. 11658 * 11659 * Scenario A) (TSA)+(TMA) (from next task perspective) 11660 * 11661 * CPU0 CPU1 11662 * 11663 * Context switch CS-1 Remote-clear 11664 * - store to rq->curr: (N)->(Y) (TSA) - cmpxchg to *pcpu_id to LAZY (TMA) 11665 * (implied barrier after cmpxchg) 11666 * - switch_mm_cid() 11667 * - memory barrier (see switch_mm_cid() 11668 * comment explaining how this barrier 11669 * is combined with other scheduler 11670 * barriers) 11671 * - mm_cid_get (next) 11672 * - READ_ONCE(*pcpu_cid) - rcu_dereference(src_rq->curr) 11673 * 11674 * This Dekker ensures that either task (Y) is observed by the 11675 * rcu_dereference() or the LAZY flag is observed by READ_ONCE(), or both are 11676 * observed. 11677 * 11678 * If task (Y) store is observed by rcu_dereference(), it means that there is 11679 * still an active task on the cpu. Remote-clear will therefore not transition 11680 * to UNSET, which fulfills property (1). 11681 * 11682 * If task (Y) is not observed, but the lazy flag is observed by READ_ONCE(), 11683 * it will move its state to UNSET, which clears the percpu cid perhaps 11684 * uselessly (which is not an issue for correctness). Because task (Y) is not 11685 * observed, CPU1 can move ahead to set the state to UNSET. Because moving 11686 * state to UNSET is done with a cmpxchg expecting that the old state has the 11687 * LAZY flag set, only one thread will successfully UNSET. 11688 * 11689 * If both states (LAZY flag and task (Y)) are observed, the thread on CPU0 11690 * will observe the LAZY flag and transition to UNSET (perhaps uselessly), and 11691 * CPU1 will observe task (Y) and do nothing more, which is fine. 11692 * 11693 * What we are effectively preventing with this Dekker is a scenario where 11694 * neither LAZY flag nor store (Y) are observed, which would fail property (1) 11695 * because this would UNSET a cid which is actively used. 11696 */ 11697 11698 void sched_mm_cid_migrate_from(struct task_struct *t) 11699 { 11700 t->migrate_from_cpu = task_cpu(t); 11701 } 11702 11703 static 11704 int __sched_mm_cid_migrate_from_fetch_cid(struct rq *src_rq, 11705 struct task_struct *t, 11706 struct mm_cid *src_pcpu_cid) 11707 { 11708 struct mm_struct *mm = t->mm; 11709 struct task_struct *src_task; 11710 int src_cid, last_mm_cid; 11711 11712 if (!mm) 11713 return -1; 11714 11715 last_mm_cid = t->last_mm_cid; 11716 /* 11717 * If the migrated task has no last cid, or if the current 11718 * task on src rq uses the cid, it means the source cid does not need 11719 * to be moved to the destination cpu. 11720 */ 11721 if (last_mm_cid == -1) 11722 return -1; 11723 src_cid = READ_ONCE(src_pcpu_cid->cid); 11724 if (!mm_cid_is_valid(src_cid) || last_mm_cid != src_cid) 11725 return -1; 11726 11727 /* 11728 * If we observe an active task using the mm on this rq, it means we 11729 * are not the last task to be migrated from this cpu for this mm, so 11730 * there is no need to move src_cid to the destination cpu. 11731 */ 11732 rcu_read_lock(); 11733 src_task = rcu_dereference(src_rq->curr); 11734 if (READ_ONCE(src_task->mm_cid_active) && src_task->mm == mm) { 11735 rcu_read_unlock(); 11736 t->last_mm_cid = -1; 11737 return -1; 11738 } 11739 rcu_read_unlock(); 11740 11741 return src_cid; 11742 } 11743 11744 static 11745 int __sched_mm_cid_migrate_from_try_steal_cid(struct rq *src_rq, 11746 struct task_struct *t, 11747 struct mm_cid *src_pcpu_cid, 11748 int src_cid) 11749 { 11750 struct task_struct *src_task; 11751 struct mm_struct *mm = t->mm; 11752 int lazy_cid; 11753 11754 if (src_cid == -1) 11755 return -1; 11756 11757 /* 11758 * Attempt to clear the source cpu cid to move it to the destination 11759 * cpu. 11760 */ 11761 lazy_cid = mm_cid_set_lazy_put(src_cid); 11762 if (!try_cmpxchg(&src_pcpu_cid->cid, &src_cid, lazy_cid)) 11763 return -1; 11764 11765 /* 11766 * The implicit barrier after cmpxchg per-mm/cpu cid before loading 11767 * rq->curr->mm matches the scheduler barrier in context_switch() 11768 * between store to rq->curr and load of prev and next task's 11769 * per-mm/cpu cid. 11770 * 11771 * The implicit barrier after cmpxchg per-mm/cpu cid before loading 11772 * rq->curr->mm_cid_active matches the barrier in 11773 * sched_mm_cid_exit_signals(), sched_mm_cid_before_execve(), and 11774 * sched_mm_cid_after_execve() between store to t->mm_cid_active and 11775 * load of per-mm/cpu cid. 11776 */ 11777 11778 /* 11779 * If we observe an active task using the mm on this rq after setting 11780 * the lazy-put flag, this task will be responsible for transitioning 11781 * from lazy-put flag set to MM_CID_UNSET. 11782 */ 11783 rcu_read_lock(); 11784 src_task = rcu_dereference(src_rq->curr); 11785 if (READ_ONCE(src_task->mm_cid_active) && src_task->mm == mm) { 11786 rcu_read_unlock(); 11787 /* 11788 * We observed an active task for this mm, there is therefore 11789 * no point in moving this cid to the destination cpu. 11790 */ 11791 t->last_mm_cid = -1; 11792 return -1; 11793 } 11794 rcu_read_unlock(); 11795 11796 /* 11797 * The src_cid is unused, so it can be unset. 11798 */ 11799 if (!try_cmpxchg(&src_pcpu_cid->cid, &lazy_cid, MM_CID_UNSET)) 11800 return -1; 11801 return src_cid; 11802 } 11803 11804 /* 11805 * Migration to dst cpu. Called with dst_rq lock held. 11806 * Interrupts are disabled, which keeps the window of cid ownership without the 11807 * source rq lock held small. 11808 */ 11809 void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t) 11810 { 11811 struct mm_cid *src_pcpu_cid, *dst_pcpu_cid; 11812 struct mm_struct *mm = t->mm; 11813 int src_cid, dst_cid, src_cpu; 11814 struct rq *src_rq; 11815 11816 lockdep_assert_rq_held(dst_rq); 11817 11818 if (!mm) 11819 return; 11820 src_cpu = t->migrate_from_cpu; 11821 if (src_cpu == -1) { 11822 t->last_mm_cid = -1; 11823 return; 11824 } 11825 /* 11826 * Move the src cid if the dst cid is unset. This keeps id 11827 * allocation closest to 0 in cases where few threads migrate around 11828 * many cpus. 11829 * 11830 * If destination cid is already set, we may have to just clear 11831 * the src cid to ensure compactness in frequent migrations 11832 * scenarios. 11833 * 11834 * It is not useful to clear the src cid when the number of threads is 11835 * greater or equal to the number of allowed cpus, because user-space 11836 * can expect that the number of allowed cids can reach the number of 11837 * allowed cpus. 11838 */ 11839 dst_pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu_of(dst_rq)); 11840 dst_cid = READ_ONCE(dst_pcpu_cid->cid); 11841 if (!mm_cid_is_unset(dst_cid) && 11842 atomic_read(&mm->mm_users) >= t->nr_cpus_allowed) 11843 return; 11844 src_pcpu_cid = per_cpu_ptr(mm->pcpu_cid, src_cpu); 11845 src_rq = cpu_rq(src_cpu); 11846 src_cid = __sched_mm_cid_migrate_from_fetch_cid(src_rq, t, src_pcpu_cid); 11847 if (src_cid == -1) 11848 return; 11849 src_cid = __sched_mm_cid_migrate_from_try_steal_cid(src_rq, t, src_pcpu_cid, 11850 src_cid); 11851 if (src_cid == -1) 11852 return; 11853 if (!mm_cid_is_unset(dst_cid)) { 11854 __mm_cid_put(mm, src_cid); 11855 return; 11856 } 11857 /* Move src_cid to dst cpu. */ 11858 mm_cid_snapshot_time(dst_rq, mm); 11859 WRITE_ONCE(dst_pcpu_cid->cid, src_cid); 11860 } 11861 11862 static void sched_mm_cid_remote_clear(struct mm_struct *mm, struct mm_cid *pcpu_cid, 11863 int cpu) 11864 { 11865 struct rq *rq = cpu_rq(cpu); 11866 struct task_struct *t; 11867 unsigned long flags; 11868 int cid, lazy_cid; 11869 11870 cid = READ_ONCE(pcpu_cid->cid); 11871 if (!mm_cid_is_valid(cid)) 11872 return; 11873 11874 /* 11875 * Clear the cpu cid if it is set to keep cid allocation compact. If 11876 * there happens to be other tasks left on the source cpu using this 11877 * mm, the next task using this mm will reallocate its cid on context 11878 * switch. 11879 */ 11880 lazy_cid = mm_cid_set_lazy_put(cid); 11881 if (!try_cmpxchg(&pcpu_cid->cid, &cid, lazy_cid)) 11882 return; 11883 11884 /* 11885 * The implicit barrier after cmpxchg per-mm/cpu cid before loading 11886 * rq->curr->mm matches the scheduler barrier in context_switch() 11887 * between store to rq->curr and load of prev and next task's 11888 * per-mm/cpu cid. 11889 * 11890 * The implicit barrier after cmpxchg per-mm/cpu cid before loading 11891 * rq->curr->mm_cid_active matches the barrier in 11892 * sched_mm_cid_exit_signals(), sched_mm_cid_before_execve(), and 11893 * sched_mm_cid_after_execve() between store to t->mm_cid_active and 11894 * load of per-mm/cpu cid. 11895 */ 11896 11897 /* 11898 * If we observe an active task using the mm on this rq after setting 11899 * the lazy-put flag, that task will be responsible for transitioning 11900 * from lazy-put flag set to MM_CID_UNSET. 11901 */ 11902 rcu_read_lock(); 11903 t = rcu_dereference(rq->curr); 11904 if (READ_ONCE(t->mm_cid_active) && t->mm == mm) { 11905 rcu_read_unlock(); 11906 return; 11907 } 11908 rcu_read_unlock(); 11909 11910 /* 11911 * The cid is unused, so it can be unset. 11912 * Disable interrupts to keep the window of cid ownership without rq 11913 * lock small. 11914 */ 11915 local_irq_save(flags); 11916 if (try_cmpxchg(&pcpu_cid->cid, &lazy_cid, MM_CID_UNSET)) 11917 __mm_cid_put(mm, cid); 11918 local_irq_restore(flags); 11919 } 11920 11921 static void sched_mm_cid_remote_clear_old(struct mm_struct *mm, int cpu) 11922 { 11923 struct rq *rq = cpu_rq(cpu); 11924 struct mm_cid *pcpu_cid; 11925 struct task_struct *curr; 11926 u64 rq_clock; 11927 11928 /* 11929 * rq->clock load is racy on 32-bit but one spurious clear once in a 11930 * while is irrelevant. 11931 */ 11932 rq_clock = READ_ONCE(rq->clock); 11933 pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu); 11934 11935 /* 11936 * In order to take care of infrequently scheduled tasks, bump the time 11937 * snapshot associated with this cid if an active task using the mm is 11938 * observed on this rq. 11939 */ 11940 rcu_read_lock(); 11941 curr = rcu_dereference(rq->curr); 11942 if (READ_ONCE(curr->mm_cid_active) && curr->mm == mm) { 11943 WRITE_ONCE(pcpu_cid->time, rq_clock); 11944 rcu_read_unlock(); 11945 return; 11946 } 11947 rcu_read_unlock(); 11948 11949 if (rq_clock < pcpu_cid->time + SCHED_MM_CID_PERIOD_NS) 11950 return; 11951 sched_mm_cid_remote_clear(mm, pcpu_cid, cpu); 11952 } 11953 11954 static void sched_mm_cid_remote_clear_weight(struct mm_struct *mm, int cpu, 11955 int weight) 11956 { 11957 struct mm_cid *pcpu_cid; 11958 int cid; 11959 11960 pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu); 11961 cid = READ_ONCE(pcpu_cid->cid); 11962 if (!mm_cid_is_valid(cid) || cid < weight) 11963 return; 11964 sched_mm_cid_remote_clear(mm, pcpu_cid, cpu); 11965 } 11966 11967 static void task_mm_cid_work(struct callback_head *work) 11968 { 11969 unsigned long now = jiffies, old_scan, next_scan; 11970 struct task_struct *t = current; 11971 struct cpumask *cidmask; 11972 struct mm_struct *mm; 11973 int weight, cpu; 11974 11975 SCHED_WARN_ON(t != container_of(work, struct task_struct, cid_work)); 11976 11977 work->next = work; /* Prevent double-add */ 11978 if (t->flags & PF_EXITING) 11979 return; 11980 mm = t->mm; 11981 if (!mm) 11982 return; 11983 old_scan = READ_ONCE(mm->mm_cid_next_scan); 11984 next_scan = now + msecs_to_jiffies(MM_CID_SCAN_DELAY); 11985 if (!old_scan) { 11986 unsigned long res; 11987 11988 res = cmpxchg(&mm->mm_cid_next_scan, old_scan, next_scan); 11989 if (res != old_scan) 11990 old_scan = res; 11991 else 11992 old_scan = next_scan; 11993 } 11994 if (time_before(now, old_scan)) 11995 return; 11996 if (!try_cmpxchg(&mm->mm_cid_next_scan, &old_scan, next_scan)) 11997 return; 11998 cidmask = mm_cidmask(mm); 11999 /* Clear cids that were not recently used. */ 12000 for_each_possible_cpu(cpu) 12001 sched_mm_cid_remote_clear_old(mm, cpu); 12002 weight = cpumask_weight(cidmask); 12003 /* 12004 * Clear cids that are greater or equal to the cidmask weight to 12005 * recompact it. 12006 */ 12007 for_each_possible_cpu(cpu) 12008 sched_mm_cid_remote_clear_weight(mm, cpu, weight); 12009 } 12010 12011 void init_sched_mm_cid(struct task_struct *t) 12012 { 12013 struct mm_struct *mm = t->mm; 12014 int mm_users = 0; 12015 12016 if (mm) { 12017 mm_users = atomic_read(&mm->mm_users); 12018 if (mm_users == 1) 12019 mm->mm_cid_next_scan = jiffies + msecs_to_jiffies(MM_CID_SCAN_DELAY); 12020 } 12021 t->cid_work.next = &t->cid_work; /* Protect against double add */ 12022 init_task_work(&t->cid_work, task_mm_cid_work); 12023 } 12024 12025 void task_tick_mm_cid(struct rq *rq, struct task_struct *curr) 12026 { 12027 struct callback_head *work = &curr->cid_work; 12028 unsigned long now = jiffies; 12029 12030 if (!curr->mm || (curr->flags & (PF_EXITING | PF_KTHREAD)) || 12031 work->next != work) 12032 return; 12033 if (time_before(now, READ_ONCE(curr->mm->mm_cid_next_scan))) 12034 return; 12035 task_work_add(curr, work, TWA_RESUME); 12036 } 12037 12038 void sched_mm_cid_exit_signals(struct task_struct *t) 12039 { 12040 struct mm_struct *mm = t->mm; 12041 struct rq_flags rf; 12042 struct rq *rq; 12043 12044 if (!mm) 12045 return; 12046 12047 preempt_disable(); 12048 rq = this_rq(); 12049 rq_lock_irqsave(rq, &rf); 12050 preempt_enable_no_resched(); /* holding spinlock */ 12051 WRITE_ONCE(t->mm_cid_active, 0); 12052 /* 12053 * Store t->mm_cid_active before loading per-mm/cpu cid. 12054 * Matches barrier in sched_mm_cid_remote_clear_old(). 12055 */ 12056 smp_mb(); 12057 mm_cid_put(mm); 12058 t->last_mm_cid = t->mm_cid = -1; 12059 rq_unlock_irqrestore(rq, &rf); 12060 } 12061 12062 void sched_mm_cid_before_execve(struct task_struct *t) 12063 { 12064 struct mm_struct *mm = t->mm; 12065 struct rq_flags rf; 12066 struct rq *rq; 12067 12068 if (!mm) 12069 return; 12070 12071 preempt_disable(); 12072 rq = this_rq(); 12073 rq_lock_irqsave(rq, &rf); 12074 preempt_enable_no_resched(); /* holding spinlock */ 12075 WRITE_ONCE(t->mm_cid_active, 0); 12076 /* 12077 * Store t->mm_cid_active before loading per-mm/cpu cid. 12078 * Matches barrier in sched_mm_cid_remote_clear_old(). 12079 */ 12080 smp_mb(); 12081 mm_cid_put(mm); 12082 t->last_mm_cid = t->mm_cid = -1; 12083 rq_unlock_irqrestore(rq, &rf); 12084 } 12085 12086 void sched_mm_cid_after_execve(struct task_struct *t) 12087 { 12088 struct mm_struct *mm = t->mm; 12089 struct rq_flags rf; 12090 struct rq *rq; 12091 12092 if (!mm) 12093 return; 12094 12095 preempt_disable(); 12096 rq = this_rq(); 12097 rq_lock_irqsave(rq, &rf); 12098 preempt_enable_no_resched(); /* holding spinlock */ 12099 WRITE_ONCE(t->mm_cid_active, 1); 12100 /* 12101 * Store t->mm_cid_active before loading per-mm/cpu cid. 12102 * Matches barrier in sched_mm_cid_remote_clear_old(). 12103 */ 12104 smp_mb(); 12105 t->last_mm_cid = t->mm_cid = mm_cid_get(rq, mm); 12106 rq_unlock_irqrestore(rq, &rf); 12107 rseq_set_notify_resume(t); 12108 } 12109 12110 void sched_mm_cid_fork(struct task_struct *t) 12111 { 12112 WARN_ON_ONCE(!t->mm || t->mm_cid != -1); 12113 t->mm_cid_active = 1; 12114 } 12115 #endif 12116