1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * kernel/sched/core.c 4 * 5 * Core kernel scheduler code and related syscalls 6 * 7 * Copyright (C) 1991-2002 Linus Torvalds 8 */ 9 #define CREATE_TRACE_POINTS 10 #include <trace/events/sched.h> 11 #undef CREATE_TRACE_POINTS 12 13 #include "sched.h" 14 15 #include <linux/nospec.h> 16 17 #include <linux/kcov.h> 18 #include <linux/scs.h> 19 20 #include <asm/switch_to.h> 21 #include <asm/tlb.h> 22 23 #include "../workqueue_internal.h" 24 #include "../../fs/io-wq.h" 25 #include "../smpboot.h" 26 27 #include "pelt.h" 28 #include "smp.h" 29 30 /* 31 * Export tracepoints that act as a bare tracehook (ie: have no trace event 32 * associated with them) to allow external modules to probe them. 33 */ 34 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_cfs_tp); 35 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_rt_tp); 36 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_dl_tp); 37 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_irq_tp); 38 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_se_tp); 39 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_cpu_capacity_tp); 40 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_overutilized_tp); 41 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_cfs_tp); 42 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_se_tp); 43 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_update_nr_running_tp); 44 45 DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); 46 47 #ifdef CONFIG_SCHED_DEBUG 48 /* 49 * Debugging: various feature bits 50 * 51 * If SCHED_DEBUG is disabled, each compilation unit has its own copy of 52 * sysctl_sched_features, defined in sched.h, to allow constants propagation 53 * at compile time and compiler optimization based on features default. 54 */ 55 #define SCHED_FEAT(name, enabled) \ 56 (1UL << __SCHED_FEAT_##name) * enabled | 57 const_debug unsigned int sysctl_sched_features = 58 #include "features.h" 59 0; 60 #undef SCHED_FEAT 61 #endif 62 63 /* 64 * Number of tasks to iterate in a single balance run. 65 * Limited because this is done with IRQs disabled. 66 */ 67 const_debug unsigned int sysctl_sched_nr_migrate = 32; 68 69 /* 70 * period over which we measure -rt task CPU usage in us. 71 * default: 1s 72 */ 73 unsigned int sysctl_sched_rt_period = 1000000; 74 75 __read_mostly int scheduler_running; 76 77 /* 78 * part of the period that we allow rt tasks to run in us. 79 * default: 0.95s 80 */ 81 int sysctl_sched_rt_runtime = 950000; 82 83 84 /* 85 * Serialization rules: 86 * 87 * Lock order: 88 * 89 * p->pi_lock 90 * rq->lock 91 * hrtimer_cpu_base->lock (hrtimer_start() for bandwidth controls) 92 * 93 * rq1->lock 94 * rq2->lock where: rq1 < rq2 95 * 96 * Regular state: 97 * 98 * Normal scheduling state is serialized by rq->lock. __schedule() takes the 99 * local CPU's rq->lock, it optionally removes the task from the runqueue and 100 * always looks at the local rq data structures to find the most elegible task 101 * to run next. 102 * 103 * Task enqueue is also under rq->lock, possibly taken from another CPU. 104 * Wakeups from another LLC domain might use an IPI to transfer the enqueue to 105 * the local CPU to avoid bouncing the runqueue state around [ see 106 * ttwu_queue_wakelist() ] 107 * 108 * Task wakeup, specifically wakeups that involve migration, are horribly 109 * complicated to avoid having to take two rq->locks. 110 * 111 * Special state: 112 * 113 * System-calls and anything external will use task_rq_lock() which acquires 114 * both p->pi_lock and rq->lock. As a consequence the state they change is 115 * stable while holding either lock: 116 * 117 * - sched_setaffinity()/ 118 * set_cpus_allowed_ptr(): p->cpus_ptr, p->nr_cpus_allowed 119 * - set_user_nice(): p->se.load, p->*prio 120 * - __sched_setscheduler(): p->sched_class, p->policy, p->*prio, 121 * p->se.load, p->rt_priority, 122 * p->dl.dl_{runtime, deadline, period, flags, bw, density} 123 * - sched_setnuma(): p->numa_preferred_nid 124 * - sched_move_task()/ 125 * cpu_cgroup_fork(): p->sched_task_group 126 * - uclamp_update_active() p->uclamp* 127 * 128 * p->state <- TASK_*: 129 * 130 * is changed locklessly using set_current_state(), __set_current_state() or 131 * set_special_state(), see their respective comments, or by 132 * try_to_wake_up(). This latter uses p->pi_lock to serialize against 133 * concurrent self. 134 * 135 * p->on_rq <- { 0, 1 = TASK_ON_RQ_QUEUED, 2 = TASK_ON_RQ_MIGRATING }: 136 * 137 * is set by activate_task() and cleared by deactivate_task(), under 138 * rq->lock. Non-zero indicates the task is runnable, the special 139 * ON_RQ_MIGRATING state is used for migration without holding both 140 * rq->locks. It indicates task_cpu() is not stable, see task_rq_lock(). 141 * 142 * p->on_cpu <- { 0, 1 }: 143 * 144 * is set by prepare_task() and cleared by finish_task() such that it will be 145 * set before p is scheduled-in and cleared after p is scheduled-out, both 146 * under rq->lock. Non-zero indicates the task is running on its CPU. 147 * 148 * [ The astute reader will observe that it is possible for two tasks on one 149 * CPU to have ->on_cpu = 1 at the same time. ] 150 * 151 * task_cpu(p): is changed by set_task_cpu(), the rules are: 152 * 153 * - Don't call set_task_cpu() on a blocked task: 154 * 155 * We don't care what CPU we're not running on, this simplifies hotplug, 156 * the CPU assignment of blocked tasks isn't required to be valid. 157 * 158 * - for try_to_wake_up(), called under p->pi_lock: 159 * 160 * This allows try_to_wake_up() to only take one rq->lock, see its comment. 161 * 162 * - for migration called under rq->lock: 163 * [ see task_on_rq_migrating() in task_rq_lock() ] 164 * 165 * o move_queued_task() 166 * o detach_task() 167 * 168 * - for migration called under double_rq_lock(): 169 * 170 * o __migrate_swap_task() 171 * o push_rt_task() / pull_rt_task() 172 * o push_dl_task() / pull_dl_task() 173 * o dl_task_offline_migration() 174 * 175 */ 176 177 /* 178 * __task_rq_lock - lock the rq @p resides on. 179 */ 180 struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf) 181 __acquires(rq->lock) 182 { 183 struct rq *rq; 184 185 lockdep_assert_held(&p->pi_lock); 186 187 for (;;) { 188 rq = task_rq(p); 189 raw_spin_lock(&rq->lock); 190 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) { 191 rq_pin_lock(rq, rf); 192 return rq; 193 } 194 raw_spin_unlock(&rq->lock); 195 196 while (unlikely(task_on_rq_migrating(p))) 197 cpu_relax(); 198 } 199 } 200 201 /* 202 * task_rq_lock - lock p->pi_lock and lock the rq @p resides on. 203 */ 204 struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf) 205 __acquires(p->pi_lock) 206 __acquires(rq->lock) 207 { 208 struct rq *rq; 209 210 for (;;) { 211 raw_spin_lock_irqsave(&p->pi_lock, rf->flags); 212 rq = task_rq(p); 213 raw_spin_lock(&rq->lock); 214 /* 215 * move_queued_task() task_rq_lock() 216 * 217 * ACQUIRE (rq->lock) 218 * [S] ->on_rq = MIGRATING [L] rq = task_rq() 219 * WMB (__set_task_cpu()) ACQUIRE (rq->lock); 220 * [S] ->cpu = new_cpu [L] task_rq() 221 * [L] ->on_rq 222 * RELEASE (rq->lock) 223 * 224 * If we observe the old CPU in task_rq_lock(), the acquire of 225 * the old rq->lock will fully serialize against the stores. 226 * 227 * If we observe the new CPU in task_rq_lock(), the address 228 * dependency headed by '[L] rq = task_rq()' and the acquire 229 * will pair with the WMB to ensure we then also see migrating. 230 */ 231 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) { 232 rq_pin_lock(rq, rf); 233 return rq; 234 } 235 raw_spin_unlock(&rq->lock); 236 raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags); 237 238 while (unlikely(task_on_rq_migrating(p))) 239 cpu_relax(); 240 } 241 } 242 243 /* 244 * RQ-clock updating methods: 245 */ 246 247 static void update_rq_clock_task(struct rq *rq, s64 delta) 248 { 249 /* 250 * In theory, the compile should just see 0 here, and optimize out the call 251 * to sched_rt_avg_update. But I don't trust it... 252 */ 253 s64 __maybe_unused steal = 0, irq_delta = 0; 254 255 #ifdef CONFIG_IRQ_TIME_ACCOUNTING 256 irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time; 257 258 /* 259 * Since irq_time is only updated on {soft,}irq_exit, we might run into 260 * this case when a previous update_rq_clock() happened inside a 261 * {soft,}irq region. 262 * 263 * When this happens, we stop ->clock_task and only update the 264 * prev_irq_time stamp to account for the part that fit, so that a next 265 * update will consume the rest. This ensures ->clock_task is 266 * monotonic. 267 * 268 * It does however cause some slight miss-attribution of {soft,}irq 269 * time, a more accurate solution would be to update the irq_time using 270 * the current rq->clock timestamp, except that would require using 271 * atomic ops. 272 */ 273 if (irq_delta > delta) 274 irq_delta = delta; 275 276 rq->prev_irq_time += irq_delta; 277 delta -= irq_delta; 278 #endif 279 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING 280 if (static_key_false((¶virt_steal_rq_enabled))) { 281 steal = paravirt_steal_clock(cpu_of(rq)); 282 steal -= rq->prev_steal_time_rq; 283 284 if (unlikely(steal > delta)) 285 steal = delta; 286 287 rq->prev_steal_time_rq += steal; 288 delta -= steal; 289 } 290 #endif 291 292 rq->clock_task += delta; 293 294 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ 295 if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY)) 296 update_irq_load_avg(rq, irq_delta + steal); 297 #endif 298 update_rq_clock_pelt(rq, delta); 299 } 300 301 void update_rq_clock(struct rq *rq) 302 { 303 s64 delta; 304 305 lockdep_assert_held(&rq->lock); 306 307 if (rq->clock_update_flags & RQCF_ACT_SKIP) 308 return; 309 310 #ifdef CONFIG_SCHED_DEBUG 311 if (sched_feat(WARN_DOUBLE_CLOCK)) 312 SCHED_WARN_ON(rq->clock_update_flags & RQCF_UPDATED); 313 rq->clock_update_flags |= RQCF_UPDATED; 314 #endif 315 316 delta = sched_clock_cpu(cpu_of(rq)) - rq->clock; 317 if (delta < 0) 318 return; 319 rq->clock += delta; 320 update_rq_clock_task(rq, delta); 321 } 322 323 static inline void 324 rq_csd_init(struct rq *rq, call_single_data_t *csd, smp_call_func_t func) 325 { 326 csd->flags = 0; 327 csd->func = func; 328 csd->info = rq; 329 } 330 331 #ifdef CONFIG_SCHED_HRTICK 332 /* 333 * Use HR-timers to deliver accurate preemption points. 334 */ 335 336 static void hrtick_clear(struct rq *rq) 337 { 338 if (hrtimer_active(&rq->hrtick_timer)) 339 hrtimer_cancel(&rq->hrtick_timer); 340 } 341 342 /* 343 * High-resolution timer tick. 344 * Runs from hardirq context with interrupts disabled. 345 */ 346 static enum hrtimer_restart hrtick(struct hrtimer *timer) 347 { 348 struct rq *rq = container_of(timer, struct rq, hrtick_timer); 349 struct rq_flags rf; 350 351 WARN_ON_ONCE(cpu_of(rq) != smp_processor_id()); 352 353 rq_lock(rq, &rf); 354 update_rq_clock(rq); 355 rq->curr->sched_class->task_tick(rq, rq->curr, 1); 356 rq_unlock(rq, &rf); 357 358 return HRTIMER_NORESTART; 359 } 360 361 #ifdef CONFIG_SMP 362 363 static void __hrtick_restart(struct rq *rq) 364 { 365 struct hrtimer *timer = &rq->hrtick_timer; 366 367 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED_HARD); 368 } 369 370 /* 371 * called from hardirq (IPI) context 372 */ 373 static void __hrtick_start(void *arg) 374 { 375 struct rq *rq = arg; 376 struct rq_flags rf; 377 378 rq_lock(rq, &rf); 379 __hrtick_restart(rq); 380 rq_unlock(rq, &rf); 381 } 382 383 /* 384 * Called to set the hrtick timer state. 385 * 386 * called with rq->lock held and irqs disabled 387 */ 388 void hrtick_start(struct rq *rq, u64 delay) 389 { 390 struct hrtimer *timer = &rq->hrtick_timer; 391 ktime_t time; 392 s64 delta; 393 394 /* 395 * Don't schedule slices shorter than 10000ns, that just 396 * doesn't make sense and can cause timer DoS. 397 */ 398 delta = max_t(s64, delay, 10000LL); 399 time = ktime_add_ns(timer->base->get_time(), delta); 400 401 hrtimer_set_expires(timer, time); 402 403 if (rq == this_rq()) 404 __hrtick_restart(rq); 405 else 406 smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd); 407 } 408 409 #else 410 /* 411 * Called to set the hrtick timer state. 412 * 413 * called with rq->lock held and irqs disabled 414 */ 415 void hrtick_start(struct rq *rq, u64 delay) 416 { 417 /* 418 * Don't schedule slices shorter than 10000ns, that just 419 * doesn't make sense. Rely on vruntime for fairness. 420 */ 421 delay = max_t(u64, delay, 10000LL); 422 hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay), 423 HRTIMER_MODE_REL_PINNED_HARD); 424 } 425 426 #endif /* CONFIG_SMP */ 427 428 static void hrtick_rq_init(struct rq *rq) 429 { 430 #ifdef CONFIG_SMP 431 rq_csd_init(rq, &rq->hrtick_csd, __hrtick_start); 432 #endif 433 hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); 434 rq->hrtick_timer.function = hrtick; 435 } 436 #else /* CONFIG_SCHED_HRTICK */ 437 static inline void hrtick_clear(struct rq *rq) 438 { 439 } 440 441 static inline void hrtick_rq_init(struct rq *rq) 442 { 443 } 444 #endif /* CONFIG_SCHED_HRTICK */ 445 446 /* 447 * cmpxchg based fetch_or, macro so it works for different integer types 448 */ 449 #define fetch_or(ptr, mask) \ 450 ({ \ 451 typeof(ptr) _ptr = (ptr); \ 452 typeof(mask) _mask = (mask); \ 453 typeof(*_ptr) _old, _val = *_ptr; \ 454 \ 455 for (;;) { \ 456 _old = cmpxchg(_ptr, _val, _val | _mask); \ 457 if (_old == _val) \ 458 break; \ 459 _val = _old; \ 460 } \ 461 _old; \ 462 }) 463 464 #if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG) 465 /* 466 * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG, 467 * this avoids any races wrt polling state changes and thereby avoids 468 * spurious IPIs. 469 */ 470 static bool set_nr_and_not_polling(struct task_struct *p) 471 { 472 struct thread_info *ti = task_thread_info(p); 473 return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG); 474 } 475 476 /* 477 * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set. 478 * 479 * If this returns true, then the idle task promises to call 480 * sched_ttwu_pending() and reschedule soon. 481 */ 482 static bool set_nr_if_polling(struct task_struct *p) 483 { 484 struct thread_info *ti = task_thread_info(p); 485 typeof(ti->flags) old, val = READ_ONCE(ti->flags); 486 487 for (;;) { 488 if (!(val & _TIF_POLLING_NRFLAG)) 489 return false; 490 if (val & _TIF_NEED_RESCHED) 491 return true; 492 old = cmpxchg(&ti->flags, val, val | _TIF_NEED_RESCHED); 493 if (old == val) 494 break; 495 val = old; 496 } 497 return true; 498 } 499 500 #else 501 static bool set_nr_and_not_polling(struct task_struct *p) 502 { 503 set_tsk_need_resched(p); 504 return true; 505 } 506 507 #ifdef CONFIG_SMP 508 static bool set_nr_if_polling(struct task_struct *p) 509 { 510 return false; 511 } 512 #endif 513 #endif 514 515 static bool __wake_q_add(struct wake_q_head *head, struct task_struct *task) 516 { 517 struct wake_q_node *node = &task->wake_q; 518 519 /* 520 * Atomically grab the task, if ->wake_q is !nil already it means 521 * its already queued (either by us or someone else) and will get the 522 * wakeup due to that. 523 * 524 * In order to ensure that a pending wakeup will observe our pending 525 * state, even in the failed case, an explicit smp_mb() must be used. 526 */ 527 smp_mb__before_atomic(); 528 if (unlikely(cmpxchg_relaxed(&node->next, NULL, WAKE_Q_TAIL))) 529 return false; 530 531 /* 532 * The head is context local, there can be no concurrency. 533 */ 534 *head->lastp = node; 535 head->lastp = &node->next; 536 return true; 537 } 538 539 /** 540 * wake_q_add() - queue a wakeup for 'later' waking. 541 * @head: the wake_q_head to add @task to 542 * @task: the task to queue for 'later' wakeup 543 * 544 * Queue a task for later wakeup, most likely by the wake_up_q() call in the 545 * same context, _HOWEVER_ this is not guaranteed, the wakeup can come 546 * instantly. 547 * 548 * This function must be used as-if it were wake_up_process(); IOW the task 549 * must be ready to be woken at this location. 550 */ 551 void wake_q_add(struct wake_q_head *head, struct task_struct *task) 552 { 553 if (__wake_q_add(head, task)) 554 get_task_struct(task); 555 } 556 557 /** 558 * wake_q_add_safe() - safely queue a wakeup for 'later' waking. 559 * @head: the wake_q_head to add @task to 560 * @task: the task to queue for 'later' wakeup 561 * 562 * Queue a task for later wakeup, most likely by the wake_up_q() call in the 563 * same context, _HOWEVER_ this is not guaranteed, the wakeup can come 564 * instantly. 565 * 566 * This function must be used as-if it were wake_up_process(); IOW the task 567 * must be ready to be woken at this location. 568 * 569 * This function is essentially a task-safe equivalent to wake_q_add(). Callers 570 * that already hold reference to @task can call the 'safe' version and trust 571 * wake_q to do the right thing depending whether or not the @task is already 572 * queued for wakeup. 573 */ 574 void wake_q_add_safe(struct wake_q_head *head, struct task_struct *task) 575 { 576 if (!__wake_q_add(head, task)) 577 put_task_struct(task); 578 } 579 580 void wake_up_q(struct wake_q_head *head) 581 { 582 struct wake_q_node *node = head->first; 583 584 while (node != WAKE_Q_TAIL) { 585 struct task_struct *task; 586 587 task = container_of(node, struct task_struct, wake_q); 588 BUG_ON(!task); 589 /* Task can safely be re-inserted now: */ 590 node = node->next; 591 task->wake_q.next = NULL; 592 593 /* 594 * wake_up_process() executes a full barrier, which pairs with 595 * the queueing in wake_q_add() so as not to miss wakeups. 596 */ 597 wake_up_process(task); 598 put_task_struct(task); 599 } 600 } 601 602 /* 603 * resched_curr - mark rq's current task 'to be rescheduled now'. 604 * 605 * On UP this means the setting of the need_resched flag, on SMP it 606 * might also involve a cross-CPU call to trigger the scheduler on 607 * the target CPU. 608 */ 609 void resched_curr(struct rq *rq) 610 { 611 struct task_struct *curr = rq->curr; 612 int cpu; 613 614 lockdep_assert_held(&rq->lock); 615 616 if (test_tsk_need_resched(curr)) 617 return; 618 619 cpu = cpu_of(rq); 620 621 if (cpu == smp_processor_id()) { 622 set_tsk_need_resched(curr); 623 set_preempt_need_resched(); 624 return; 625 } 626 627 if (set_nr_and_not_polling(curr)) 628 smp_send_reschedule(cpu); 629 else 630 trace_sched_wake_idle_without_ipi(cpu); 631 } 632 633 void resched_cpu(int cpu) 634 { 635 struct rq *rq = cpu_rq(cpu); 636 unsigned long flags; 637 638 raw_spin_lock_irqsave(&rq->lock, flags); 639 if (cpu_online(cpu) || cpu == smp_processor_id()) 640 resched_curr(rq); 641 raw_spin_unlock_irqrestore(&rq->lock, flags); 642 } 643 644 #ifdef CONFIG_SMP 645 #ifdef CONFIG_NO_HZ_COMMON 646 /* 647 * In the semi idle case, use the nearest busy CPU for migrating timers 648 * from an idle CPU. This is good for power-savings. 649 * 650 * We don't do similar optimization for completely idle system, as 651 * selecting an idle CPU will add more delays to the timers than intended 652 * (as that CPU's timer base may not be uptodate wrt jiffies etc). 653 */ 654 int get_nohz_timer_target(void) 655 { 656 int i, cpu = smp_processor_id(), default_cpu = -1; 657 struct sched_domain *sd; 658 659 if (housekeeping_cpu(cpu, HK_FLAG_TIMER)) { 660 if (!idle_cpu(cpu)) 661 return cpu; 662 default_cpu = cpu; 663 } 664 665 rcu_read_lock(); 666 for_each_domain(cpu, sd) { 667 for_each_cpu_and(i, sched_domain_span(sd), 668 housekeeping_cpumask(HK_FLAG_TIMER)) { 669 if (cpu == i) 670 continue; 671 672 if (!idle_cpu(i)) { 673 cpu = i; 674 goto unlock; 675 } 676 } 677 } 678 679 if (default_cpu == -1) 680 default_cpu = housekeeping_any_cpu(HK_FLAG_TIMER); 681 cpu = default_cpu; 682 unlock: 683 rcu_read_unlock(); 684 return cpu; 685 } 686 687 /* 688 * When add_timer_on() enqueues a timer into the timer wheel of an 689 * idle CPU then this timer might expire before the next timer event 690 * which is scheduled to wake up that CPU. In case of a completely 691 * idle system the next event might even be infinite time into the 692 * future. wake_up_idle_cpu() ensures that the CPU is woken up and 693 * leaves the inner idle loop so the newly added timer is taken into 694 * account when the CPU goes back to idle and evaluates the timer 695 * wheel for the next timer event. 696 */ 697 static void wake_up_idle_cpu(int cpu) 698 { 699 struct rq *rq = cpu_rq(cpu); 700 701 if (cpu == smp_processor_id()) 702 return; 703 704 if (set_nr_and_not_polling(rq->idle)) 705 smp_send_reschedule(cpu); 706 else 707 trace_sched_wake_idle_without_ipi(cpu); 708 } 709 710 static bool wake_up_full_nohz_cpu(int cpu) 711 { 712 /* 713 * We just need the target to call irq_exit() and re-evaluate 714 * the next tick. The nohz full kick at least implies that. 715 * If needed we can still optimize that later with an 716 * empty IRQ. 717 */ 718 if (cpu_is_offline(cpu)) 719 return true; /* Don't try to wake offline CPUs. */ 720 if (tick_nohz_full_cpu(cpu)) { 721 if (cpu != smp_processor_id() || 722 tick_nohz_tick_stopped()) 723 tick_nohz_full_kick_cpu(cpu); 724 return true; 725 } 726 727 return false; 728 } 729 730 /* 731 * Wake up the specified CPU. If the CPU is going offline, it is the 732 * caller's responsibility to deal with the lost wakeup, for example, 733 * by hooking into the CPU_DEAD notifier like timers and hrtimers do. 734 */ 735 void wake_up_nohz_cpu(int cpu) 736 { 737 if (!wake_up_full_nohz_cpu(cpu)) 738 wake_up_idle_cpu(cpu); 739 } 740 741 static void nohz_csd_func(void *info) 742 { 743 struct rq *rq = info; 744 int cpu = cpu_of(rq); 745 unsigned int flags; 746 747 /* 748 * Release the rq::nohz_csd. 749 */ 750 flags = atomic_fetch_andnot(NOHZ_KICK_MASK, nohz_flags(cpu)); 751 WARN_ON(!(flags & NOHZ_KICK_MASK)); 752 753 rq->idle_balance = idle_cpu(cpu); 754 if (rq->idle_balance && !need_resched()) { 755 rq->nohz_idle_balance = flags; 756 raise_softirq_irqoff(SCHED_SOFTIRQ); 757 } 758 } 759 760 #endif /* CONFIG_NO_HZ_COMMON */ 761 762 #ifdef CONFIG_NO_HZ_FULL 763 bool sched_can_stop_tick(struct rq *rq) 764 { 765 int fifo_nr_running; 766 767 /* Deadline tasks, even if single, need the tick */ 768 if (rq->dl.dl_nr_running) 769 return false; 770 771 /* 772 * If there are more than one RR tasks, we need the tick to effect the 773 * actual RR behaviour. 774 */ 775 if (rq->rt.rr_nr_running) { 776 if (rq->rt.rr_nr_running == 1) 777 return true; 778 else 779 return false; 780 } 781 782 /* 783 * If there's no RR tasks, but FIFO tasks, we can skip the tick, no 784 * forced preemption between FIFO tasks. 785 */ 786 fifo_nr_running = rq->rt.rt_nr_running - rq->rt.rr_nr_running; 787 if (fifo_nr_running) 788 return true; 789 790 /* 791 * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left; 792 * if there's more than one we need the tick for involuntary 793 * preemption. 794 */ 795 if (rq->nr_running > 1) 796 return false; 797 798 return true; 799 } 800 #endif /* CONFIG_NO_HZ_FULL */ 801 #endif /* CONFIG_SMP */ 802 803 #if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \ 804 (defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH))) 805 /* 806 * Iterate task_group tree rooted at *from, calling @down when first entering a 807 * node and @up when leaving it for the final time. 808 * 809 * Caller must hold rcu_lock or sufficient equivalent. 810 */ 811 int walk_tg_tree_from(struct task_group *from, 812 tg_visitor down, tg_visitor up, void *data) 813 { 814 struct task_group *parent, *child; 815 int ret; 816 817 parent = from; 818 819 down: 820 ret = (*down)(parent, data); 821 if (ret) 822 goto out; 823 list_for_each_entry_rcu(child, &parent->children, siblings) { 824 parent = child; 825 goto down; 826 827 up: 828 continue; 829 } 830 ret = (*up)(parent, data); 831 if (ret || parent == from) 832 goto out; 833 834 child = parent; 835 parent = parent->parent; 836 if (parent) 837 goto up; 838 out: 839 return ret; 840 } 841 842 int tg_nop(struct task_group *tg, void *data) 843 { 844 return 0; 845 } 846 #endif 847 848 static void set_load_weight(struct task_struct *p, bool update_load) 849 { 850 int prio = p->static_prio - MAX_RT_PRIO; 851 struct load_weight *load = &p->se.load; 852 853 /* 854 * SCHED_IDLE tasks get minimal weight: 855 */ 856 if (task_has_idle_policy(p)) { 857 load->weight = scale_load(WEIGHT_IDLEPRIO); 858 load->inv_weight = WMULT_IDLEPRIO; 859 return; 860 } 861 862 /* 863 * SCHED_OTHER tasks have to update their load when changing their 864 * weight 865 */ 866 if (update_load && p->sched_class == &fair_sched_class) { 867 reweight_task(p, prio); 868 } else { 869 load->weight = scale_load(sched_prio_to_weight[prio]); 870 load->inv_weight = sched_prio_to_wmult[prio]; 871 } 872 } 873 874 #ifdef CONFIG_UCLAMP_TASK 875 /* 876 * Serializes updates of utilization clamp values 877 * 878 * The (slow-path) user-space triggers utilization clamp value updates which 879 * can require updates on (fast-path) scheduler's data structures used to 880 * support enqueue/dequeue operations. 881 * While the per-CPU rq lock protects fast-path update operations, user-space 882 * requests are serialized using a mutex to reduce the risk of conflicting 883 * updates or API abuses. 884 */ 885 static DEFINE_MUTEX(uclamp_mutex); 886 887 /* Max allowed minimum utilization */ 888 unsigned int sysctl_sched_uclamp_util_min = SCHED_CAPACITY_SCALE; 889 890 /* Max allowed maximum utilization */ 891 unsigned int sysctl_sched_uclamp_util_max = SCHED_CAPACITY_SCALE; 892 893 /* 894 * By default RT tasks run at the maximum performance point/capacity of the 895 * system. Uclamp enforces this by always setting UCLAMP_MIN of RT tasks to 896 * SCHED_CAPACITY_SCALE. 897 * 898 * This knob allows admins to change the default behavior when uclamp is being 899 * used. In battery powered devices, particularly, running at the maximum 900 * capacity and frequency will increase energy consumption and shorten the 901 * battery life. 902 * 903 * This knob only affects RT tasks that their uclamp_se->user_defined == false. 904 * 905 * This knob will not override the system default sched_util_clamp_min defined 906 * above. 907 */ 908 unsigned int sysctl_sched_uclamp_util_min_rt_default = SCHED_CAPACITY_SCALE; 909 910 /* All clamps are required to be less or equal than these values */ 911 static struct uclamp_se uclamp_default[UCLAMP_CNT]; 912 913 /* 914 * This static key is used to reduce the uclamp overhead in the fast path. It 915 * primarily disables the call to uclamp_rq_{inc, dec}() in 916 * enqueue/dequeue_task(). 917 * 918 * This allows users to continue to enable uclamp in their kernel config with 919 * minimum uclamp overhead in the fast path. 920 * 921 * As soon as userspace modifies any of the uclamp knobs, the static key is 922 * enabled, since we have an actual users that make use of uclamp 923 * functionality. 924 * 925 * The knobs that would enable this static key are: 926 * 927 * * A task modifying its uclamp value with sched_setattr(). 928 * * An admin modifying the sysctl_sched_uclamp_{min, max} via procfs. 929 * * An admin modifying the cgroup cpu.uclamp.{min, max} 930 */ 931 DEFINE_STATIC_KEY_FALSE(sched_uclamp_used); 932 933 /* Integer rounded range for each bucket */ 934 #define UCLAMP_BUCKET_DELTA DIV_ROUND_CLOSEST(SCHED_CAPACITY_SCALE, UCLAMP_BUCKETS) 935 936 #define for_each_clamp_id(clamp_id) \ 937 for ((clamp_id) = 0; (clamp_id) < UCLAMP_CNT; (clamp_id)++) 938 939 static inline unsigned int uclamp_bucket_id(unsigned int clamp_value) 940 { 941 return clamp_value / UCLAMP_BUCKET_DELTA; 942 } 943 944 static inline unsigned int uclamp_none(enum uclamp_id clamp_id) 945 { 946 if (clamp_id == UCLAMP_MIN) 947 return 0; 948 return SCHED_CAPACITY_SCALE; 949 } 950 951 static inline void uclamp_se_set(struct uclamp_se *uc_se, 952 unsigned int value, bool user_defined) 953 { 954 uc_se->value = value; 955 uc_se->bucket_id = uclamp_bucket_id(value); 956 uc_se->user_defined = user_defined; 957 } 958 959 static inline unsigned int 960 uclamp_idle_value(struct rq *rq, enum uclamp_id clamp_id, 961 unsigned int clamp_value) 962 { 963 /* 964 * Avoid blocked utilization pushing up the frequency when we go 965 * idle (which drops the max-clamp) by retaining the last known 966 * max-clamp. 967 */ 968 if (clamp_id == UCLAMP_MAX) { 969 rq->uclamp_flags |= UCLAMP_FLAG_IDLE; 970 return clamp_value; 971 } 972 973 return uclamp_none(UCLAMP_MIN); 974 } 975 976 static inline void uclamp_idle_reset(struct rq *rq, enum uclamp_id clamp_id, 977 unsigned int clamp_value) 978 { 979 /* Reset max-clamp retention only on idle exit */ 980 if (!(rq->uclamp_flags & UCLAMP_FLAG_IDLE)) 981 return; 982 983 WRITE_ONCE(rq->uclamp[clamp_id].value, clamp_value); 984 } 985 986 static inline 987 unsigned int uclamp_rq_max_value(struct rq *rq, enum uclamp_id clamp_id, 988 unsigned int clamp_value) 989 { 990 struct uclamp_bucket *bucket = rq->uclamp[clamp_id].bucket; 991 int bucket_id = UCLAMP_BUCKETS - 1; 992 993 /* 994 * Since both min and max clamps are max aggregated, find the 995 * top most bucket with tasks in. 996 */ 997 for ( ; bucket_id >= 0; bucket_id--) { 998 if (!bucket[bucket_id].tasks) 999 continue; 1000 return bucket[bucket_id].value; 1001 } 1002 1003 /* No tasks -- default clamp values */ 1004 return uclamp_idle_value(rq, clamp_id, clamp_value); 1005 } 1006 1007 static void __uclamp_update_util_min_rt_default(struct task_struct *p) 1008 { 1009 unsigned int default_util_min; 1010 struct uclamp_se *uc_se; 1011 1012 lockdep_assert_held(&p->pi_lock); 1013 1014 uc_se = &p->uclamp_req[UCLAMP_MIN]; 1015 1016 /* Only sync if user didn't override the default */ 1017 if (uc_se->user_defined) 1018 return; 1019 1020 default_util_min = sysctl_sched_uclamp_util_min_rt_default; 1021 uclamp_se_set(uc_se, default_util_min, false); 1022 } 1023 1024 static void uclamp_update_util_min_rt_default(struct task_struct *p) 1025 { 1026 struct rq_flags rf; 1027 struct rq *rq; 1028 1029 if (!rt_task(p)) 1030 return; 1031 1032 /* Protect updates to p->uclamp_* */ 1033 rq = task_rq_lock(p, &rf); 1034 __uclamp_update_util_min_rt_default(p); 1035 task_rq_unlock(rq, p, &rf); 1036 } 1037 1038 static void uclamp_sync_util_min_rt_default(void) 1039 { 1040 struct task_struct *g, *p; 1041 1042 /* 1043 * copy_process() sysctl_uclamp 1044 * uclamp_min_rt = X; 1045 * write_lock(&tasklist_lock) read_lock(&tasklist_lock) 1046 * // link thread smp_mb__after_spinlock() 1047 * write_unlock(&tasklist_lock) read_unlock(&tasklist_lock); 1048 * sched_post_fork() for_each_process_thread() 1049 * __uclamp_sync_rt() __uclamp_sync_rt() 1050 * 1051 * Ensures that either sched_post_fork() will observe the new 1052 * uclamp_min_rt or for_each_process_thread() will observe the new 1053 * task. 1054 */ 1055 read_lock(&tasklist_lock); 1056 smp_mb__after_spinlock(); 1057 read_unlock(&tasklist_lock); 1058 1059 rcu_read_lock(); 1060 for_each_process_thread(g, p) 1061 uclamp_update_util_min_rt_default(p); 1062 rcu_read_unlock(); 1063 } 1064 1065 static inline struct uclamp_se 1066 uclamp_tg_restrict(struct task_struct *p, enum uclamp_id clamp_id) 1067 { 1068 struct uclamp_se uc_req = p->uclamp_req[clamp_id]; 1069 #ifdef CONFIG_UCLAMP_TASK_GROUP 1070 struct uclamp_se uc_max; 1071 1072 /* 1073 * Tasks in autogroups or root task group will be 1074 * restricted by system defaults. 1075 */ 1076 if (task_group_is_autogroup(task_group(p))) 1077 return uc_req; 1078 if (task_group(p) == &root_task_group) 1079 return uc_req; 1080 1081 uc_max = task_group(p)->uclamp[clamp_id]; 1082 if (uc_req.value > uc_max.value || !uc_req.user_defined) 1083 return uc_max; 1084 #endif 1085 1086 return uc_req; 1087 } 1088 1089 /* 1090 * The effective clamp bucket index of a task depends on, by increasing 1091 * priority: 1092 * - the task specific clamp value, when explicitly requested from userspace 1093 * - the task group effective clamp value, for tasks not either in the root 1094 * group or in an autogroup 1095 * - the system default clamp value, defined by the sysadmin 1096 */ 1097 static inline struct uclamp_se 1098 uclamp_eff_get(struct task_struct *p, enum uclamp_id clamp_id) 1099 { 1100 struct uclamp_se uc_req = uclamp_tg_restrict(p, clamp_id); 1101 struct uclamp_se uc_max = uclamp_default[clamp_id]; 1102 1103 /* System default restrictions always apply */ 1104 if (unlikely(uc_req.value > uc_max.value)) 1105 return uc_max; 1106 1107 return uc_req; 1108 } 1109 1110 unsigned long uclamp_eff_value(struct task_struct *p, enum uclamp_id clamp_id) 1111 { 1112 struct uclamp_se uc_eff; 1113 1114 /* Task currently refcounted: use back-annotated (effective) value */ 1115 if (p->uclamp[clamp_id].active) 1116 return (unsigned long)p->uclamp[clamp_id].value; 1117 1118 uc_eff = uclamp_eff_get(p, clamp_id); 1119 1120 return (unsigned long)uc_eff.value; 1121 } 1122 1123 /* 1124 * When a task is enqueued on a rq, the clamp bucket currently defined by the 1125 * task's uclamp::bucket_id is refcounted on that rq. This also immediately 1126 * updates the rq's clamp value if required. 1127 * 1128 * Tasks can have a task-specific value requested from user-space, track 1129 * within each bucket the maximum value for tasks refcounted in it. 1130 * This "local max aggregation" allows to track the exact "requested" value 1131 * for each bucket when all its RUNNABLE tasks require the same clamp. 1132 */ 1133 static inline void uclamp_rq_inc_id(struct rq *rq, struct task_struct *p, 1134 enum uclamp_id clamp_id) 1135 { 1136 struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id]; 1137 struct uclamp_se *uc_se = &p->uclamp[clamp_id]; 1138 struct uclamp_bucket *bucket; 1139 1140 lockdep_assert_held(&rq->lock); 1141 1142 /* Update task effective clamp */ 1143 p->uclamp[clamp_id] = uclamp_eff_get(p, clamp_id); 1144 1145 bucket = &uc_rq->bucket[uc_se->bucket_id]; 1146 bucket->tasks++; 1147 uc_se->active = true; 1148 1149 uclamp_idle_reset(rq, clamp_id, uc_se->value); 1150 1151 /* 1152 * Local max aggregation: rq buckets always track the max 1153 * "requested" clamp value of its RUNNABLE tasks. 1154 */ 1155 if (bucket->tasks == 1 || uc_se->value > bucket->value) 1156 bucket->value = uc_se->value; 1157 1158 if (uc_se->value > READ_ONCE(uc_rq->value)) 1159 WRITE_ONCE(uc_rq->value, uc_se->value); 1160 } 1161 1162 /* 1163 * When a task is dequeued from a rq, the clamp bucket refcounted by the task 1164 * is released. If this is the last task reference counting the rq's max 1165 * active clamp value, then the rq's clamp value is updated. 1166 * 1167 * Both refcounted tasks and rq's cached clamp values are expected to be 1168 * always valid. If it's detected they are not, as defensive programming, 1169 * enforce the expected state and warn. 1170 */ 1171 static inline void uclamp_rq_dec_id(struct rq *rq, struct task_struct *p, 1172 enum uclamp_id clamp_id) 1173 { 1174 struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id]; 1175 struct uclamp_se *uc_se = &p->uclamp[clamp_id]; 1176 struct uclamp_bucket *bucket; 1177 unsigned int bkt_clamp; 1178 unsigned int rq_clamp; 1179 1180 lockdep_assert_held(&rq->lock); 1181 1182 /* 1183 * If sched_uclamp_used was enabled after task @p was enqueued, 1184 * we could end up with unbalanced call to uclamp_rq_dec_id(). 1185 * 1186 * In this case the uc_se->active flag should be false since no uclamp 1187 * accounting was performed at enqueue time and we can just return 1188 * here. 1189 * 1190 * Need to be careful of the following enqeueue/dequeue ordering 1191 * problem too 1192 * 1193 * enqueue(taskA) 1194 * // sched_uclamp_used gets enabled 1195 * enqueue(taskB) 1196 * dequeue(taskA) 1197 * // Must not decrement bukcet->tasks here 1198 * dequeue(taskB) 1199 * 1200 * where we could end up with stale data in uc_se and 1201 * bucket[uc_se->bucket_id]. 1202 * 1203 * The following check here eliminates the possibility of such race. 1204 */ 1205 if (unlikely(!uc_se->active)) 1206 return; 1207 1208 bucket = &uc_rq->bucket[uc_se->bucket_id]; 1209 1210 SCHED_WARN_ON(!bucket->tasks); 1211 if (likely(bucket->tasks)) 1212 bucket->tasks--; 1213 1214 uc_se->active = false; 1215 1216 /* 1217 * Keep "local max aggregation" simple and accept to (possibly) 1218 * overboost some RUNNABLE tasks in the same bucket. 1219 * The rq clamp bucket value is reset to its base value whenever 1220 * there are no more RUNNABLE tasks refcounting it. 1221 */ 1222 if (likely(bucket->tasks)) 1223 return; 1224 1225 rq_clamp = READ_ONCE(uc_rq->value); 1226 /* 1227 * Defensive programming: this should never happen. If it happens, 1228 * e.g. due to future modification, warn and fixup the expected value. 1229 */ 1230 SCHED_WARN_ON(bucket->value > rq_clamp); 1231 if (bucket->value >= rq_clamp) { 1232 bkt_clamp = uclamp_rq_max_value(rq, clamp_id, uc_se->value); 1233 WRITE_ONCE(uc_rq->value, bkt_clamp); 1234 } 1235 } 1236 1237 static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p) 1238 { 1239 enum uclamp_id clamp_id; 1240 1241 /* 1242 * Avoid any overhead until uclamp is actually used by the userspace. 1243 * 1244 * The condition is constructed such that a NOP is generated when 1245 * sched_uclamp_used is disabled. 1246 */ 1247 if (!static_branch_unlikely(&sched_uclamp_used)) 1248 return; 1249 1250 if (unlikely(!p->sched_class->uclamp_enabled)) 1251 return; 1252 1253 for_each_clamp_id(clamp_id) 1254 uclamp_rq_inc_id(rq, p, clamp_id); 1255 1256 /* Reset clamp idle holding when there is one RUNNABLE task */ 1257 if (rq->uclamp_flags & UCLAMP_FLAG_IDLE) 1258 rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE; 1259 } 1260 1261 static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) 1262 { 1263 enum uclamp_id clamp_id; 1264 1265 /* 1266 * Avoid any overhead until uclamp is actually used by the userspace. 1267 * 1268 * The condition is constructed such that a NOP is generated when 1269 * sched_uclamp_used is disabled. 1270 */ 1271 if (!static_branch_unlikely(&sched_uclamp_used)) 1272 return; 1273 1274 if (unlikely(!p->sched_class->uclamp_enabled)) 1275 return; 1276 1277 for_each_clamp_id(clamp_id) 1278 uclamp_rq_dec_id(rq, p, clamp_id); 1279 } 1280 1281 static inline void 1282 uclamp_update_active(struct task_struct *p, enum uclamp_id clamp_id) 1283 { 1284 struct rq_flags rf; 1285 struct rq *rq; 1286 1287 /* 1288 * Lock the task and the rq where the task is (or was) queued. 1289 * 1290 * We might lock the (previous) rq of a !RUNNABLE task, but that's the 1291 * price to pay to safely serialize util_{min,max} updates with 1292 * enqueues, dequeues and migration operations. 1293 * This is the same locking schema used by __set_cpus_allowed_ptr(). 1294 */ 1295 rq = task_rq_lock(p, &rf); 1296 1297 /* 1298 * Setting the clamp bucket is serialized by task_rq_lock(). 1299 * If the task is not yet RUNNABLE and its task_struct is not 1300 * affecting a valid clamp bucket, the next time it's enqueued, 1301 * it will already see the updated clamp bucket value. 1302 */ 1303 if (p->uclamp[clamp_id].active) { 1304 uclamp_rq_dec_id(rq, p, clamp_id); 1305 uclamp_rq_inc_id(rq, p, clamp_id); 1306 } 1307 1308 task_rq_unlock(rq, p, &rf); 1309 } 1310 1311 #ifdef CONFIG_UCLAMP_TASK_GROUP 1312 static inline void 1313 uclamp_update_active_tasks(struct cgroup_subsys_state *css, 1314 unsigned int clamps) 1315 { 1316 enum uclamp_id clamp_id; 1317 struct css_task_iter it; 1318 struct task_struct *p; 1319 1320 css_task_iter_start(css, 0, &it); 1321 while ((p = css_task_iter_next(&it))) { 1322 for_each_clamp_id(clamp_id) { 1323 if ((0x1 << clamp_id) & clamps) 1324 uclamp_update_active(p, clamp_id); 1325 } 1326 } 1327 css_task_iter_end(&it); 1328 } 1329 1330 static void cpu_util_update_eff(struct cgroup_subsys_state *css); 1331 static void uclamp_update_root_tg(void) 1332 { 1333 struct task_group *tg = &root_task_group; 1334 1335 uclamp_se_set(&tg->uclamp_req[UCLAMP_MIN], 1336 sysctl_sched_uclamp_util_min, false); 1337 uclamp_se_set(&tg->uclamp_req[UCLAMP_MAX], 1338 sysctl_sched_uclamp_util_max, false); 1339 1340 rcu_read_lock(); 1341 cpu_util_update_eff(&root_task_group.css); 1342 rcu_read_unlock(); 1343 } 1344 #else 1345 static void uclamp_update_root_tg(void) { } 1346 #endif 1347 1348 int sysctl_sched_uclamp_handler(struct ctl_table *table, int write, 1349 void *buffer, size_t *lenp, loff_t *ppos) 1350 { 1351 bool update_root_tg = false; 1352 int old_min, old_max, old_min_rt; 1353 int result; 1354 1355 mutex_lock(&uclamp_mutex); 1356 old_min = sysctl_sched_uclamp_util_min; 1357 old_max = sysctl_sched_uclamp_util_max; 1358 old_min_rt = sysctl_sched_uclamp_util_min_rt_default; 1359 1360 result = proc_dointvec(table, write, buffer, lenp, ppos); 1361 if (result) 1362 goto undo; 1363 if (!write) 1364 goto done; 1365 1366 if (sysctl_sched_uclamp_util_min > sysctl_sched_uclamp_util_max || 1367 sysctl_sched_uclamp_util_max > SCHED_CAPACITY_SCALE || 1368 sysctl_sched_uclamp_util_min_rt_default > SCHED_CAPACITY_SCALE) { 1369 1370 result = -EINVAL; 1371 goto undo; 1372 } 1373 1374 if (old_min != sysctl_sched_uclamp_util_min) { 1375 uclamp_se_set(&uclamp_default[UCLAMP_MIN], 1376 sysctl_sched_uclamp_util_min, false); 1377 update_root_tg = true; 1378 } 1379 if (old_max != sysctl_sched_uclamp_util_max) { 1380 uclamp_se_set(&uclamp_default[UCLAMP_MAX], 1381 sysctl_sched_uclamp_util_max, false); 1382 update_root_tg = true; 1383 } 1384 1385 if (update_root_tg) { 1386 static_branch_enable(&sched_uclamp_used); 1387 uclamp_update_root_tg(); 1388 } 1389 1390 if (old_min_rt != sysctl_sched_uclamp_util_min_rt_default) { 1391 static_branch_enable(&sched_uclamp_used); 1392 uclamp_sync_util_min_rt_default(); 1393 } 1394 1395 /* 1396 * We update all RUNNABLE tasks only when task groups are in use. 1397 * Otherwise, keep it simple and do just a lazy update at each next 1398 * task enqueue time. 1399 */ 1400 1401 goto done; 1402 1403 undo: 1404 sysctl_sched_uclamp_util_min = old_min; 1405 sysctl_sched_uclamp_util_max = old_max; 1406 sysctl_sched_uclamp_util_min_rt_default = old_min_rt; 1407 done: 1408 mutex_unlock(&uclamp_mutex); 1409 1410 return result; 1411 } 1412 1413 static int uclamp_validate(struct task_struct *p, 1414 const struct sched_attr *attr) 1415 { 1416 unsigned int lower_bound = p->uclamp_req[UCLAMP_MIN].value; 1417 unsigned int upper_bound = p->uclamp_req[UCLAMP_MAX].value; 1418 1419 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MIN) 1420 lower_bound = attr->sched_util_min; 1421 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MAX) 1422 upper_bound = attr->sched_util_max; 1423 1424 if (lower_bound > upper_bound) 1425 return -EINVAL; 1426 if (upper_bound > SCHED_CAPACITY_SCALE) 1427 return -EINVAL; 1428 1429 /* 1430 * We have valid uclamp attributes; make sure uclamp is enabled. 1431 * 1432 * We need to do that here, because enabling static branches is a 1433 * blocking operation which obviously cannot be done while holding 1434 * scheduler locks. 1435 */ 1436 static_branch_enable(&sched_uclamp_used); 1437 1438 return 0; 1439 } 1440 1441 static void __setscheduler_uclamp(struct task_struct *p, 1442 const struct sched_attr *attr) 1443 { 1444 enum uclamp_id clamp_id; 1445 1446 /* 1447 * On scheduling class change, reset to default clamps for tasks 1448 * without a task-specific value. 1449 */ 1450 for_each_clamp_id(clamp_id) { 1451 struct uclamp_se *uc_se = &p->uclamp_req[clamp_id]; 1452 1453 /* Keep using defined clamps across class changes */ 1454 if (uc_se->user_defined) 1455 continue; 1456 1457 /* 1458 * RT by default have a 100% boost value that could be modified 1459 * at runtime. 1460 */ 1461 if (unlikely(rt_task(p) && clamp_id == UCLAMP_MIN)) 1462 __uclamp_update_util_min_rt_default(p); 1463 else 1464 uclamp_se_set(uc_se, uclamp_none(clamp_id), false); 1465 1466 } 1467 1468 if (likely(!(attr->sched_flags & SCHED_FLAG_UTIL_CLAMP))) 1469 return; 1470 1471 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MIN) { 1472 uclamp_se_set(&p->uclamp_req[UCLAMP_MIN], 1473 attr->sched_util_min, true); 1474 } 1475 1476 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MAX) { 1477 uclamp_se_set(&p->uclamp_req[UCLAMP_MAX], 1478 attr->sched_util_max, true); 1479 } 1480 } 1481 1482 static void uclamp_fork(struct task_struct *p) 1483 { 1484 enum uclamp_id clamp_id; 1485 1486 /* 1487 * We don't need to hold task_rq_lock() when updating p->uclamp_* here 1488 * as the task is still at its early fork stages. 1489 */ 1490 for_each_clamp_id(clamp_id) 1491 p->uclamp[clamp_id].active = false; 1492 1493 if (likely(!p->sched_reset_on_fork)) 1494 return; 1495 1496 for_each_clamp_id(clamp_id) { 1497 uclamp_se_set(&p->uclamp_req[clamp_id], 1498 uclamp_none(clamp_id), false); 1499 } 1500 } 1501 1502 static void uclamp_post_fork(struct task_struct *p) 1503 { 1504 uclamp_update_util_min_rt_default(p); 1505 } 1506 1507 static void __init init_uclamp_rq(struct rq *rq) 1508 { 1509 enum uclamp_id clamp_id; 1510 struct uclamp_rq *uc_rq = rq->uclamp; 1511 1512 for_each_clamp_id(clamp_id) { 1513 uc_rq[clamp_id] = (struct uclamp_rq) { 1514 .value = uclamp_none(clamp_id) 1515 }; 1516 } 1517 1518 rq->uclamp_flags = 0; 1519 } 1520 1521 static void __init init_uclamp(void) 1522 { 1523 struct uclamp_se uc_max = {}; 1524 enum uclamp_id clamp_id; 1525 int cpu; 1526 1527 for_each_possible_cpu(cpu) 1528 init_uclamp_rq(cpu_rq(cpu)); 1529 1530 for_each_clamp_id(clamp_id) { 1531 uclamp_se_set(&init_task.uclamp_req[clamp_id], 1532 uclamp_none(clamp_id), false); 1533 } 1534 1535 /* System defaults allow max clamp values for both indexes */ 1536 uclamp_se_set(&uc_max, uclamp_none(UCLAMP_MAX), false); 1537 for_each_clamp_id(clamp_id) { 1538 uclamp_default[clamp_id] = uc_max; 1539 #ifdef CONFIG_UCLAMP_TASK_GROUP 1540 root_task_group.uclamp_req[clamp_id] = uc_max; 1541 root_task_group.uclamp[clamp_id] = uc_max; 1542 #endif 1543 } 1544 } 1545 1546 #else /* CONFIG_UCLAMP_TASK */ 1547 static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p) { } 1548 static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) { } 1549 static inline int uclamp_validate(struct task_struct *p, 1550 const struct sched_attr *attr) 1551 { 1552 return -EOPNOTSUPP; 1553 } 1554 static void __setscheduler_uclamp(struct task_struct *p, 1555 const struct sched_attr *attr) { } 1556 static inline void uclamp_fork(struct task_struct *p) { } 1557 static inline void uclamp_post_fork(struct task_struct *p) { } 1558 static inline void init_uclamp(void) { } 1559 #endif /* CONFIG_UCLAMP_TASK */ 1560 1561 static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags) 1562 { 1563 if (!(flags & ENQUEUE_NOCLOCK)) 1564 update_rq_clock(rq); 1565 1566 if (!(flags & ENQUEUE_RESTORE)) { 1567 sched_info_queued(rq, p); 1568 psi_enqueue(p, flags & ENQUEUE_WAKEUP); 1569 } 1570 1571 uclamp_rq_inc(rq, p); 1572 p->sched_class->enqueue_task(rq, p, flags); 1573 } 1574 1575 static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags) 1576 { 1577 if (!(flags & DEQUEUE_NOCLOCK)) 1578 update_rq_clock(rq); 1579 1580 if (!(flags & DEQUEUE_SAVE)) { 1581 sched_info_dequeued(rq, p); 1582 psi_dequeue(p, flags & DEQUEUE_SLEEP); 1583 } 1584 1585 uclamp_rq_dec(rq, p); 1586 p->sched_class->dequeue_task(rq, p, flags); 1587 } 1588 1589 void activate_task(struct rq *rq, struct task_struct *p, int flags) 1590 { 1591 enqueue_task(rq, p, flags); 1592 1593 p->on_rq = TASK_ON_RQ_QUEUED; 1594 } 1595 1596 void deactivate_task(struct rq *rq, struct task_struct *p, int flags) 1597 { 1598 p->on_rq = (flags & DEQUEUE_SLEEP) ? 0 : TASK_ON_RQ_MIGRATING; 1599 1600 dequeue_task(rq, p, flags); 1601 } 1602 1603 /* 1604 * __normal_prio - return the priority that is based on the static prio 1605 */ 1606 static inline int __normal_prio(struct task_struct *p) 1607 { 1608 return p->static_prio; 1609 } 1610 1611 /* 1612 * Calculate the expected normal priority: i.e. priority 1613 * without taking RT-inheritance into account. Might be 1614 * boosted by interactivity modifiers. Changes upon fork, 1615 * setprio syscalls, and whenever the interactivity 1616 * estimator recalculates. 1617 */ 1618 static inline int normal_prio(struct task_struct *p) 1619 { 1620 int prio; 1621 1622 if (task_has_dl_policy(p)) 1623 prio = MAX_DL_PRIO-1; 1624 else if (task_has_rt_policy(p)) 1625 prio = MAX_RT_PRIO-1 - p->rt_priority; 1626 else 1627 prio = __normal_prio(p); 1628 return prio; 1629 } 1630 1631 /* 1632 * Calculate the current priority, i.e. the priority 1633 * taken into account by the scheduler. This value might 1634 * be boosted by RT tasks, or might be boosted by 1635 * interactivity modifiers. Will be RT if the task got 1636 * RT-boosted. If not then it returns p->normal_prio. 1637 */ 1638 static int effective_prio(struct task_struct *p) 1639 { 1640 p->normal_prio = normal_prio(p); 1641 /* 1642 * If we are RT tasks or we were boosted to RT priority, 1643 * keep the priority unchanged. Otherwise, update priority 1644 * to the normal priority: 1645 */ 1646 if (!rt_prio(p->prio)) 1647 return p->normal_prio; 1648 return p->prio; 1649 } 1650 1651 /** 1652 * task_curr - is this task currently executing on a CPU? 1653 * @p: the task in question. 1654 * 1655 * Return: 1 if the task is currently executing. 0 otherwise. 1656 */ 1657 inline int task_curr(const struct task_struct *p) 1658 { 1659 return cpu_curr(task_cpu(p)) == p; 1660 } 1661 1662 /* 1663 * switched_from, switched_to and prio_changed must _NOT_ drop rq->lock, 1664 * use the balance_callback list if you want balancing. 1665 * 1666 * this means any call to check_class_changed() must be followed by a call to 1667 * balance_callback(). 1668 */ 1669 static inline void check_class_changed(struct rq *rq, struct task_struct *p, 1670 const struct sched_class *prev_class, 1671 int oldprio) 1672 { 1673 if (prev_class != p->sched_class) { 1674 if (prev_class->switched_from) 1675 prev_class->switched_from(rq, p); 1676 1677 p->sched_class->switched_to(rq, p); 1678 } else if (oldprio != p->prio || dl_task(p)) 1679 p->sched_class->prio_changed(rq, p, oldprio); 1680 } 1681 1682 void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags) 1683 { 1684 if (p->sched_class == rq->curr->sched_class) 1685 rq->curr->sched_class->check_preempt_curr(rq, p, flags); 1686 else if (p->sched_class > rq->curr->sched_class) 1687 resched_curr(rq); 1688 1689 /* 1690 * A queue event has occurred, and we're going to schedule. In 1691 * this case, we can save a useless back to back clock update. 1692 */ 1693 if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr)) 1694 rq_clock_skip_update(rq); 1695 } 1696 1697 #ifdef CONFIG_SMP 1698 1699 /* 1700 * Per-CPU kthreads are allowed to run on !active && online CPUs, see 1701 * __set_cpus_allowed_ptr() and select_fallback_rq(). 1702 */ 1703 static inline bool is_cpu_allowed(struct task_struct *p, int cpu) 1704 { 1705 if (!cpumask_test_cpu(cpu, p->cpus_ptr)) 1706 return false; 1707 1708 if (is_per_cpu_kthread(p)) 1709 return cpu_online(cpu); 1710 1711 return cpu_active(cpu); 1712 } 1713 1714 /* 1715 * This is how migration works: 1716 * 1717 * 1) we invoke migration_cpu_stop() on the target CPU using 1718 * stop_one_cpu(). 1719 * 2) stopper starts to run (implicitly forcing the migrated thread 1720 * off the CPU) 1721 * 3) it checks whether the migrated task is still in the wrong runqueue. 1722 * 4) if it's in the wrong runqueue then the migration thread removes 1723 * it and puts it into the right queue. 1724 * 5) stopper completes and stop_one_cpu() returns and the migration 1725 * is done. 1726 */ 1727 1728 /* 1729 * move_queued_task - move a queued task to new rq. 1730 * 1731 * Returns (locked) new rq. Old rq's lock is released. 1732 */ 1733 static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf, 1734 struct task_struct *p, int new_cpu) 1735 { 1736 lockdep_assert_held(&rq->lock); 1737 1738 deactivate_task(rq, p, DEQUEUE_NOCLOCK); 1739 set_task_cpu(p, new_cpu); 1740 rq_unlock(rq, rf); 1741 1742 rq = cpu_rq(new_cpu); 1743 1744 rq_lock(rq, rf); 1745 BUG_ON(task_cpu(p) != new_cpu); 1746 activate_task(rq, p, 0); 1747 check_preempt_curr(rq, p, 0); 1748 1749 return rq; 1750 } 1751 1752 struct migration_arg { 1753 struct task_struct *task; 1754 int dest_cpu; 1755 }; 1756 1757 /* 1758 * Move (not current) task off this CPU, onto the destination CPU. We're doing 1759 * this because either it can't run here any more (set_cpus_allowed() 1760 * away from this CPU, or CPU going down), or because we're 1761 * attempting to rebalance this task on exec (sched_exec). 1762 * 1763 * So we race with normal scheduler movements, but that's OK, as long 1764 * as the task is no longer on this CPU. 1765 */ 1766 static struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf, 1767 struct task_struct *p, int dest_cpu) 1768 { 1769 /* Affinity changed (again). */ 1770 if (!is_cpu_allowed(p, dest_cpu)) 1771 return rq; 1772 1773 update_rq_clock(rq); 1774 rq = move_queued_task(rq, rf, p, dest_cpu); 1775 1776 return rq; 1777 } 1778 1779 /* 1780 * migration_cpu_stop - this will be executed by a highprio stopper thread 1781 * and performs thread migration by bumping thread off CPU then 1782 * 'pushing' onto another runqueue. 1783 */ 1784 static int migration_cpu_stop(void *data) 1785 { 1786 struct migration_arg *arg = data; 1787 struct task_struct *p = arg->task; 1788 struct rq *rq = this_rq(); 1789 struct rq_flags rf; 1790 1791 /* 1792 * The original target CPU might have gone down and we might 1793 * be on another CPU but it doesn't matter. 1794 */ 1795 local_irq_disable(); 1796 /* 1797 * We need to explicitly wake pending tasks before running 1798 * __migrate_task() such that we will not miss enforcing cpus_ptr 1799 * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test. 1800 */ 1801 flush_smp_call_function_from_idle(); 1802 1803 raw_spin_lock(&p->pi_lock); 1804 rq_lock(rq, &rf); 1805 /* 1806 * If task_rq(p) != rq, it cannot be migrated here, because we're 1807 * holding rq->lock, if p->on_rq == 0 it cannot get enqueued because 1808 * we're holding p->pi_lock. 1809 */ 1810 if (task_rq(p) == rq) { 1811 if (task_on_rq_queued(p)) 1812 rq = __migrate_task(rq, &rf, p, arg->dest_cpu); 1813 else 1814 p->wake_cpu = arg->dest_cpu; 1815 } 1816 rq_unlock(rq, &rf); 1817 raw_spin_unlock(&p->pi_lock); 1818 1819 local_irq_enable(); 1820 return 0; 1821 } 1822 1823 /* 1824 * sched_class::set_cpus_allowed must do the below, but is not required to 1825 * actually call this function. 1826 */ 1827 void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask) 1828 { 1829 cpumask_copy(&p->cpus_mask, new_mask); 1830 p->nr_cpus_allowed = cpumask_weight(new_mask); 1831 } 1832 1833 void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) 1834 { 1835 struct rq *rq = task_rq(p); 1836 bool queued, running; 1837 1838 lockdep_assert_held(&p->pi_lock); 1839 1840 queued = task_on_rq_queued(p); 1841 running = task_current(rq, p); 1842 1843 if (queued) { 1844 /* 1845 * Because __kthread_bind() calls this on blocked tasks without 1846 * holding rq->lock. 1847 */ 1848 lockdep_assert_held(&rq->lock); 1849 dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); 1850 } 1851 if (running) 1852 put_prev_task(rq, p); 1853 1854 p->sched_class->set_cpus_allowed(p, new_mask); 1855 1856 if (queued) 1857 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); 1858 if (running) 1859 set_next_task(rq, p); 1860 } 1861 1862 /* 1863 * Change a given task's CPU affinity. Migrate the thread to a 1864 * proper CPU and schedule it away if the CPU it's executing on 1865 * is removed from the allowed bitmask. 1866 * 1867 * NOTE: the caller must have a valid reference to the task, the 1868 * task must not exit() & deallocate itself prematurely. The 1869 * call is not atomic; no spinlocks may be held. 1870 */ 1871 static int __set_cpus_allowed_ptr(struct task_struct *p, 1872 const struct cpumask *new_mask, bool check) 1873 { 1874 const struct cpumask *cpu_valid_mask = cpu_active_mask; 1875 unsigned int dest_cpu; 1876 struct rq_flags rf; 1877 struct rq *rq; 1878 int ret = 0; 1879 1880 rq = task_rq_lock(p, &rf); 1881 update_rq_clock(rq); 1882 1883 if (p->flags & PF_KTHREAD) { 1884 /* 1885 * Kernel threads are allowed on online && !active CPUs 1886 */ 1887 cpu_valid_mask = cpu_online_mask; 1888 } 1889 1890 /* 1891 * Must re-check here, to close a race against __kthread_bind(), 1892 * sched_setaffinity() is not guaranteed to observe the flag. 1893 */ 1894 if (check && (p->flags & PF_NO_SETAFFINITY)) { 1895 ret = -EINVAL; 1896 goto out; 1897 } 1898 1899 if (cpumask_equal(&p->cpus_mask, new_mask)) 1900 goto out; 1901 1902 /* 1903 * Picking a ~random cpu helps in cases where we are changing affinity 1904 * for groups of tasks (ie. cpuset), so that load balancing is not 1905 * immediately required to distribute the tasks within their new mask. 1906 */ 1907 dest_cpu = cpumask_any_and_distribute(cpu_valid_mask, new_mask); 1908 if (dest_cpu >= nr_cpu_ids) { 1909 ret = -EINVAL; 1910 goto out; 1911 } 1912 1913 do_set_cpus_allowed(p, new_mask); 1914 1915 if (p->flags & PF_KTHREAD) { 1916 /* 1917 * For kernel threads that do indeed end up on online && 1918 * !active we want to ensure they are strict per-CPU threads. 1919 */ 1920 WARN_ON(cpumask_intersects(new_mask, cpu_online_mask) && 1921 !cpumask_intersects(new_mask, cpu_active_mask) && 1922 p->nr_cpus_allowed != 1); 1923 } 1924 1925 /* Can the task run on the task's current CPU? If so, we're done */ 1926 if (cpumask_test_cpu(task_cpu(p), new_mask)) 1927 goto out; 1928 1929 if (task_running(rq, p) || p->state == TASK_WAKING) { 1930 struct migration_arg arg = { p, dest_cpu }; 1931 /* Need help from migration thread: drop lock and wait. */ 1932 task_rq_unlock(rq, p, &rf); 1933 stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg); 1934 return 0; 1935 } else if (task_on_rq_queued(p)) { 1936 /* 1937 * OK, since we're going to drop the lock immediately 1938 * afterwards anyway. 1939 */ 1940 rq = move_queued_task(rq, &rf, p, dest_cpu); 1941 } 1942 out: 1943 task_rq_unlock(rq, p, &rf); 1944 1945 return ret; 1946 } 1947 1948 int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) 1949 { 1950 return __set_cpus_allowed_ptr(p, new_mask, false); 1951 } 1952 EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr); 1953 1954 void set_task_cpu(struct task_struct *p, unsigned int new_cpu) 1955 { 1956 #ifdef CONFIG_SCHED_DEBUG 1957 /* 1958 * We should never call set_task_cpu() on a blocked task, 1959 * ttwu() will sort out the placement. 1960 */ 1961 WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING && 1962 !p->on_rq); 1963 1964 /* 1965 * Migrating fair class task must have p->on_rq = TASK_ON_RQ_MIGRATING, 1966 * because schedstat_wait_{start,end} rebase migrating task's wait_start 1967 * time relying on p->on_rq. 1968 */ 1969 WARN_ON_ONCE(p->state == TASK_RUNNING && 1970 p->sched_class == &fair_sched_class && 1971 (p->on_rq && !task_on_rq_migrating(p))); 1972 1973 #ifdef CONFIG_LOCKDEP 1974 /* 1975 * The caller should hold either p->pi_lock or rq->lock, when changing 1976 * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks. 1977 * 1978 * sched_move_task() holds both and thus holding either pins the cgroup, 1979 * see task_group(). 1980 * 1981 * Furthermore, all task_rq users should acquire both locks, see 1982 * task_rq_lock(). 1983 */ 1984 WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) || 1985 lockdep_is_held(&task_rq(p)->lock))); 1986 #endif 1987 /* 1988 * Clearly, migrating tasks to offline CPUs is a fairly daft thing. 1989 */ 1990 WARN_ON_ONCE(!cpu_online(new_cpu)); 1991 #endif 1992 1993 trace_sched_migrate_task(p, new_cpu); 1994 1995 if (task_cpu(p) != new_cpu) { 1996 if (p->sched_class->migrate_task_rq) 1997 p->sched_class->migrate_task_rq(p, new_cpu); 1998 p->se.nr_migrations++; 1999 rseq_migrate(p); 2000 perf_event_task_migrate(p); 2001 } 2002 2003 __set_task_cpu(p, new_cpu); 2004 } 2005 2006 #ifdef CONFIG_NUMA_BALANCING 2007 static void __migrate_swap_task(struct task_struct *p, int cpu) 2008 { 2009 if (task_on_rq_queued(p)) { 2010 struct rq *src_rq, *dst_rq; 2011 struct rq_flags srf, drf; 2012 2013 src_rq = task_rq(p); 2014 dst_rq = cpu_rq(cpu); 2015 2016 rq_pin_lock(src_rq, &srf); 2017 rq_pin_lock(dst_rq, &drf); 2018 2019 deactivate_task(src_rq, p, 0); 2020 set_task_cpu(p, cpu); 2021 activate_task(dst_rq, p, 0); 2022 check_preempt_curr(dst_rq, p, 0); 2023 2024 rq_unpin_lock(dst_rq, &drf); 2025 rq_unpin_lock(src_rq, &srf); 2026 2027 } else { 2028 /* 2029 * Task isn't running anymore; make it appear like we migrated 2030 * it before it went to sleep. This means on wakeup we make the 2031 * previous CPU our target instead of where it really is. 2032 */ 2033 p->wake_cpu = cpu; 2034 } 2035 } 2036 2037 struct migration_swap_arg { 2038 struct task_struct *src_task, *dst_task; 2039 int src_cpu, dst_cpu; 2040 }; 2041 2042 static int migrate_swap_stop(void *data) 2043 { 2044 struct migration_swap_arg *arg = data; 2045 struct rq *src_rq, *dst_rq; 2046 int ret = -EAGAIN; 2047 2048 if (!cpu_active(arg->src_cpu) || !cpu_active(arg->dst_cpu)) 2049 return -EAGAIN; 2050 2051 src_rq = cpu_rq(arg->src_cpu); 2052 dst_rq = cpu_rq(arg->dst_cpu); 2053 2054 double_raw_lock(&arg->src_task->pi_lock, 2055 &arg->dst_task->pi_lock); 2056 double_rq_lock(src_rq, dst_rq); 2057 2058 if (task_cpu(arg->dst_task) != arg->dst_cpu) 2059 goto unlock; 2060 2061 if (task_cpu(arg->src_task) != arg->src_cpu) 2062 goto unlock; 2063 2064 if (!cpumask_test_cpu(arg->dst_cpu, arg->src_task->cpus_ptr)) 2065 goto unlock; 2066 2067 if (!cpumask_test_cpu(arg->src_cpu, arg->dst_task->cpus_ptr)) 2068 goto unlock; 2069 2070 __migrate_swap_task(arg->src_task, arg->dst_cpu); 2071 __migrate_swap_task(arg->dst_task, arg->src_cpu); 2072 2073 ret = 0; 2074 2075 unlock: 2076 double_rq_unlock(src_rq, dst_rq); 2077 raw_spin_unlock(&arg->dst_task->pi_lock); 2078 raw_spin_unlock(&arg->src_task->pi_lock); 2079 2080 return ret; 2081 } 2082 2083 /* 2084 * Cross migrate two tasks 2085 */ 2086 int migrate_swap(struct task_struct *cur, struct task_struct *p, 2087 int target_cpu, int curr_cpu) 2088 { 2089 struct migration_swap_arg arg; 2090 int ret = -EINVAL; 2091 2092 arg = (struct migration_swap_arg){ 2093 .src_task = cur, 2094 .src_cpu = curr_cpu, 2095 .dst_task = p, 2096 .dst_cpu = target_cpu, 2097 }; 2098 2099 if (arg.src_cpu == arg.dst_cpu) 2100 goto out; 2101 2102 /* 2103 * These three tests are all lockless; this is OK since all of them 2104 * will be re-checked with proper locks held further down the line. 2105 */ 2106 if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu)) 2107 goto out; 2108 2109 if (!cpumask_test_cpu(arg.dst_cpu, arg.src_task->cpus_ptr)) 2110 goto out; 2111 2112 if (!cpumask_test_cpu(arg.src_cpu, arg.dst_task->cpus_ptr)) 2113 goto out; 2114 2115 trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu); 2116 ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg); 2117 2118 out: 2119 return ret; 2120 } 2121 #endif /* CONFIG_NUMA_BALANCING */ 2122 2123 /* 2124 * wait_task_inactive - wait for a thread to unschedule. 2125 * 2126 * If @match_state is nonzero, it's the @p->state value just checked and 2127 * not expected to change. If it changes, i.e. @p might have woken up, 2128 * then return zero. When we succeed in waiting for @p to be off its CPU, 2129 * we return a positive number (its total switch count). If a second call 2130 * a short while later returns the same number, the caller can be sure that 2131 * @p has remained unscheduled the whole time. 2132 * 2133 * The caller must ensure that the task *will* unschedule sometime soon, 2134 * else this function might spin for a *long* time. This function can't 2135 * be called with interrupts off, or it may introduce deadlock with 2136 * smp_call_function() if an IPI is sent by the same process we are 2137 * waiting to become inactive. 2138 */ 2139 unsigned long wait_task_inactive(struct task_struct *p, long match_state) 2140 { 2141 int running, queued; 2142 struct rq_flags rf; 2143 unsigned long ncsw; 2144 struct rq *rq; 2145 2146 for (;;) { 2147 /* 2148 * We do the initial early heuristics without holding 2149 * any task-queue locks at all. We'll only try to get 2150 * the runqueue lock when things look like they will 2151 * work out! 2152 */ 2153 rq = task_rq(p); 2154 2155 /* 2156 * If the task is actively running on another CPU 2157 * still, just relax and busy-wait without holding 2158 * any locks. 2159 * 2160 * NOTE! Since we don't hold any locks, it's not 2161 * even sure that "rq" stays as the right runqueue! 2162 * But we don't care, since "task_running()" will 2163 * return false if the runqueue has changed and p 2164 * is actually now running somewhere else! 2165 */ 2166 while (task_running(rq, p)) { 2167 if (match_state && unlikely(p->state != match_state)) 2168 return 0; 2169 cpu_relax(); 2170 } 2171 2172 /* 2173 * Ok, time to look more closely! We need the rq 2174 * lock now, to be *sure*. If we're wrong, we'll 2175 * just go back and repeat. 2176 */ 2177 rq = task_rq_lock(p, &rf); 2178 trace_sched_wait_task(p); 2179 running = task_running(rq, p); 2180 queued = task_on_rq_queued(p); 2181 ncsw = 0; 2182 if (!match_state || p->state == match_state) 2183 ncsw = p->nvcsw | LONG_MIN; /* sets MSB */ 2184 task_rq_unlock(rq, p, &rf); 2185 2186 /* 2187 * If it changed from the expected state, bail out now. 2188 */ 2189 if (unlikely(!ncsw)) 2190 break; 2191 2192 /* 2193 * Was it really running after all now that we 2194 * checked with the proper locks actually held? 2195 * 2196 * Oops. Go back and try again.. 2197 */ 2198 if (unlikely(running)) { 2199 cpu_relax(); 2200 continue; 2201 } 2202 2203 /* 2204 * It's not enough that it's not actively running, 2205 * it must be off the runqueue _entirely_, and not 2206 * preempted! 2207 * 2208 * So if it was still runnable (but just not actively 2209 * running right now), it's preempted, and we should 2210 * yield - it could be a while. 2211 */ 2212 if (unlikely(queued)) { 2213 ktime_t to = NSEC_PER_SEC / HZ; 2214 2215 set_current_state(TASK_UNINTERRUPTIBLE); 2216 schedule_hrtimeout(&to, HRTIMER_MODE_REL); 2217 continue; 2218 } 2219 2220 /* 2221 * Ahh, all good. It wasn't running, and it wasn't 2222 * runnable, which means that it will never become 2223 * running in the future either. We're all done! 2224 */ 2225 break; 2226 } 2227 2228 return ncsw; 2229 } 2230 2231 /*** 2232 * kick_process - kick a running thread to enter/exit the kernel 2233 * @p: the to-be-kicked thread 2234 * 2235 * Cause a process which is running on another CPU to enter 2236 * kernel-mode, without any delay. (to get signals handled.) 2237 * 2238 * NOTE: this function doesn't have to take the runqueue lock, 2239 * because all it wants to ensure is that the remote task enters 2240 * the kernel. If the IPI races and the task has been migrated 2241 * to another CPU then no harm is done and the purpose has been 2242 * achieved as well. 2243 */ 2244 void kick_process(struct task_struct *p) 2245 { 2246 int cpu; 2247 2248 preempt_disable(); 2249 cpu = task_cpu(p); 2250 if ((cpu != smp_processor_id()) && task_curr(p)) 2251 smp_send_reschedule(cpu); 2252 preempt_enable(); 2253 } 2254 EXPORT_SYMBOL_GPL(kick_process); 2255 2256 /* 2257 * ->cpus_ptr is protected by both rq->lock and p->pi_lock 2258 * 2259 * A few notes on cpu_active vs cpu_online: 2260 * 2261 * - cpu_active must be a subset of cpu_online 2262 * 2263 * - on CPU-up we allow per-CPU kthreads on the online && !active CPU, 2264 * see __set_cpus_allowed_ptr(). At this point the newly online 2265 * CPU isn't yet part of the sched domains, and balancing will not 2266 * see it. 2267 * 2268 * - on CPU-down we clear cpu_active() to mask the sched domains and 2269 * avoid the load balancer to place new tasks on the to be removed 2270 * CPU. Existing tasks will remain running there and will be taken 2271 * off. 2272 * 2273 * This means that fallback selection must not select !active CPUs. 2274 * And can assume that any active CPU must be online. Conversely 2275 * select_task_rq() below may allow selection of !active CPUs in order 2276 * to satisfy the above rules. 2277 */ 2278 static int select_fallback_rq(int cpu, struct task_struct *p) 2279 { 2280 int nid = cpu_to_node(cpu); 2281 const struct cpumask *nodemask = NULL; 2282 enum { cpuset, possible, fail } state = cpuset; 2283 int dest_cpu; 2284 2285 /* 2286 * If the node that the CPU is on has been offlined, cpu_to_node() 2287 * will return -1. There is no CPU on the node, and we should 2288 * select the CPU on the other node. 2289 */ 2290 if (nid != -1) { 2291 nodemask = cpumask_of_node(nid); 2292 2293 /* Look for allowed, online CPU in same node. */ 2294 for_each_cpu(dest_cpu, nodemask) { 2295 if (!cpu_active(dest_cpu)) 2296 continue; 2297 if (cpumask_test_cpu(dest_cpu, p->cpus_ptr)) 2298 return dest_cpu; 2299 } 2300 } 2301 2302 for (;;) { 2303 /* Any allowed, online CPU? */ 2304 for_each_cpu(dest_cpu, p->cpus_ptr) { 2305 if (!is_cpu_allowed(p, dest_cpu)) 2306 continue; 2307 2308 goto out; 2309 } 2310 2311 /* No more Mr. Nice Guy. */ 2312 switch (state) { 2313 case cpuset: 2314 if (IS_ENABLED(CONFIG_CPUSETS)) { 2315 cpuset_cpus_allowed_fallback(p); 2316 state = possible; 2317 break; 2318 } 2319 fallthrough; 2320 case possible: 2321 do_set_cpus_allowed(p, cpu_possible_mask); 2322 state = fail; 2323 break; 2324 2325 case fail: 2326 BUG(); 2327 break; 2328 } 2329 } 2330 2331 out: 2332 if (state != cpuset) { 2333 /* 2334 * Don't tell them about moving exiting tasks or 2335 * kernel threads (both mm NULL), since they never 2336 * leave kernel. 2337 */ 2338 if (p->mm && printk_ratelimit()) { 2339 printk_deferred("process %d (%s) no longer affine to cpu%d\n", 2340 task_pid_nr(p), p->comm, cpu); 2341 } 2342 } 2343 2344 return dest_cpu; 2345 } 2346 2347 /* 2348 * The caller (fork, wakeup) owns p->pi_lock, ->cpus_ptr is stable. 2349 */ 2350 static inline 2351 int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags) 2352 { 2353 lockdep_assert_held(&p->pi_lock); 2354 2355 if (p->nr_cpus_allowed > 1) 2356 cpu = p->sched_class->select_task_rq(p, cpu, sd_flags, wake_flags); 2357 else 2358 cpu = cpumask_any(p->cpus_ptr); 2359 2360 /* 2361 * In order not to call set_task_cpu() on a blocking task we need 2362 * to rely on ttwu() to place the task on a valid ->cpus_ptr 2363 * CPU. 2364 * 2365 * Since this is common to all placement strategies, this lives here. 2366 * 2367 * [ this allows ->select_task() to simply return task_cpu(p) and 2368 * not worry about this generic constraint ] 2369 */ 2370 if (unlikely(!is_cpu_allowed(p, cpu))) 2371 cpu = select_fallback_rq(task_cpu(p), p); 2372 2373 return cpu; 2374 } 2375 2376 void sched_set_stop_task(int cpu, struct task_struct *stop) 2377 { 2378 struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 }; 2379 struct task_struct *old_stop = cpu_rq(cpu)->stop; 2380 2381 if (stop) { 2382 /* 2383 * Make it appear like a SCHED_FIFO task, its something 2384 * userspace knows about and won't get confused about. 2385 * 2386 * Also, it will make PI more or less work without too 2387 * much confusion -- but then, stop work should not 2388 * rely on PI working anyway. 2389 */ 2390 sched_setscheduler_nocheck(stop, SCHED_FIFO, ¶m); 2391 2392 stop->sched_class = &stop_sched_class; 2393 } 2394 2395 cpu_rq(cpu)->stop = stop; 2396 2397 if (old_stop) { 2398 /* 2399 * Reset it back to a normal scheduling class so that 2400 * it can die in pieces. 2401 */ 2402 old_stop->sched_class = &rt_sched_class; 2403 } 2404 } 2405 2406 #else 2407 2408 static inline int __set_cpus_allowed_ptr(struct task_struct *p, 2409 const struct cpumask *new_mask, bool check) 2410 { 2411 return set_cpus_allowed_ptr(p, new_mask); 2412 } 2413 2414 #endif /* CONFIG_SMP */ 2415 2416 static void 2417 ttwu_stat(struct task_struct *p, int cpu, int wake_flags) 2418 { 2419 struct rq *rq; 2420 2421 if (!schedstat_enabled()) 2422 return; 2423 2424 rq = this_rq(); 2425 2426 #ifdef CONFIG_SMP 2427 if (cpu == rq->cpu) { 2428 __schedstat_inc(rq->ttwu_local); 2429 __schedstat_inc(p->se.statistics.nr_wakeups_local); 2430 } else { 2431 struct sched_domain *sd; 2432 2433 __schedstat_inc(p->se.statistics.nr_wakeups_remote); 2434 rcu_read_lock(); 2435 for_each_domain(rq->cpu, sd) { 2436 if (cpumask_test_cpu(cpu, sched_domain_span(sd))) { 2437 __schedstat_inc(sd->ttwu_wake_remote); 2438 break; 2439 } 2440 } 2441 rcu_read_unlock(); 2442 } 2443 2444 if (wake_flags & WF_MIGRATED) 2445 __schedstat_inc(p->se.statistics.nr_wakeups_migrate); 2446 #endif /* CONFIG_SMP */ 2447 2448 __schedstat_inc(rq->ttwu_count); 2449 __schedstat_inc(p->se.statistics.nr_wakeups); 2450 2451 if (wake_flags & WF_SYNC) 2452 __schedstat_inc(p->se.statistics.nr_wakeups_sync); 2453 } 2454 2455 /* 2456 * Mark the task runnable and perform wakeup-preemption. 2457 */ 2458 static void ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags, 2459 struct rq_flags *rf) 2460 { 2461 check_preempt_curr(rq, p, wake_flags); 2462 p->state = TASK_RUNNING; 2463 trace_sched_wakeup(p); 2464 2465 #ifdef CONFIG_SMP 2466 if (p->sched_class->task_woken) { 2467 /* 2468 * Our task @p is fully woken up and running; so its safe to 2469 * drop the rq->lock, hereafter rq is only used for statistics. 2470 */ 2471 rq_unpin_lock(rq, rf); 2472 p->sched_class->task_woken(rq, p); 2473 rq_repin_lock(rq, rf); 2474 } 2475 2476 if (rq->idle_stamp) { 2477 u64 delta = rq_clock(rq) - rq->idle_stamp; 2478 u64 max = 2*rq->max_idle_balance_cost; 2479 2480 update_avg(&rq->avg_idle, delta); 2481 2482 if (rq->avg_idle > max) 2483 rq->avg_idle = max; 2484 2485 rq->idle_stamp = 0; 2486 } 2487 #endif 2488 } 2489 2490 static void 2491 ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags, 2492 struct rq_flags *rf) 2493 { 2494 int en_flags = ENQUEUE_WAKEUP | ENQUEUE_NOCLOCK; 2495 2496 lockdep_assert_held(&rq->lock); 2497 2498 if (p->sched_contributes_to_load) 2499 rq->nr_uninterruptible--; 2500 2501 #ifdef CONFIG_SMP 2502 if (wake_flags & WF_MIGRATED) 2503 en_flags |= ENQUEUE_MIGRATED; 2504 #endif 2505 2506 activate_task(rq, p, en_flags); 2507 ttwu_do_wakeup(rq, p, wake_flags, rf); 2508 } 2509 2510 /* 2511 * Consider @p being inside a wait loop: 2512 * 2513 * for (;;) { 2514 * set_current_state(TASK_UNINTERRUPTIBLE); 2515 * 2516 * if (CONDITION) 2517 * break; 2518 * 2519 * schedule(); 2520 * } 2521 * __set_current_state(TASK_RUNNING); 2522 * 2523 * between set_current_state() and schedule(). In this case @p is still 2524 * runnable, so all that needs doing is change p->state back to TASK_RUNNING in 2525 * an atomic manner. 2526 * 2527 * By taking task_rq(p)->lock we serialize against schedule(), if @p->on_rq 2528 * then schedule() must still happen and p->state can be changed to 2529 * TASK_RUNNING. Otherwise we lost the race, schedule() has happened, and we 2530 * need to do a full wakeup with enqueue. 2531 * 2532 * Returns: %true when the wakeup is done, 2533 * %false otherwise. 2534 */ 2535 static int ttwu_runnable(struct task_struct *p, int wake_flags) 2536 { 2537 struct rq_flags rf; 2538 struct rq *rq; 2539 int ret = 0; 2540 2541 rq = __task_rq_lock(p, &rf); 2542 if (task_on_rq_queued(p)) { 2543 /* check_preempt_curr() may use rq clock */ 2544 update_rq_clock(rq); 2545 ttwu_do_wakeup(rq, p, wake_flags, &rf); 2546 ret = 1; 2547 } 2548 __task_rq_unlock(rq, &rf); 2549 2550 return ret; 2551 } 2552 2553 #ifdef CONFIG_SMP 2554 void sched_ttwu_pending(void *arg) 2555 { 2556 struct llist_node *llist = arg; 2557 struct rq *rq = this_rq(); 2558 struct task_struct *p, *t; 2559 struct rq_flags rf; 2560 2561 if (!llist) 2562 return; 2563 2564 /* 2565 * rq::ttwu_pending racy indication of out-standing wakeups. 2566 * Races such that false-negatives are possible, since they 2567 * are shorter lived that false-positives would be. 2568 */ 2569 WRITE_ONCE(rq->ttwu_pending, 0); 2570 2571 rq_lock_irqsave(rq, &rf); 2572 update_rq_clock(rq); 2573 2574 llist_for_each_entry_safe(p, t, llist, wake_entry.llist) { 2575 if (WARN_ON_ONCE(p->on_cpu)) 2576 smp_cond_load_acquire(&p->on_cpu, !VAL); 2577 2578 if (WARN_ON_ONCE(task_cpu(p) != cpu_of(rq))) 2579 set_task_cpu(p, cpu_of(rq)); 2580 2581 ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0, &rf); 2582 } 2583 2584 rq_unlock_irqrestore(rq, &rf); 2585 } 2586 2587 void send_call_function_single_ipi(int cpu) 2588 { 2589 struct rq *rq = cpu_rq(cpu); 2590 2591 if (!set_nr_if_polling(rq->idle)) 2592 arch_send_call_function_single_ipi(cpu); 2593 else 2594 trace_sched_wake_idle_without_ipi(cpu); 2595 } 2596 2597 /* 2598 * Queue a task on the target CPUs wake_list and wake the CPU via IPI if 2599 * necessary. The wakee CPU on receipt of the IPI will queue the task 2600 * via sched_ttwu_wakeup() for activation so the wakee incurs the cost 2601 * of the wakeup instead of the waker. 2602 */ 2603 static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) 2604 { 2605 struct rq *rq = cpu_rq(cpu); 2606 2607 p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED); 2608 2609 WRITE_ONCE(rq->ttwu_pending, 1); 2610 __smp_call_single_queue(cpu, &p->wake_entry.llist); 2611 } 2612 2613 void wake_up_if_idle(int cpu) 2614 { 2615 struct rq *rq = cpu_rq(cpu); 2616 struct rq_flags rf; 2617 2618 rcu_read_lock(); 2619 2620 if (!is_idle_task(rcu_dereference(rq->curr))) 2621 goto out; 2622 2623 if (set_nr_if_polling(rq->idle)) { 2624 trace_sched_wake_idle_without_ipi(cpu); 2625 } else { 2626 rq_lock_irqsave(rq, &rf); 2627 if (is_idle_task(rq->curr)) 2628 smp_send_reschedule(cpu); 2629 /* Else CPU is not idle, do nothing here: */ 2630 rq_unlock_irqrestore(rq, &rf); 2631 } 2632 2633 out: 2634 rcu_read_unlock(); 2635 } 2636 2637 bool cpus_share_cache(int this_cpu, int that_cpu) 2638 { 2639 return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu); 2640 } 2641 2642 static inline bool ttwu_queue_cond(int cpu, int wake_flags) 2643 { 2644 /* 2645 * If the CPU does not share cache, then queue the task on the 2646 * remote rqs wakelist to avoid accessing remote data. 2647 */ 2648 if (!cpus_share_cache(smp_processor_id(), cpu)) 2649 return true; 2650 2651 /* 2652 * If the task is descheduling and the only running task on the 2653 * CPU then use the wakelist to offload the task activation to 2654 * the soon-to-be-idle CPU as the current CPU is likely busy. 2655 * nr_running is checked to avoid unnecessary task stacking. 2656 */ 2657 if ((wake_flags & WF_ON_CPU) && cpu_rq(cpu)->nr_running <= 1) 2658 return true; 2659 2660 return false; 2661 } 2662 2663 static bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) 2664 { 2665 if (sched_feat(TTWU_QUEUE) && ttwu_queue_cond(cpu, wake_flags)) { 2666 if (WARN_ON_ONCE(cpu == smp_processor_id())) 2667 return false; 2668 2669 sched_clock_cpu(cpu); /* Sync clocks across CPUs */ 2670 __ttwu_queue_wakelist(p, cpu, wake_flags); 2671 return true; 2672 } 2673 2674 return false; 2675 } 2676 2677 #else /* !CONFIG_SMP */ 2678 2679 static inline bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) 2680 { 2681 return false; 2682 } 2683 2684 #endif /* CONFIG_SMP */ 2685 2686 static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags) 2687 { 2688 struct rq *rq = cpu_rq(cpu); 2689 struct rq_flags rf; 2690 2691 if (ttwu_queue_wakelist(p, cpu, wake_flags)) 2692 return; 2693 2694 rq_lock(rq, &rf); 2695 update_rq_clock(rq); 2696 ttwu_do_activate(rq, p, wake_flags, &rf); 2697 rq_unlock(rq, &rf); 2698 } 2699 2700 /* 2701 * Notes on Program-Order guarantees on SMP systems. 2702 * 2703 * MIGRATION 2704 * 2705 * The basic program-order guarantee on SMP systems is that when a task [t] 2706 * migrates, all its activity on its old CPU [c0] happens-before any subsequent 2707 * execution on its new CPU [c1]. 2708 * 2709 * For migration (of runnable tasks) this is provided by the following means: 2710 * 2711 * A) UNLOCK of the rq(c0)->lock scheduling out task t 2712 * B) migration for t is required to synchronize *both* rq(c0)->lock and 2713 * rq(c1)->lock (if not at the same time, then in that order). 2714 * C) LOCK of the rq(c1)->lock scheduling in task 2715 * 2716 * Release/acquire chaining guarantees that B happens after A and C after B. 2717 * Note: the CPU doing B need not be c0 or c1 2718 * 2719 * Example: 2720 * 2721 * CPU0 CPU1 CPU2 2722 * 2723 * LOCK rq(0)->lock 2724 * sched-out X 2725 * sched-in Y 2726 * UNLOCK rq(0)->lock 2727 * 2728 * LOCK rq(0)->lock // orders against CPU0 2729 * dequeue X 2730 * UNLOCK rq(0)->lock 2731 * 2732 * LOCK rq(1)->lock 2733 * enqueue X 2734 * UNLOCK rq(1)->lock 2735 * 2736 * LOCK rq(1)->lock // orders against CPU2 2737 * sched-out Z 2738 * sched-in X 2739 * UNLOCK rq(1)->lock 2740 * 2741 * 2742 * BLOCKING -- aka. SLEEP + WAKEUP 2743 * 2744 * For blocking we (obviously) need to provide the same guarantee as for 2745 * migration. However the means are completely different as there is no lock 2746 * chain to provide order. Instead we do: 2747 * 2748 * 1) smp_store_release(X->on_cpu, 0) -- finish_task() 2749 * 2) smp_cond_load_acquire(!X->on_cpu) -- try_to_wake_up() 2750 * 2751 * Example: 2752 * 2753 * CPU0 (schedule) CPU1 (try_to_wake_up) CPU2 (schedule) 2754 * 2755 * LOCK rq(0)->lock LOCK X->pi_lock 2756 * dequeue X 2757 * sched-out X 2758 * smp_store_release(X->on_cpu, 0); 2759 * 2760 * smp_cond_load_acquire(&X->on_cpu, !VAL); 2761 * X->state = WAKING 2762 * set_task_cpu(X,2) 2763 * 2764 * LOCK rq(2)->lock 2765 * enqueue X 2766 * X->state = RUNNING 2767 * UNLOCK rq(2)->lock 2768 * 2769 * LOCK rq(2)->lock // orders against CPU1 2770 * sched-out Z 2771 * sched-in X 2772 * UNLOCK rq(2)->lock 2773 * 2774 * UNLOCK X->pi_lock 2775 * UNLOCK rq(0)->lock 2776 * 2777 * 2778 * However, for wakeups there is a second guarantee we must provide, namely we 2779 * must ensure that CONDITION=1 done by the caller can not be reordered with 2780 * accesses to the task state; see try_to_wake_up() and set_current_state(). 2781 */ 2782 2783 /** 2784 * try_to_wake_up - wake up a thread 2785 * @p: the thread to be awakened 2786 * @state: the mask of task states that can be woken 2787 * @wake_flags: wake modifier flags (WF_*) 2788 * 2789 * Conceptually does: 2790 * 2791 * If (@state & @p->state) @p->state = TASK_RUNNING. 2792 * 2793 * If the task was not queued/runnable, also place it back on a runqueue. 2794 * 2795 * This function is atomic against schedule() which would dequeue the task. 2796 * 2797 * It issues a full memory barrier before accessing @p->state, see the comment 2798 * with set_current_state(). 2799 * 2800 * Uses p->pi_lock to serialize against concurrent wake-ups. 2801 * 2802 * Relies on p->pi_lock stabilizing: 2803 * - p->sched_class 2804 * - p->cpus_ptr 2805 * - p->sched_task_group 2806 * in order to do migration, see its use of select_task_rq()/set_task_cpu(). 2807 * 2808 * Tries really hard to only take one task_rq(p)->lock for performance. 2809 * Takes rq->lock in: 2810 * - ttwu_runnable() -- old rq, unavoidable, see comment there; 2811 * - ttwu_queue() -- new rq, for enqueue of the task; 2812 * - psi_ttwu_dequeue() -- much sadness :-( accounting will kill us. 2813 * 2814 * As a consequence we race really badly with just about everything. See the 2815 * many memory barriers and their comments for details. 2816 * 2817 * Return: %true if @p->state changes (an actual wakeup was done), 2818 * %false otherwise. 2819 */ 2820 static int 2821 try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) 2822 { 2823 unsigned long flags; 2824 int cpu, success = 0; 2825 2826 preempt_disable(); 2827 if (p == current) { 2828 /* 2829 * We're waking current, this means 'p->on_rq' and 'task_cpu(p) 2830 * == smp_processor_id()'. Together this means we can special 2831 * case the whole 'p->on_rq && ttwu_runnable()' case below 2832 * without taking any locks. 2833 * 2834 * In particular: 2835 * - we rely on Program-Order guarantees for all the ordering, 2836 * - we're serialized against set_special_state() by virtue of 2837 * it disabling IRQs (this allows not taking ->pi_lock). 2838 */ 2839 if (!(p->state & state)) 2840 goto out; 2841 2842 success = 1; 2843 trace_sched_waking(p); 2844 p->state = TASK_RUNNING; 2845 trace_sched_wakeup(p); 2846 goto out; 2847 } 2848 2849 /* 2850 * If we are going to wake up a thread waiting for CONDITION we 2851 * need to ensure that CONDITION=1 done by the caller can not be 2852 * reordered with p->state check below. This pairs with smp_store_mb() 2853 * in set_current_state() that the waiting thread does. 2854 */ 2855 raw_spin_lock_irqsave(&p->pi_lock, flags); 2856 smp_mb__after_spinlock(); 2857 if (!(p->state & state)) 2858 goto unlock; 2859 2860 trace_sched_waking(p); 2861 2862 /* We're going to change ->state: */ 2863 success = 1; 2864 2865 /* 2866 * Ensure we load p->on_rq _after_ p->state, otherwise it would 2867 * be possible to, falsely, observe p->on_rq == 0 and get stuck 2868 * in smp_cond_load_acquire() below. 2869 * 2870 * sched_ttwu_pending() try_to_wake_up() 2871 * STORE p->on_rq = 1 LOAD p->state 2872 * UNLOCK rq->lock 2873 * 2874 * __schedule() (switch to task 'p') 2875 * LOCK rq->lock smp_rmb(); 2876 * smp_mb__after_spinlock(); 2877 * UNLOCK rq->lock 2878 * 2879 * [task p] 2880 * STORE p->state = UNINTERRUPTIBLE LOAD p->on_rq 2881 * 2882 * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in 2883 * __schedule(). See the comment for smp_mb__after_spinlock(). 2884 * 2885 * A similar smb_rmb() lives in try_invoke_on_locked_down_task(). 2886 */ 2887 smp_rmb(); 2888 if (READ_ONCE(p->on_rq) && ttwu_runnable(p, wake_flags)) 2889 goto unlock; 2890 2891 if (p->in_iowait) { 2892 delayacct_blkio_end(p); 2893 atomic_dec(&task_rq(p)->nr_iowait); 2894 } 2895 2896 #ifdef CONFIG_SMP 2897 /* 2898 * Ensure we load p->on_cpu _after_ p->on_rq, otherwise it would be 2899 * possible to, falsely, observe p->on_cpu == 0. 2900 * 2901 * One must be running (->on_cpu == 1) in order to remove oneself 2902 * from the runqueue. 2903 * 2904 * __schedule() (switch to task 'p') try_to_wake_up() 2905 * STORE p->on_cpu = 1 LOAD p->on_rq 2906 * UNLOCK rq->lock 2907 * 2908 * __schedule() (put 'p' to sleep) 2909 * LOCK rq->lock smp_rmb(); 2910 * smp_mb__after_spinlock(); 2911 * STORE p->on_rq = 0 LOAD p->on_cpu 2912 * 2913 * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in 2914 * __schedule(). See the comment for smp_mb__after_spinlock(). 2915 * 2916 * Form a control-dep-acquire with p->on_rq == 0 above, to ensure 2917 * schedule()'s deactivate_task() has 'happened' and p will no longer 2918 * care about it's own p->state. See the comment in __schedule(). 2919 */ 2920 smp_acquire__after_ctrl_dep(); 2921 2922 /* 2923 * We're doing the wakeup (@success == 1), they did a dequeue (p->on_rq 2924 * == 0), which means we need to do an enqueue, change p->state to 2925 * TASK_WAKING such that we can unlock p->pi_lock before doing the 2926 * enqueue, such as ttwu_queue_wakelist(). 2927 */ 2928 p->state = TASK_WAKING; 2929 2930 /* 2931 * If the owning (remote) CPU is still in the middle of schedule() with 2932 * this task as prev, considering queueing p on the remote CPUs wake_list 2933 * which potentially sends an IPI instead of spinning on p->on_cpu to 2934 * let the waker make forward progress. This is safe because IRQs are 2935 * disabled and the IPI will deliver after on_cpu is cleared. 2936 * 2937 * Ensure we load task_cpu(p) after p->on_cpu: 2938 * 2939 * set_task_cpu(p, cpu); 2940 * STORE p->cpu = @cpu 2941 * __schedule() (switch to task 'p') 2942 * LOCK rq->lock 2943 * smp_mb__after_spin_lock() smp_cond_load_acquire(&p->on_cpu) 2944 * STORE p->on_cpu = 1 LOAD p->cpu 2945 * 2946 * to ensure we observe the correct CPU on which the task is currently 2947 * scheduling. 2948 */ 2949 if (smp_load_acquire(&p->on_cpu) && 2950 ttwu_queue_wakelist(p, task_cpu(p), wake_flags | WF_ON_CPU)) 2951 goto unlock; 2952 2953 /* 2954 * If the owning (remote) CPU is still in the middle of schedule() with 2955 * this task as prev, wait until its done referencing the task. 2956 * 2957 * Pairs with the smp_store_release() in finish_task(). 2958 * 2959 * This ensures that tasks getting woken will be fully ordered against 2960 * their previous state and preserve Program Order. 2961 */ 2962 smp_cond_load_acquire(&p->on_cpu, !VAL); 2963 2964 cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags); 2965 if (task_cpu(p) != cpu) { 2966 wake_flags |= WF_MIGRATED; 2967 psi_ttwu_dequeue(p); 2968 set_task_cpu(p, cpu); 2969 } 2970 #else 2971 cpu = task_cpu(p); 2972 #endif /* CONFIG_SMP */ 2973 2974 ttwu_queue(p, cpu, wake_flags); 2975 unlock: 2976 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 2977 out: 2978 if (success) 2979 ttwu_stat(p, task_cpu(p), wake_flags); 2980 preempt_enable(); 2981 2982 return success; 2983 } 2984 2985 /** 2986 * try_invoke_on_locked_down_task - Invoke a function on task in fixed state 2987 * @p: Process for which the function is to be invoked. 2988 * @func: Function to invoke. 2989 * @arg: Argument to function. 2990 * 2991 * If the specified task can be quickly locked into a definite state 2992 * (either sleeping or on a given runqueue), arrange to keep it in that 2993 * state while invoking @func(@arg). This function can use ->on_rq and 2994 * task_curr() to work out what the state is, if required. Given that 2995 * @func can be invoked with a runqueue lock held, it had better be quite 2996 * lightweight. 2997 * 2998 * Returns: 2999 * @false if the task slipped out from under the locks. 3000 * @true if the task was locked onto a runqueue or is sleeping. 3001 * However, @func can override this by returning @false. 3002 */ 3003 bool try_invoke_on_locked_down_task(struct task_struct *p, bool (*func)(struct task_struct *t, void *arg), void *arg) 3004 { 3005 bool ret = false; 3006 struct rq_flags rf; 3007 struct rq *rq; 3008 3009 lockdep_assert_irqs_enabled(); 3010 raw_spin_lock_irq(&p->pi_lock); 3011 if (p->on_rq) { 3012 rq = __task_rq_lock(p, &rf); 3013 if (task_rq(p) == rq) 3014 ret = func(p, arg); 3015 rq_unlock(rq, &rf); 3016 } else { 3017 switch (p->state) { 3018 case TASK_RUNNING: 3019 case TASK_WAKING: 3020 break; 3021 default: 3022 smp_rmb(); // See smp_rmb() comment in try_to_wake_up(). 3023 if (!p->on_rq) 3024 ret = func(p, arg); 3025 } 3026 } 3027 raw_spin_unlock_irq(&p->pi_lock); 3028 return ret; 3029 } 3030 3031 /** 3032 * wake_up_process - Wake up a specific process 3033 * @p: The process to be woken up. 3034 * 3035 * Attempt to wake up the nominated process and move it to the set of runnable 3036 * processes. 3037 * 3038 * Return: 1 if the process was woken up, 0 if it was already running. 3039 * 3040 * This function executes a full memory barrier before accessing the task state. 3041 */ 3042 int wake_up_process(struct task_struct *p) 3043 { 3044 return try_to_wake_up(p, TASK_NORMAL, 0); 3045 } 3046 EXPORT_SYMBOL(wake_up_process); 3047 3048 int wake_up_state(struct task_struct *p, unsigned int state) 3049 { 3050 return try_to_wake_up(p, state, 0); 3051 } 3052 3053 /* 3054 * Perform scheduler related setup for a newly forked process p. 3055 * p is forked by current. 3056 * 3057 * __sched_fork() is basic setup used by init_idle() too: 3058 */ 3059 static void __sched_fork(unsigned long clone_flags, struct task_struct *p) 3060 { 3061 p->on_rq = 0; 3062 3063 p->se.on_rq = 0; 3064 p->se.exec_start = 0; 3065 p->se.sum_exec_runtime = 0; 3066 p->se.prev_sum_exec_runtime = 0; 3067 p->se.nr_migrations = 0; 3068 p->se.vruntime = 0; 3069 INIT_LIST_HEAD(&p->se.group_node); 3070 3071 #ifdef CONFIG_FAIR_GROUP_SCHED 3072 p->se.cfs_rq = NULL; 3073 #endif 3074 3075 #ifdef CONFIG_SCHEDSTATS 3076 /* Even if schedstat is disabled, there should not be garbage */ 3077 memset(&p->se.statistics, 0, sizeof(p->se.statistics)); 3078 #endif 3079 3080 RB_CLEAR_NODE(&p->dl.rb_node); 3081 init_dl_task_timer(&p->dl); 3082 init_dl_inactive_task_timer(&p->dl); 3083 __dl_clear_params(p); 3084 3085 INIT_LIST_HEAD(&p->rt.run_list); 3086 p->rt.timeout = 0; 3087 p->rt.time_slice = sched_rr_timeslice; 3088 p->rt.on_rq = 0; 3089 p->rt.on_list = 0; 3090 3091 #ifdef CONFIG_PREEMPT_NOTIFIERS 3092 INIT_HLIST_HEAD(&p->preempt_notifiers); 3093 #endif 3094 3095 #ifdef CONFIG_COMPACTION 3096 p->capture_control = NULL; 3097 #endif 3098 init_numa_balancing(clone_flags, p); 3099 #ifdef CONFIG_SMP 3100 p->wake_entry.u_flags = CSD_TYPE_TTWU; 3101 #endif 3102 } 3103 3104 DEFINE_STATIC_KEY_FALSE(sched_numa_balancing); 3105 3106 #ifdef CONFIG_NUMA_BALANCING 3107 3108 void set_numabalancing_state(bool enabled) 3109 { 3110 if (enabled) 3111 static_branch_enable(&sched_numa_balancing); 3112 else 3113 static_branch_disable(&sched_numa_balancing); 3114 } 3115 3116 #ifdef CONFIG_PROC_SYSCTL 3117 int sysctl_numa_balancing(struct ctl_table *table, int write, 3118 void *buffer, size_t *lenp, loff_t *ppos) 3119 { 3120 struct ctl_table t; 3121 int err; 3122 int state = static_branch_likely(&sched_numa_balancing); 3123 3124 if (write && !capable(CAP_SYS_ADMIN)) 3125 return -EPERM; 3126 3127 t = *table; 3128 t.data = &state; 3129 err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); 3130 if (err < 0) 3131 return err; 3132 if (write) 3133 set_numabalancing_state(state); 3134 return err; 3135 } 3136 #endif 3137 #endif 3138 3139 #ifdef CONFIG_SCHEDSTATS 3140 3141 DEFINE_STATIC_KEY_FALSE(sched_schedstats); 3142 static bool __initdata __sched_schedstats = false; 3143 3144 static void set_schedstats(bool enabled) 3145 { 3146 if (enabled) 3147 static_branch_enable(&sched_schedstats); 3148 else 3149 static_branch_disable(&sched_schedstats); 3150 } 3151 3152 void force_schedstat_enabled(void) 3153 { 3154 if (!schedstat_enabled()) { 3155 pr_info("kernel profiling enabled schedstats, disable via kernel.sched_schedstats.\n"); 3156 static_branch_enable(&sched_schedstats); 3157 } 3158 } 3159 3160 static int __init setup_schedstats(char *str) 3161 { 3162 int ret = 0; 3163 if (!str) 3164 goto out; 3165 3166 /* 3167 * This code is called before jump labels have been set up, so we can't 3168 * change the static branch directly just yet. Instead set a temporary 3169 * variable so init_schedstats() can do it later. 3170 */ 3171 if (!strcmp(str, "enable")) { 3172 __sched_schedstats = true; 3173 ret = 1; 3174 } else if (!strcmp(str, "disable")) { 3175 __sched_schedstats = false; 3176 ret = 1; 3177 } 3178 out: 3179 if (!ret) 3180 pr_warn("Unable to parse schedstats=\n"); 3181 3182 return ret; 3183 } 3184 __setup("schedstats=", setup_schedstats); 3185 3186 static void __init init_schedstats(void) 3187 { 3188 set_schedstats(__sched_schedstats); 3189 } 3190 3191 #ifdef CONFIG_PROC_SYSCTL 3192 int sysctl_schedstats(struct ctl_table *table, int write, void *buffer, 3193 size_t *lenp, loff_t *ppos) 3194 { 3195 struct ctl_table t; 3196 int err; 3197 int state = static_branch_likely(&sched_schedstats); 3198 3199 if (write && !capable(CAP_SYS_ADMIN)) 3200 return -EPERM; 3201 3202 t = *table; 3203 t.data = &state; 3204 err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); 3205 if (err < 0) 3206 return err; 3207 if (write) 3208 set_schedstats(state); 3209 return err; 3210 } 3211 #endif /* CONFIG_PROC_SYSCTL */ 3212 #else /* !CONFIG_SCHEDSTATS */ 3213 static inline void init_schedstats(void) {} 3214 #endif /* CONFIG_SCHEDSTATS */ 3215 3216 /* 3217 * fork()/clone()-time setup: 3218 */ 3219 int sched_fork(unsigned long clone_flags, struct task_struct *p) 3220 { 3221 unsigned long flags; 3222 3223 __sched_fork(clone_flags, p); 3224 /* 3225 * We mark the process as NEW here. This guarantees that 3226 * nobody will actually run it, and a signal or other external 3227 * event cannot wake it up and insert it on the runqueue either. 3228 */ 3229 p->state = TASK_NEW; 3230 3231 /* 3232 * Make sure we do not leak PI boosting priority to the child. 3233 */ 3234 p->prio = current->normal_prio; 3235 3236 uclamp_fork(p); 3237 3238 /* 3239 * Revert to default priority/policy on fork if requested. 3240 */ 3241 if (unlikely(p->sched_reset_on_fork)) { 3242 if (task_has_dl_policy(p) || task_has_rt_policy(p)) { 3243 p->policy = SCHED_NORMAL; 3244 p->static_prio = NICE_TO_PRIO(0); 3245 p->rt_priority = 0; 3246 } else if (PRIO_TO_NICE(p->static_prio) < 0) 3247 p->static_prio = NICE_TO_PRIO(0); 3248 3249 p->prio = p->normal_prio = __normal_prio(p); 3250 set_load_weight(p, false); 3251 3252 /* 3253 * We don't need the reset flag anymore after the fork. It has 3254 * fulfilled its duty: 3255 */ 3256 p->sched_reset_on_fork = 0; 3257 } 3258 3259 if (dl_prio(p->prio)) 3260 return -EAGAIN; 3261 else if (rt_prio(p->prio)) 3262 p->sched_class = &rt_sched_class; 3263 else 3264 p->sched_class = &fair_sched_class; 3265 3266 init_entity_runnable_average(&p->se); 3267 3268 /* 3269 * The child is not yet in the pid-hash so no cgroup attach races, 3270 * and the cgroup is pinned to this child due to cgroup_fork() 3271 * is ran before sched_fork(). 3272 * 3273 * Silence PROVE_RCU. 3274 */ 3275 raw_spin_lock_irqsave(&p->pi_lock, flags); 3276 rseq_migrate(p); 3277 /* 3278 * We're setting the CPU for the first time, we don't migrate, 3279 * so use __set_task_cpu(). 3280 */ 3281 __set_task_cpu(p, smp_processor_id()); 3282 if (p->sched_class->task_fork) 3283 p->sched_class->task_fork(p); 3284 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 3285 3286 #ifdef CONFIG_SCHED_INFO 3287 if (likely(sched_info_on())) 3288 memset(&p->sched_info, 0, sizeof(p->sched_info)); 3289 #endif 3290 #if defined(CONFIG_SMP) 3291 p->on_cpu = 0; 3292 #endif 3293 init_task_preempt_count(p); 3294 #ifdef CONFIG_SMP 3295 plist_node_init(&p->pushable_tasks, MAX_PRIO); 3296 RB_CLEAR_NODE(&p->pushable_dl_tasks); 3297 #endif 3298 return 0; 3299 } 3300 3301 void sched_post_fork(struct task_struct *p) 3302 { 3303 uclamp_post_fork(p); 3304 } 3305 3306 unsigned long to_ratio(u64 period, u64 runtime) 3307 { 3308 if (runtime == RUNTIME_INF) 3309 return BW_UNIT; 3310 3311 /* 3312 * Doing this here saves a lot of checks in all 3313 * the calling paths, and returning zero seems 3314 * safe for them anyway. 3315 */ 3316 if (period == 0) 3317 return 0; 3318 3319 return div64_u64(runtime << BW_SHIFT, period); 3320 } 3321 3322 /* 3323 * wake_up_new_task - wake up a newly created task for the first time. 3324 * 3325 * This function will do some initial scheduler statistics housekeeping 3326 * that must be done for every newly created context, then puts the task 3327 * on the runqueue and wakes it. 3328 */ 3329 void wake_up_new_task(struct task_struct *p) 3330 { 3331 struct rq_flags rf; 3332 struct rq *rq; 3333 3334 raw_spin_lock_irqsave(&p->pi_lock, rf.flags); 3335 p->state = TASK_RUNNING; 3336 #ifdef CONFIG_SMP 3337 /* 3338 * Fork balancing, do it here and not earlier because: 3339 * - cpus_ptr can change in the fork path 3340 * - any previously selected CPU might disappear through hotplug 3341 * 3342 * Use __set_task_cpu() to avoid calling sched_class::migrate_task_rq, 3343 * as we're not fully set-up yet. 3344 */ 3345 p->recent_used_cpu = task_cpu(p); 3346 rseq_migrate(p); 3347 __set_task_cpu(p, select_task_rq(p, task_cpu(p), SD_BALANCE_FORK, 0)); 3348 #endif 3349 rq = __task_rq_lock(p, &rf); 3350 update_rq_clock(rq); 3351 post_init_entity_util_avg(p); 3352 3353 activate_task(rq, p, ENQUEUE_NOCLOCK); 3354 trace_sched_wakeup_new(p); 3355 check_preempt_curr(rq, p, WF_FORK); 3356 #ifdef CONFIG_SMP 3357 if (p->sched_class->task_woken) { 3358 /* 3359 * Nothing relies on rq->lock after this, so its fine to 3360 * drop it. 3361 */ 3362 rq_unpin_lock(rq, &rf); 3363 p->sched_class->task_woken(rq, p); 3364 rq_repin_lock(rq, &rf); 3365 } 3366 #endif 3367 task_rq_unlock(rq, p, &rf); 3368 } 3369 3370 #ifdef CONFIG_PREEMPT_NOTIFIERS 3371 3372 static DEFINE_STATIC_KEY_FALSE(preempt_notifier_key); 3373 3374 void preempt_notifier_inc(void) 3375 { 3376 static_branch_inc(&preempt_notifier_key); 3377 } 3378 EXPORT_SYMBOL_GPL(preempt_notifier_inc); 3379 3380 void preempt_notifier_dec(void) 3381 { 3382 static_branch_dec(&preempt_notifier_key); 3383 } 3384 EXPORT_SYMBOL_GPL(preempt_notifier_dec); 3385 3386 /** 3387 * preempt_notifier_register - tell me when current is being preempted & rescheduled 3388 * @notifier: notifier struct to register 3389 */ 3390 void preempt_notifier_register(struct preempt_notifier *notifier) 3391 { 3392 if (!static_branch_unlikely(&preempt_notifier_key)) 3393 WARN(1, "registering preempt_notifier while notifiers disabled\n"); 3394 3395 hlist_add_head(¬ifier->link, ¤t->preempt_notifiers); 3396 } 3397 EXPORT_SYMBOL_GPL(preempt_notifier_register); 3398 3399 /** 3400 * preempt_notifier_unregister - no longer interested in preemption notifications 3401 * @notifier: notifier struct to unregister 3402 * 3403 * This is *not* safe to call from within a preemption notifier. 3404 */ 3405 void preempt_notifier_unregister(struct preempt_notifier *notifier) 3406 { 3407 hlist_del(¬ifier->link); 3408 } 3409 EXPORT_SYMBOL_GPL(preempt_notifier_unregister); 3410 3411 static void __fire_sched_in_preempt_notifiers(struct task_struct *curr) 3412 { 3413 struct preempt_notifier *notifier; 3414 3415 hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) 3416 notifier->ops->sched_in(notifier, raw_smp_processor_id()); 3417 } 3418 3419 static __always_inline void fire_sched_in_preempt_notifiers(struct task_struct *curr) 3420 { 3421 if (static_branch_unlikely(&preempt_notifier_key)) 3422 __fire_sched_in_preempt_notifiers(curr); 3423 } 3424 3425 static void 3426 __fire_sched_out_preempt_notifiers(struct task_struct *curr, 3427 struct task_struct *next) 3428 { 3429 struct preempt_notifier *notifier; 3430 3431 hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) 3432 notifier->ops->sched_out(notifier, next); 3433 } 3434 3435 static __always_inline void 3436 fire_sched_out_preempt_notifiers(struct task_struct *curr, 3437 struct task_struct *next) 3438 { 3439 if (static_branch_unlikely(&preempt_notifier_key)) 3440 __fire_sched_out_preempt_notifiers(curr, next); 3441 } 3442 3443 #else /* !CONFIG_PREEMPT_NOTIFIERS */ 3444 3445 static inline void fire_sched_in_preempt_notifiers(struct task_struct *curr) 3446 { 3447 } 3448 3449 static inline void 3450 fire_sched_out_preempt_notifiers(struct task_struct *curr, 3451 struct task_struct *next) 3452 { 3453 } 3454 3455 #endif /* CONFIG_PREEMPT_NOTIFIERS */ 3456 3457 static inline void prepare_task(struct task_struct *next) 3458 { 3459 #ifdef CONFIG_SMP 3460 /* 3461 * Claim the task as running, we do this before switching to it 3462 * such that any running task will have this set. 3463 * 3464 * See the ttwu() WF_ON_CPU case and its ordering comment. 3465 */ 3466 WRITE_ONCE(next->on_cpu, 1); 3467 #endif 3468 } 3469 3470 static inline void finish_task(struct task_struct *prev) 3471 { 3472 #ifdef CONFIG_SMP 3473 /* 3474 * This must be the very last reference to @prev from this CPU. After 3475 * p->on_cpu is cleared, the task can be moved to a different CPU. We 3476 * must ensure this doesn't happen until the switch is completely 3477 * finished. 3478 * 3479 * In particular, the load of prev->state in finish_task_switch() must 3480 * happen before this. 3481 * 3482 * Pairs with the smp_cond_load_acquire() in try_to_wake_up(). 3483 */ 3484 smp_store_release(&prev->on_cpu, 0); 3485 #endif 3486 } 3487 3488 static inline void 3489 prepare_lock_switch(struct rq *rq, struct task_struct *next, struct rq_flags *rf) 3490 { 3491 /* 3492 * Since the runqueue lock will be released by the next 3493 * task (which is an invalid locking op but in the case 3494 * of the scheduler it's an obvious special-case), so we 3495 * do an early lockdep release here: 3496 */ 3497 rq_unpin_lock(rq, rf); 3498 spin_release(&rq->lock.dep_map, _THIS_IP_); 3499 #ifdef CONFIG_DEBUG_SPINLOCK 3500 /* this is a valid case when another task releases the spinlock */ 3501 rq->lock.owner = next; 3502 #endif 3503 } 3504 3505 static inline void finish_lock_switch(struct rq *rq) 3506 { 3507 /* 3508 * If we are tracking spinlock dependencies then we have to 3509 * fix up the runqueue lock - which gets 'carried over' from 3510 * prev into current: 3511 */ 3512 spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_); 3513 raw_spin_unlock_irq(&rq->lock); 3514 } 3515 3516 /* 3517 * NOP if the arch has not defined these: 3518 */ 3519 3520 #ifndef prepare_arch_switch 3521 # define prepare_arch_switch(next) do { } while (0) 3522 #endif 3523 3524 #ifndef finish_arch_post_lock_switch 3525 # define finish_arch_post_lock_switch() do { } while (0) 3526 #endif 3527 3528 /** 3529 * prepare_task_switch - prepare to switch tasks 3530 * @rq: the runqueue preparing to switch 3531 * @prev: the current task that is being switched out 3532 * @next: the task we are going to switch to. 3533 * 3534 * This is called with the rq lock held and interrupts off. It must 3535 * be paired with a subsequent finish_task_switch after the context 3536 * switch. 3537 * 3538 * prepare_task_switch sets up locking and calls architecture specific 3539 * hooks. 3540 */ 3541 static inline void 3542 prepare_task_switch(struct rq *rq, struct task_struct *prev, 3543 struct task_struct *next) 3544 { 3545 kcov_prepare_switch(prev); 3546 sched_info_switch(rq, prev, next); 3547 perf_event_task_sched_out(prev, next); 3548 rseq_preempt(prev); 3549 fire_sched_out_preempt_notifiers(prev, next); 3550 prepare_task(next); 3551 prepare_arch_switch(next); 3552 } 3553 3554 /** 3555 * finish_task_switch - clean up after a task-switch 3556 * @prev: the thread we just switched away from. 3557 * 3558 * finish_task_switch must be called after the context switch, paired 3559 * with a prepare_task_switch call before the context switch. 3560 * finish_task_switch will reconcile locking set up by prepare_task_switch, 3561 * and do any other architecture-specific cleanup actions. 3562 * 3563 * Note that we may have delayed dropping an mm in context_switch(). If 3564 * so, we finish that here outside of the runqueue lock. (Doing it 3565 * with the lock held can cause deadlocks; see schedule() for 3566 * details.) 3567 * 3568 * The context switch have flipped the stack from under us and restored the 3569 * local variables which were saved when this task called schedule() in the 3570 * past. prev == current is still correct but we need to recalculate this_rq 3571 * because prev may have moved to another CPU. 3572 */ 3573 static struct rq *finish_task_switch(struct task_struct *prev) 3574 __releases(rq->lock) 3575 { 3576 struct rq *rq = this_rq(); 3577 struct mm_struct *mm = rq->prev_mm; 3578 long prev_state; 3579 3580 /* 3581 * The previous task will have left us with a preempt_count of 2 3582 * because it left us after: 3583 * 3584 * schedule() 3585 * preempt_disable(); // 1 3586 * __schedule() 3587 * raw_spin_lock_irq(&rq->lock) // 2 3588 * 3589 * Also, see FORK_PREEMPT_COUNT. 3590 */ 3591 if (WARN_ONCE(preempt_count() != 2*PREEMPT_DISABLE_OFFSET, 3592 "corrupted preempt_count: %s/%d/0x%x\n", 3593 current->comm, current->pid, preempt_count())) 3594 preempt_count_set(FORK_PREEMPT_COUNT); 3595 3596 rq->prev_mm = NULL; 3597 3598 /* 3599 * A task struct has one reference for the use as "current". 3600 * If a task dies, then it sets TASK_DEAD in tsk->state and calls 3601 * schedule one last time. The schedule call will never return, and 3602 * the scheduled task must drop that reference. 3603 * 3604 * We must observe prev->state before clearing prev->on_cpu (in 3605 * finish_task), otherwise a concurrent wakeup can get prev 3606 * running on another CPU and we could rave with its RUNNING -> DEAD 3607 * transition, resulting in a double drop. 3608 */ 3609 prev_state = prev->state; 3610 vtime_task_switch(prev); 3611 perf_event_task_sched_in(prev, current); 3612 finish_task(prev); 3613 finish_lock_switch(rq); 3614 finish_arch_post_lock_switch(); 3615 kcov_finish_switch(current); 3616 3617 fire_sched_in_preempt_notifiers(current); 3618 /* 3619 * When switching through a kernel thread, the loop in 3620 * membarrier_{private,global}_expedited() may have observed that 3621 * kernel thread and not issued an IPI. It is therefore possible to 3622 * schedule between user->kernel->user threads without passing though 3623 * switch_mm(). Membarrier requires a barrier after storing to 3624 * rq->curr, before returning to userspace, so provide them here: 3625 * 3626 * - a full memory barrier for {PRIVATE,GLOBAL}_EXPEDITED, implicitly 3627 * provided by mmdrop(), 3628 * - a sync_core for SYNC_CORE. 3629 */ 3630 if (mm) { 3631 membarrier_mm_sync_core_before_usermode(mm); 3632 mmdrop(mm); 3633 } 3634 if (unlikely(prev_state == TASK_DEAD)) { 3635 if (prev->sched_class->task_dead) 3636 prev->sched_class->task_dead(prev); 3637 3638 /* 3639 * Remove function-return probe instances associated with this 3640 * task and put them back on the free list. 3641 */ 3642 kprobe_flush_task(prev); 3643 3644 /* Task is done with its stack. */ 3645 put_task_stack(prev); 3646 3647 put_task_struct_rcu_user(prev); 3648 } 3649 3650 tick_nohz_task_switch(); 3651 return rq; 3652 } 3653 3654 #ifdef CONFIG_SMP 3655 3656 /* rq->lock is NOT held, but preemption is disabled */ 3657 static void __balance_callback(struct rq *rq) 3658 { 3659 struct callback_head *head, *next; 3660 void (*func)(struct rq *rq); 3661 unsigned long flags; 3662 3663 raw_spin_lock_irqsave(&rq->lock, flags); 3664 head = rq->balance_callback; 3665 rq->balance_callback = NULL; 3666 while (head) { 3667 func = (void (*)(struct rq *))head->func; 3668 next = head->next; 3669 head->next = NULL; 3670 head = next; 3671 3672 func(rq); 3673 } 3674 raw_spin_unlock_irqrestore(&rq->lock, flags); 3675 } 3676 3677 static inline void balance_callback(struct rq *rq) 3678 { 3679 if (unlikely(rq->balance_callback)) 3680 __balance_callback(rq); 3681 } 3682 3683 #else 3684 3685 static inline void balance_callback(struct rq *rq) 3686 { 3687 } 3688 3689 #endif 3690 3691 /** 3692 * schedule_tail - first thing a freshly forked thread must call. 3693 * @prev: the thread we just switched away from. 3694 */ 3695 asmlinkage __visible void schedule_tail(struct task_struct *prev) 3696 __releases(rq->lock) 3697 { 3698 struct rq *rq; 3699 3700 /* 3701 * New tasks start with FORK_PREEMPT_COUNT, see there and 3702 * finish_task_switch() for details. 3703 * 3704 * finish_task_switch() will drop rq->lock() and lower preempt_count 3705 * and the preempt_enable() will end up enabling preemption (on 3706 * PREEMPT_COUNT kernels). 3707 */ 3708 3709 rq = finish_task_switch(prev); 3710 balance_callback(rq); 3711 preempt_enable(); 3712 3713 if (current->set_child_tid) 3714 put_user(task_pid_vnr(current), current->set_child_tid); 3715 3716 calculate_sigpending(); 3717 } 3718 3719 /* 3720 * context_switch - switch to the new MM and the new thread's register state. 3721 */ 3722 static __always_inline struct rq * 3723 context_switch(struct rq *rq, struct task_struct *prev, 3724 struct task_struct *next, struct rq_flags *rf) 3725 { 3726 prepare_task_switch(rq, prev, next); 3727 3728 /* 3729 * For paravirt, this is coupled with an exit in switch_to to 3730 * combine the page table reload and the switch backend into 3731 * one hypercall. 3732 */ 3733 arch_start_context_switch(prev); 3734 3735 /* 3736 * kernel -> kernel lazy + transfer active 3737 * user -> kernel lazy + mmgrab() active 3738 * 3739 * kernel -> user switch + mmdrop() active 3740 * user -> user switch 3741 */ 3742 if (!next->mm) { // to kernel 3743 enter_lazy_tlb(prev->active_mm, next); 3744 3745 next->active_mm = prev->active_mm; 3746 if (prev->mm) // from user 3747 mmgrab(prev->active_mm); 3748 else 3749 prev->active_mm = NULL; 3750 } else { // to user 3751 membarrier_switch_mm(rq, prev->active_mm, next->mm); 3752 /* 3753 * sys_membarrier() requires an smp_mb() between setting 3754 * rq->curr / membarrier_switch_mm() and returning to userspace. 3755 * 3756 * The below provides this either through switch_mm(), or in 3757 * case 'prev->active_mm == next->mm' through 3758 * finish_task_switch()'s mmdrop(). 3759 */ 3760 switch_mm_irqs_off(prev->active_mm, next->mm, next); 3761 3762 if (!prev->mm) { // from kernel 3763 /* will mmdrop() in finish_task_switch(). */ 3764 rq->prev_mm = prev->active_mm; 3765 prev->active_mm = NULL; 3766 } 3767 } 3768 3769 rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP); 3770 3771 prepare_lock_switch(rq, next, rf); 3772 3773 /* Here we just switch the register state and the stack. */ 3774 switch_to(prev, next, prev); 3775 barrier(); 3776 3777 return finish_task_switch(prev); 3778 } 3779 3780 /* 3781 * nr_running and nr_context_switches: 3782 * 3783 * externally visible scheduler statistics: current number of runnable 3784 * threads, total number of context switches performed since bootup. 3785 */ 3786 unsigned long nr_running(void) 3787 { 3788 unsigned long i, sum = 0; 3789 3790 for_each_online_cpu(i) 3791 sum += cpu_rq(i)->nr_running; 3792 3793 return sum; 3794 } 3795 3796 /* 3797 * Check if only the current task is running on the CPU. 3798 * 3799 * Caution: this function does not check that the caller has disabled 3800 * preemption, thus the result might have a time-of-check-to-time-of-use 3801 * race. The caller is responsible to use it correctly, for example: 3802 * 3803 * - from a non-preemptible section (of course) 3804 * 3805 * - from a thread that is bound to a single CPU 3806 * 3807 * - in a loop with very short iterations (e.g. a polling loop) 3808 */ 3809 bool single_task_running(void) 3810 { 3811 return raw_rq()->nr_running == 1; 3812 } 3813 EXPORT_SYMBOL(single_task_running); 3814 3815 unsigned long long nr_context_switches(void) 3816 { 3817 int i; 3818 unsigned long long sum = 0; 3819 3820 for_each_possible_cpu(i) 3821 sum += cpu_rq(i)->nr_switches; 3822 3823 return sum; 3824 } 3825 3826 /* 3827 * Consumers of these two interfaces, like for example the cpuidle menu 3828 * governor, are using nonsensical data. Preferring shallow idle state selection 3829 * for a CPU that has IO-wait which might not even end up running the task when 3830 * it does become runnable. 3831 */ 3832 3833 unsigned long nr_iowait_cpu(int cpu) 3834 { 3835 return atomic_read(&cpu_rq(cpu)->nr_iowait); 3836 } 3837 3838 /* 3839 * IO-wait accounting, and how its mostly bollocks (on SMP). 3840 * 3841 * The idea behind IO-wait account is to account the idle time that we could 3842 * have spend running if it were not for IO. That is, if we were to improve the 3843 * storage performance, we'd have a proportional reduction in IO-wait time. 3844 * 3845 * This all works nicely on UP, where, when a task blocks on IO, we account 3846 * idle time as IO-wait, because if the storage were faster, it could've been 3847 * running and we'd not be idle. 3848 * 3849 * This has been extended to SMP, by doing the same for each CPU. This however 3850 * is broken. 3851 * 3852 * Imagine for instance the case where two tasks block on one CPU, only the one 3853 * CPU will have IO-wait accounted, while the other has regular idle. Even 3854 * though, if the storage were faster, both could've ran at the same time, 3855 * utilising both CPUs. 3856 * 3857 * This means, that when looking globally, the current IO-wait accounting on 3858 * SMP is a lower bound, by reason of under accounting. 3859 * 3860 * Worse, since the numbers are provided per CPU, they are sometimes 3861 * interpreted per CPU, and that is nonsensical. A blocked task isn't strictly 3862 * associated with any one particular CPU, it can wake to another CPU than it 3863 * blocked on. This means the per CPU IO-wait number is meaningless. 3864 * 3865 * Task CPU affinities can make all that even more 'interesting'. 3866 */ 3867 3868 unsigned long nr_iowait(void) 3869 { 3870 unsigned long i, sum = 0; 3871 3872 for_each_possible_cpu(i) 3873 sum += nr_iowait_cpu(i); 3874 3875 return sum; 3876 } 3877 3878 #ifdef CONFIG_SMP 3879 3880 /* 3881 * sched_exec - execve() is a valuable balancing opportunity, because at 3882 * this point the task has the smallest effective memory and cache footprint. 3883 */ 3884 void sched_exec(void) 3885 { 3886 struct task_struct *p = current; 3887 unsigned long flags; 3888 int dest_cpu; 3889 3890 raw_spin_lock_irqsave(&p->pi_lock, flags); 3891 dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), SD_BALANCE_EXEC, 0); 3892 if (dest_cpu == smp_processor_id()) 3893 goto unlock; 3894 3895 if (likely(cpu_active(dest_cpu))) { 3896 struct migration_arg arg = { p, dest_cpu }; 3897 3898 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 3899 stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg); 3900 return; 3901 } 3902 unlock: 3903 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 3904 } 3905 3906 #endif 3907 3908 DEFINE_PER_CPU(struct kernel_stat, kstat); 3909 DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat); 3910 3911 EXPORT_PER_CPU_SYMBOL(kstat); 3912 EXPORT_PER_CPU_SYMBOL(kernel_cpustat); 3913 3914 /* 3915 * The function fair_sched_class.update_curr accesses the struct curr 3916 * and its field curr->exec_start; when called from task_sched_runtime(), 3917 * we observe a high rate of cache misses in practice. 3918 * Prefetching this data results in improved performance. 3919 */ 3920 static inline void prefetch_curr_exec_start(struct task_struct *p) 3921 { 3922 #ifdef CONFIG_FAIR_GROUP_SCHED 3923 struct sched_entity *curr = (&p->se)->cfs_rq->curr; 3924 #else 3925 struct sched_entity *curr = (&task_rq(p)->cfs)->curr; 3926 #endif 3927 prefetch(curr); 3928 prefetch(&curr->exec_start); 3929 } 3930 3931 /* 3932 * Return accounted runtime for the task. 3933 * In case the task is currently running, return the runtime plus current's 3934 * pending runtime that have not been accounted yet. 3935 */ 3936 unsigned long long task_sched_runtime(struct task_struct *p) 3937 { 3938 struct rq_flags rf; 3939 struct rq *rq; 3940 u64 ns; 3941 3942 #if defined(CONFIG_64BIT) && defined(CONFIG_SMP) 3943 /* 3944 * 64-bit doesn't need locks to atomically read a 64-bit value. 3945 * So we have a optimization chance when the task's delta_exec is 0. 3946 * Reading ->on_cpu is racy, but this is ok. 3947 * 3948 * If we race with it leaving CPU, we'll take a lock. So we're correct. 3949 * If we race with it entering CPU, unaccounted time is 0. This is 3950 * indistinguishable from the read occurring a few cycles earlier. 3951 * If we see ->on_cpu without ->on_rq, the task is leaving, and has 3952 * been accounted, so we're correct here as well. 3953 */ 3954 if (!p->on_cpu || !task_on_rq_queued(p)) 3955 return p->se.sum_exec_runtime; 3956 #endif 3957 3958 rq = task_rq_lock(p, &rf); 3959 /* 3960 * Must be ->curr _and_ ->on_rq. If dequeued, we would 3961 * project cycles that may never be accounted to this 3962 * thread, breaking clock_gettime(). 3963 */ 3964 if (task_current(rq, p) && task_on_rq_queued(p)) { 3965 prefetch_curr_exec_start(p); 3966 update_rq_clock(rq); 3967 p->sched_class->update_curr(rq); 3968 } 3969 ns = p->se.sum_exec_runtime; 3970 task_rq_unlock(rq, p, &rf); 3971 3972 return ns; 3973 } 3974 3975 /* 3976 * This function gets called by the timer code, with HZ frequency. 3977 * We call it with interrupts disabled. 3978 */ 3979 void scheduler_tick(void) 3980 { 3981 int cpu = smp_processor_id(); 3982 struct rq *rq = cpu_rq(cpu); 3983 struct task_struct *curr = rq->curr; 3984 struct rq_flags rf; 3985 unsigned long thermal_pressure; 3986 3987 arch_scale_freq_tick(); 3988 sched_clock_tick(); 3989 3990 rq_lock(rq, &rf); 3991 3992 update_rq_clock(rq); 3993 thermal_pressure = arch_scale_thermal_pressure(cpu_of(rq)); 3994 update_thermal_load_avg(rq_clock_thermal(rq), rq, thermal_pressure); 3995 curr->sched_class->task_tick(rq, curr, 0); 3996 calc_global_load_tick(rq); 3997 psi_task_tick(rq); 3998 3999 rq_unlock(rq, &rf); 4000 4001 perf_event_task_tick(); 4002 4003 #ifdef CONFIG_SMP 4004 rq->idle_balance = idle_cpu(cpu); 4005 trigger_load_balance(rq); 4006 #endif 4007 } 4008 4009 #ifdef CONFIG_NO_HZ_FULL 4010 4011 struct tick_work { 4012 int cpu; 4013 atomic_t state; 4014 struct delayed_work work; 4015 }; 4016 /* Values for ->state, see diagram below. */ 4017 #define TICK_SCHED_REMOTE_OFFLINE 0 4018 #define TICK_SCHED_REMOTE_OFFLINING 1 4019 #define TICK_SCHED_REMOTE_RUNNING 2 4020 4021 /* 4022 * State diagram for ->state: 4023 * 4024 * 4025 * TICK_SCHED_REMOTE_OFFLINE 4026 * | ^ 4027 * | | 4028 * | | sched_tick_remote() 4029 * | | 4030 * | | 4031 * +--TICK_SCHED_REMOTE_OFFLINING 4032 * | ^ 4033 * | | 4034 * sched_tick_start() | | sched_tick_stop() 4035 * | | 4036 * V | 4037 * TICK_SCHED_REMOTE_RUNNING 4038 * 4039 * 4040 * Other transitions get WARN_ON_ONCE(), except that sched_tick_remote() 4041 * and sched_tick_start() are happy to leave the state in RUNNING. 4042 */ 4043 4044 static struct tick_work __percpu *tick_work_cpu; 4045 4046 static void sched_tick_remote(struct work_struct *work) 4047 { 4048 struct delayed_work *dwork = to_delayed_work(work); 4049 struct tick_work *twork = container_of(dwork, struct tick_work, work); 4050 int cpu = twork->cpu; 4051 struct rq *rq = cpu_rq(cpu); 4052 struct task_struct *curr; 4053 struct rq_flags rf; 4054 u64 delta; 4055 int os; 4056 4057 /* 4058 * Handle the tick only if it appears the remote CPU is running in full 4059 * dynticks mode. The check is racy by nature, but missing a tick or 4060 * having one too much is no big deal because the scheduler tick updates 4061 * statistics and checks timeslices in a time-independent way, regardless 4062 * of when exactly it is running. 4063 */ 4064 if (!tick_nohz_tick_stopped_cpu(cpu)) 4065 goto out_requeue; 4066 4067 rq_lock_irq(rq, &rf); 4068 curr = rq->curr; 4069 if (cpu_is_offline(cpu)) 4070 goto out_unlock; 4071 4072 update_rq_clock(rq); 4073 4074 if (!is_idle_task(curr)) { 4075 /* 4076 * Make sure the next tick runs within a reasonable 4077 * amount of time. 4078 */ 4079 delta = rq_clock_task(rq) - curr->se.exec_start; 4080 WARN_ON_ONCE(delta > (u64)NSEC_PER_SEC * 3); 4081 } 4082 curr->sched_class->task_tick(rq, curr, 0); 4083 4084 calc_load_nohz_remote(rq); 4085 out_unlock: 4086 rq_unlock_irq(rq, &rf); 4087 out_requeue: 4088 4089 /* 4090 * Run the remote tick once per second (1Hz). This arbitrary 4091 * frequency is large enough to avoid overload but short enough 4092 * to keep scheduler internal stats reasonably up to date. But 4093 * first update state to reflect hotplug activity if required. 4094 */ 4095 os = atomic_fetch_add_unless(&twork->state, -1, TICK_SCHED_REMOTE_RUNNING); 4096 WARN_ON_ONCE(os == TICK_SCHED_REMOTE_OFFLINE); 4097 if (os == TICK_SCHED_REMOTE_RUNNING) 4098 queue_delayed_work(system_unbound_wq, dwork, HZ); 4099 } 4100 4101 static void sched_tick_start(int cpu) 4102 { 4103 int os; 4104 struct tick_work *twork; 4105 4106 if (housekeeping_cpu(cpu, HK_FLAG_TICK)) 4107 return; 4108 4109 WARN_ON_ONCE(!tick_work_cpu); 4110 4111 twork = per_cpu_ptr(tick_work_cpu, cpu); 4112 os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_RUNNING); 4113 WARN_ON_ONCE(os == TICK_SCHED_REMOTE_RUNNING); 4114 if (os == TICK_SCHED_REMOTE_OFFLINE) { 4115 twork->cpu = cpu; 4116 INIT_DELAYED_WORK(&twork->work, sched_tick_remote); 4117 queue_delayed_work(system_unbound_wq, &twork->work, HZ); 4118 } 4119 } 4120 4121 #ifdef CONFIG_HOTPLUG_CPU 4122 static void sched_tick_stop(int cpu) 4123 { 4124 struct tick_work *twork; 4125 int os; 4126 4127 if (housekeeping_cpu(cpu, HK_FLAG_TICK)) 4128 return; 4129 4130 WARN_ON_ONCE(!tick_work_cpu); 4131 4132 twork = per_cpu_ptr(tick_work_cpu, cpu); 4133 /* There cannot be competing actions, but don't rely on stop-machine. */ 4134 os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_OFFLINING); 4135 WARN_ON_ONCE(os != TICK_SCHED_REMOTE_RUNNING); 4136 /* Don't cancel, as this would mess up the state machine. */ 4137 } 4138 #endif /* CONFIG_HOTPLUG_CPU */ 4139 4140 int __init sched_tick_offload_init(void) 4141 { 4142 tick_work_cpu = alloc_percpu(struct tick_work); 4143 BUG_ON(!tick_work_cpu); 4144 return 0; 4145 } 4146 4147 #else /* !CONFIG_NO_HZ_FULL */ 4148 static inline void sched_tick_start(int cpu) { } 4149 static inline void sched_tick_stop(int cpu) { } 4150 #endif 4151 4152 #if defined(CONFIG_PREEMPTION) && (defined(CONFIG_DEBUG_PREEMPT) || \ 4153 defined(CONFIG_TRACE_PREEMPT_TOGGLE)) 4154 /* 4155 * If the value passed in is equal to the current preempt count 4156 * then we just disabled preemption. Start timing the latency. 4157 */ 4158 static inline void preempt_latency_start(int val) 4159 { 4160 if (preempt_count() == val) { 4161 unsigned long ip = get_lock_parent_ip(); 4162 #ifdef CONFIG_DEBUG_PREEMPT 4163 current->preempt_disable_ip = ip; 4164 #endif 4165 trace_preempt_off(CALLER_ADDR0, ip); 4166 } 4167 } 4168 4169 void preempt_count_add(int val) 4170 { 4171 #ifdef CONFIG_DEBUG_PREEMPT 4172 /* 4173 * Underflow? 4174 */ 4175 if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) 4176 return; 4177 #endif 4178 __preempt_count_add(val); 4179 #ifdef CONFIG_DEBUG_PREEMPT 4180 /* 4181 * Spinlock count overflowing soon? 4182 */ 4183 DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >= 4184 PREEMPT_MASK - 10); 4185 #endif 4186 preempt_latency_start(val); 4187 } 4188 EXPORT_SYMBOL(preempt_count_add); 4189 NOKPROBE_SYMBOL(preempt_count_add); 4190 4191 /* 4192 * If the value passed in equals to the current preempt count 4193 * then we just enabled preemption. Stop timing the latency. 4194 */ 4195 static inline void preempt_latency_stop(int val) 4196 { 4197 if (preempt_count() == val) 4198 trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip()); 4199 } 4200 4201 void preempt_count_sub(int val) 4202 { 4203 #ifdef CONFIG_DEBUG_PREEMPT 4204 /* 4205 * Underflow? 4206 */ 4207 if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) 4208 return; 4209 /* 4210 * Is the spinlock portion underflowing? 4211 */ 4212 if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) && 4213 !(preempt_count() & PREEMPT_MASK))) 4214 return; 4215 #endif 4216 4217 preempt_latency_stop(val); 4218 __preempt_count_sub(val); 4219 } 4220 EXPORT_SYMBOL(preempt_count_sub); 4221 NOKPROBE_SYMBOL(preempt_count_sub); 4222 4223 #else 4224 static inline void preempt_latency_start(int val) { } 4225 static inline void preempt_latency_stop(int val) { } 4226 #endif 4227 4228 static inline unsigned long get_preempt_disable_ip(struct task_struct *p) 4229 { 4230 #ifdef CONFIG_DEBUG_PREEMPT 4231 return p->preempt_disable_ip; 4232 #else 4233 return 0; 4234 #endif 4235 } 4236 4237 /* 4238 * Print scheduling while atomic bug: 4239 */ 4240 static noinline void __schedule_bug(struct task_struct *prev) 4241 { 4242 /* Save this before calling printk(), since that will clobber it */ 4243 unsigned long preempt_disable_ip = get_preempt_disable_ip(current); 4244 4245 if (oops_in_progress) 4246 return; 4247 4248 printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n", 4249 prev->comm, prev->pid, preempt_count()); 4250 4251 debug_show_held_locks(prev); 4252 print_modules(); 4253 if (irqs_disabled()) 4254 print_irqtrace_events(prev); 4255 if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) 4256 && in_atomic_preempt_off()) { 4257 pr_err("Preemption disabled at:"); 4258 print_ip_sym(KERN_ERR, preempt_disable_ip); 4259 } 4260 if (panic_on_warn) 4261 panic("scheduling while atomic\n"); 4262 4263 dump_stack(); 4264 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 4265 } 4266 4267 /* 4268 * Various schedule()-time debugging checks and statistics: 4269 */ 4270 static inline void schedule_debug(struct task_struct *prev, bool preempt) 4271 { 4272 #ifdef CONFIG_SCHED_STACK_END_CHECK 4273 if (task_stack_end_corrupted(prev)) 4274 panic("corrupted stack end detected inside scheduler\n"); 4275 4276 if (task_scs_end_corrupted(prev)) 4277 panic("corrupted shadow stack detected inside scheduler\n"); 4278 #endif 4279 4280 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP 4281 if (!preempt && prev->state && prev->non_block_count) { 4282 printk(KERN_ERR "BUG: scheduling in a non-blocking section: %s/%d/%i\n", 4283 prev->comm, prev->pid, prev->non_block_count); 4284 dump_stack(); 4285 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 4286 } 4287 #endif 4288 4289 if (unlikely(in_atomic_preempt_off())) { 4290 __schedule_bug(prev); 4291 preempt_count_set(PREEMPT_DISABLED); 4292 } 4293 rcu_sleep_check(); 4294 SCHED_WARN_ON(ct_state() == CONTEXT_USER); 4295 4296 profile_hit(SCHED_PROFILING, __builtin_return_address(0)); 4297 4298 schedstat_inc(this_rq()->sched_count); 4299 } 4300 4301 static void put_prev_task_balance(struct rq *rq, struct task_struct *prev, 4302 struct rq_flags *rf) 4303 { 4304 #ifdef CONFIG_SMP 4305 const struct sched_class *class; 4306 /* 4307 * We must do the balancing pass before put_prev_task(), such 4308 * that when we release the rq->lock the task is in the same 4309 * state as before we took rq->lock. 4310 * 4311 * We can terminate the balance pass as soon as we know there is 4312 * a runnable task of @class priority or higher. 4313 */ 4314 for_class_range(class, prev->sched_class, &idle_sched_class) { 4315 if (class->balance(rq, prev, rf)) 4316 break; 4317 } 4318 #endif 4319 4320 put_prev_task(rq, prev); 4321 } 4322 4323 /* 4324 * Pick up the highest-prio task: 4325 */ 4326 static inline struct task_struct * 4327 pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 4328 { 4329 const struct sched_class *class; 4330 struct task_struct *p; 4331 4332 /* 4333 * Optimization: we know that if all tasks are in the fair class we can 4334 * call that function directly, but only if the @prev task wasn't of a 4335 * higher scheduling class, because otherwise those loose the 4336 * opportunity to pull in more work from other CPUs. 4337 */ 4338 if (likely(prev->sched_class <= &fair_sched_class && 4339 rq->nr_running == rq->cfs.h_nr_running)) { 4340 4341 p = pick_next_task_fair(rq, prev, rf); 4342 if (unlikely(p == RETRY_TASK)) 4343 goto restart; 4344 4345 /* Assumes fair_sched_class->next == idle_sched_class */ 4346 if (!p) { 4347 put_prev_task(rq, prev); 4348 p = pick_next_task_idle(rq); 4349 } 4350 4351 return p; 4352 } 4353 4354 restart: 4355 put_prev_task_balance(rq, prev, rf); 4356 4357 for_each_class(class) { 4358 p = class->pick_next_task(rq); 4359 if (p) 4360 return p; 4361 } 4362 4363 /* The idle class should always have a runnable task: */ 4364 BUG(); 4365 } 4366 4367 /* 4368 * __schedule() is the main scheduler function. 4369 * 4370 * The main means of driving the scheduler and thus entering this function are: 4371 * 4372 * 1. Explicit blocking: mutex, semaphore, waitqueue, etc. 4373 * 4374 * 2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return 4375 * paths. For example, see arch/x86/entry_64.S. 4376 * 4377 * To drive preemption between tasks, the scheduler sets the flag in timer 4378 * interrupt handler scheduler_tick(). 4379 * 4380 * 3. Wakeups don't really cause entry into schedule(). They add a 4381 * task to the run-queue and that's it. 4382 * 4383 * Now, if the new task added to the run-queue preempts the current 4384 * task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets 4385 * called on the nearest possible occasion: 4386 * 4387 * - If the kernel is preemptible (CONFIG_PREEMPTION=y): 4388 * 4389 * - in syscall or exception context, at the next outmost 4390 * preempt_enable(). (this might be as soon as the wake_up()'s 4391 * spin_unlock()!) 4392 * 4393 * - in IRQ context, return from interrupt-handler to 4394 * preemptible context 4395 * 4396 * - If the kernel is not preemptible (CONFIG_PREEMPTION is not set) 4397 * then at the next: 4398 * 4399 * - cond_resched() call 4400 * - explicit schedule() call 4401 * - return from syscall or exception to user-space 4402 * - return from interrupt-handler to user-space 4403 * 4404 * WARNING: must be called with preemption disabled! 4405 */ 4406 static void __sched notrace __schedule(bool preempt) 4407 { 4408 struct task_struct *prev, *next; 4409 unsigned long *switch_count; 4410 unsigned long prev_state; 4411 struct rq_flags rf; 4412 struct rq *rq; 4413 int cpu; 4414 4415 cpu = smp_processor_id(); 4416 rq = cpu_rq(cpu); 4417 prev = rq->curr; 4418 4419 schedule_debug(prev, preempt); 4420 4421 if (sched_feat(HRTICK)) 4422 hrtick_clear(rq); 4423 4424 local_irq_disable(); 4425 rcu_note_context_switch(preempt); 4426 4427 /* 4428 * Make sure that signal_pending_state()->signal_pending() below 4429 * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE) 4430 * done by the caller to avoid the race with signal_wake_up(): 4431 * 4432 * __set_current_state(@state) signal_wake_up() 4433 * schedule() set_tsk_thread_flag(p, TIF_SIGPENDING) 4434 * wake_up_state(p, state) 4435 * LOCK rq->lock LOCK p->pi_state 4436 * smp_mb__after_spinlock() smp_mb__after_spinlock() 4437 * if (signal_pending_state()) if (p->state & @state) 4438 * 4439 * Also, the membarrier system call requires a full memory barrier 4440 * after coming from user-space, before storing to rq->curr. 4441 */ 4442 rq_lock(rq, &rf); 4443 smp_mb__after_spinlock(); 4444 4445 /* Promote REQ to ACT */ 4446 rq->clock_update_flags <<= 1; 4447 update_rq_clock(rq); 4448 4449 switch_count = &prev->nivcsw; 4450 4451 /* 4452 * We must load prev->state once (task_struct::state is volatile), such 4453 * that: 4454 * 4455 * - we form a control dependency vs deactivate_task() below. 4456 * - ptrace_{,un}freeze_traced() can change ->state underneath us. 4457 */ 4458 prev_state = prev->state; 4459 if (!preempt && prev_state) { 4460 if (signal_pending_state(prev_state, prev)) { 4461 prev->state = TASK_RUNNING; 4462 } else { 4463 prev->sched_contributes_to_load = 4464 (prev_state & TASK_UNINTERRUPTIBLE) && 4465 !(prev_state & TASK_NOLOAD) && 4466 !(prev->flags & PF_FROZEN); 4467 4468 if (prev->sched_contributes_to_load) 4469 rq->nr_uninterruptible++; 4470 4471 /* 4472 * __schedule() ttwu() 4473 * prev_state = prev->state; if (p->on_rq && ...) 4474 * if (prev_state) goto out; 4475 * p->on_rq = 0; smp_acquire__after_ctrl_dep(); 4476 * p->state = TASK_WAKING 4477 * 4478 * Where __schedule() and ttwu() have matching control dependencies. 4479 * 4480 * After this, schedule() must not care about p->state any more. 4481 */ 4482 deactivate_task(rq, prev, DEQUEUE_SLEEP | DEQUEUE_NOCLOCK); 4483 4484 if (prev->in_iowait) { 4485 atomic_inc(&rq->nr_iowait); 4486 delayacct_blkio_start(); 4487 } 4488 } 4489 switch_count = &prev->nvcsw; 4490 } 4491 4492 next = pick_next_task(rq, prev, &rf); 4493 clear_tsk_need_resched(prev); 4494 clear_preempt_need_resched(); 4495 4496 if (likely(prev != next)) { 4497 rq->nr_switches++; 4498 /* 4499 * RCU users of rcu_dereference(rq->curr) may not see 4500 * changes to task_struct made by pick_next_task(). 4501 */ 4502 RCU_INIT_POINTER(rq->curr, next); 4503 /* 4504 * The membarrier system call requires each architecture 4505 * to have a full memory barrier after updating 4506 * rq->curr, before returning to user-space. 4507 * 4508 * Here are the schemes providing that barrier on the 4509 * various architectures: 4510 * - mm ? switch_mm() : mmdrop() for x86, s390, sparc, PowerPC. 4511 * switch_mm() rely on membarrier_arch_switch_mm() on PowerPC. 4512 * - finish_lock_switch() for weakly-ordered 4513 * architectures where spin_unlock is a full barrier, 4514 * - switch_to() for arm64 (weakly-ordered, spin_unlock 4515 * is a RELEASE barrier), 4516 */ 4517 ++*switch_count; 4518 4519 psi_sched_switch(prev, next, !task_on_rq_queued(prev)); 4520 4521 trace_sched_switch(preempt, prev, next); 4522 4523 /* Also unlocks the rq: */ 4524 rq = context_switch(rq, prev, next, &rf); 4525 } else { 4526 rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP); 4527 rq_unlock_irq(rq, &rf); 4528 } 4529 4530 balance_callback(rq); 4531 } 4532 4533 void __noreturn do_task_dead(void) 4534 { 4535 /* Causes final put_task_struct in finish_task_switch(): */ 4536 set_special_state(TASK_DEAD); 4537 4538 /* Tell freezer to ignore us: */ 4539 current->flags |= PF_NOFREEZE; 4540 4541 __schedule(false); 4542 BUG(); 4543 4544 /* Avoid "noreturn function does return" - but don't continue if BUG() is a NOP: */ 4545 for (;;) 4546 cpu_relax(); 4547 } 4548 4549 static inline void sched_submit_work(struct task_struct *tsk) 4550 { 4551 unsigned int task_flags; 4552 4553 if (!tsk->state) 4554 return; 4555 4556 task_flags = tsk->flags; 4557 /* 4558 * If a worker went to sleep, notify and ask workqueue whether 4559 * it wants to wake up a task to maintain concurrency. 4560 * As this function is called inside the schedule() context, 4561 * we disable preemption to avoid it calling schedule() again 4562 * in the possible wakeup of a kworker and because wq_worker_sleeping() 4563 * requires it. 4564 */ 4565 if (task_flags & (PF_WQ_WORKER | PF_IO_WORKER)) { 4566 preempt_disable(); 4567 if (task_flags & PF_WQ_WORKER) 4568 wq_worker_sleeping(tsk); 4569 else 4570 io_wq_worker_sleeping(tsk); 4571 preempt_enable_no_resched(); 4572 } 4573 4574 if (tsk_is_pi_blocked(tsk)) 4575 return; 4576 4577 /* 4578 * If we are going to sleep and we have plugged IO queued, 4579 * make sure to submit it to avoid deadlocks. 4580 */ 4581 if (blk_needs_flush_plug(tsk)) 4582 blk_schedule_flush_plug(tsk); 4583 } 4584 4585 static void sched_update_worker(struct task_struct *tsk) 4586 { 4587 if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER)) { 4588 if (tsk->flags & PF_WQ_WORKER) 4589 wq_worker_running(tsk); 4590 else 4591 io_wq_worker_running(tsk); 4592 } 4593 } 4594 4595 asmlinkage __visible void __sched schedule(void) 4596 { 4597 struct task_struct *tsk = current; 4598 4599 sched_submit_work(tsk); 4600 do { 4601 preempt_disable(); 4602 __schedule(false); 4603 sched_preempt_enable_no_resched(); 4604 } while (need_resched()); 4605 sched_update_worker(tsk); 4606 } 4607 EXPORT_SYMBOL(schedule); 4608 4609 /* 4610 * synchronize_rcu_tasks() makes sure that no task is stuck in preempted 4611 * state (have scheduled out non-voluntarily) by making sure that all 4612 * tasks have either left the run queue or have gone into user space. 4613 * As idle tasks do not do either, they must not ever be preempted 4614 * (schedule out non-voluntarily). 4615 * 4616 * schedule_idle() is similar to schedule_preempt_disable() except that it 4617 * never enables preemption because it does not call sched_submit_work(). 4618 */ 4619 void __sched schedule_idle(void) 4620 { 4621 /* 4622 * As this skips calling sched_submit_work(), which the idle task does 4623 * regardless because that function is a nop when the task is in a 4624 * TASK_RUNNING state, make sure this isn't used someplace that the 4625 * current task can be in any other state. Note, idle is always in the 4626 * TASK_RUNNING state. 4627 */ 4628 WARN_ON_ONCE(current->state); 4629 do { 4630 __schedule(false); 4631 } while (need_resched()); 4632 } 4633 4634 #if defined(CONFIG_CONTEXT_TRACKING) && !defined(CONFIG_HAVE_CONTEXT_TRACKING_OFFSTACK) 4635 asmlinkage __visible void __sched schedule_user(void) 4636 { 4637 /* 4638 * If we come here after a random call to set_need_resched(), 4639 * or we have been woken up remotely but the IPI has not yet arrived, 4640 * we haven't yet exited the RCU idle mode. Do it here manually until 4641 * we find a better solution. 4642 * 4643 * NB: There are buggy callers of this function. Ideally we 4644 * should warn if prev_state != CONTEXT_USER, but that will trigger 4645 * too frequently to make sense yet. 4646 */ 4647 enum ctx_state prev_state = exception_enter(); 4648 schedule(); 4649 exception_exit(prev_state); 4650 } 4651 #endif 4652 4653 /** 4654 * schedule_preempt_disabled - called with preemption disabled 4655 * 4656 * Returns with preemption disabled. Note: preempt_count must be 1 4657 */ 4658 void __sched schedule_preempt_disabled(void) 4659 { 4660 sched_preempt_enable_no_resched(); 4661 schedule(); 4662 preempt_disable(); 4663 } 4664 4665 static void __sched notrace preempt_schedule_common(void) 4666 { 4667 do { 4668 /* 4669 * Because the function tracer can trace preempt_count_sub() 4670 * and it also uses preempt_enable/disable_notrace(), if 4671 * NEED_RESCHED is set, the preempt_enable_notrace() called 4672 * by the function tracer will call this function again and 4673 * cause infinite recursion. 4674 * 4675 * Preemption must be disabled here before the function 4676 * tracer can trace. Break up preempt_disable() into two 4677 * calls. One to disable preemption without fear of being 4678 * traced. The other to still record the preemption latency, 4679 * which can also be traced by the function tracer. 4680 */ 4681 preempt_disable_notrace(); 4682 preempt_latency_start(1); 4683 __schedule(true); 4684 preempt_latency_stop(1); 4685 preempt_enable_no_resched_notrace(); 4686 4687 /* 4688 * Check again in case we missed a preemption opportunity 4689 * between schedule and now. 4690 */ 4691 } while (need_resched()); 4692 } 4693 4694 #ifdef CONFIG_PREEMPTION 4695 /* 4696 * This is the entry point to schedule() from in-kernel preemption 4697 * off of preempt_enable. 4698 */ 4699 asmlinkage __visible void __sched notrace preempt_schedule(void) 4700 { 4701 /* 4702 * If there is a non-zero preempt_count or interrupts are disabled, 4703 * we do not want to preempt the current task. Just return.. 4704 */ 4705 if (likely(!preemptible())) 4706 return; 4707 4708 preempt_schedule_common(); 4709 } 4710 NOKPROBE_SYMBOL(preempt_schedule); 4711 EXPORT_SYMBOL(preempt_schedule); 4712 4713 /** 4714 * preempt_schedule_notrace - preempt_schedule called by tracing 4715 * 4716 * The tracing infrastructure uses preempt_enable_notrace to prevent 4717 * recursion and tracing preempt enabling caused by the tracing 4718 * infrastructure itself. But as tracing can happen in areas coming 4719 * from userspace or just about to enter userspace, a preempt enable 4720 * can occur before user_exit() is called. This will cause the scheduler 4721 * to be called when the system is still in usermode. 4722 * 4723 * To prevent this, the preempt_enable_notrace will use this function 4724 * instead of preempt_schedule() to exit user context if needed before 4725 * calling the scheduler. 4726 */ 4727 asmlinkage __visible void __sched notrace preempt_schedule_notrace(void) 4728 { 4729 enum ctx_state prev_ctx; 4730 4731 if (likely(!preemptible())) 4732 return; 4733 4734 do { 4735 /* 4736 * Because the function tracer can trace preempt_count_sub() 4737 * and it also uses preempt_enable/disable_notrace(), if 4738 * NEED_RESCHED is set, the preempt_enable_notrace() called 4739 * by the function tracer will call this function again and 4740 * cause infinite recursion. 4741 * 4742 * Preemption must be disabled here before the function 4743 * tracer can trace. Break up preempt_disable() into two 4744 * calls. One to disable preemption without fear of being 4745 * traced. The other to still record the preemption latency, 4746 * which can also be traced by the function tracer. 4747 */ 4748 preempt_disable_notrace(); 4749 preempt_latency_start(1); 4750 /* 4751 * Needs preempt disabled in case user_exit() is traced 4752 * and the tracer calls preempt_enable_notrace() causing 4753 * an infinite recursion. 4754 */ 4755 prev_ctx = exception_enter(); 4756 __schedule(true); 4757 exception_exit(prev_ctx); 4758 4759 preempt_latency_stop(1); 4760 preempt_enable_no_resched_notrace(); 4761 } while (need_resched()); 4762 } 4763 EXPORT_SYMBOL_GPL(preempt_schedule_notrace); 4764 4765 #endif /* CONFIG_PREEMPTION */ 4766 4767 /* 4768 * This is the entry point to schedule() from kernel preemption 4769 * off of irq context. 4770 * Note, that this is called and return with irqs disabled. This will 4771 * protect us against recursive calling from irq. 4772 */ 4773 asmlinkage __visible void __sched preempt_schedule_irq(void) 4774 { 4775 enum ctx_state prev_state; 4776 4777 /* Catch callers which need to be fixed */ 4778 BUG_ON(preempt_count() || !irqs_disabled()); 4779 4780 prev_state = exception_enter(); 4781 4782 do { 4783 preempt_disable(); 4784 local_irq_enable(); 4785 __schedule(true); 4786 local_irq_disable(); 4787 sched_preempt_enable_no_resched(); 4788 } while (need_resched()); 4789 4790 exception_exit(prev_state); 4791 } 4792 4793 int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flags, 4794 void *key) 4795 { 4796 WARN_ON_ONCE(IS_ENABLED(CONFIG_SCHED_DEBUG) && wake_flags & ~WF_SYNC); 4797 return try_to_wake_up(curr->private, mode, wake_flags); 4798 } 4799 EXPORT_SYMBOL(default_wake_function); 4800 4801 #ifdef CONFIG_RT_MUTEXES 4802 4803 static inline int __rt_effective_prio(struct task_struct *pi_task, int prio) 4804 { 4805 if (pi_task) 4806 prio = min(prio, pi_task->prio); 4807 4808 return prio; 4809 } 4810 4811 static inline int rt_effective_prio(struct task_struct *p, int prio) 4812 { 4813 struct task_struct *pi_task = rt_mutex_get_top_task(p); 4814 4815 return __rt_effective_prio(pi_task, prio); 4816 } 4817 4818 /* 4819 * rt_mutex_setprio - set the current priority of a task 4820 * @p: task to boost 4821 * @pi_task: donor task 4822 * 4823 * This function changes the 'effective' priority of a task. It does 4824 * not touch ->normal_prio like __setscheduler(). 4825 * 4826 * Used by the rt_mutex code to implement priority inheritance 4827 * logic. Call site only calls if the priority of the task changed. 4828 */ 4829 void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) 4830 { 4831 int prio, oldprio, queued, running, queue_flag = 4832 DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; 4833 const struct sched_class *prev_class; 4834 struct rq_flags rf; 4835 struct rq *rq; 4836 4837 /* XXX used to be waiter->prio, not waiter->task->prio */ 4838 prio = __rt_effective_prio(pi_task, p->normal_prio); 4839 4840 /* 4841 * If nothing changed; bail early. 4842 */ 4843 if (p->pi_top_task == pi_task && prio == p->prio && !dl_prio(prio)) 4844 return; 4845 4846 rq = __task_rq_lock(p, &rf); 4847 update_rq_clock(rq); 4848 /* 4849 * Set under pi_lock && rq->lock, such that the value can be used under 4850 * either lock. 4851 * 4852 * Note that there is loads of tricky to make this pointer cache work 4853 * right. rt_mutex_slowunlock()+rt_mutex_postunlock() work together to 4854 * ensure a task is de-boosted (pi_task is set to NULL) before the 4855 * task is allowed to run again (and can exit). This ensures the pointer 4856 * points to a blocked task -- which guaratees the task is present. 4857 */ 4858 p->pi_top_task = pi_task; 4859 4860 /* 4861 * For FIFO/RR we only need to set prio, if that matches we're done. 4862 */ 4863 if (prio == p->prio && !dl_prio(prio)) 4864 goto out_unlock; 4865 4866 /* 4867 * Idle task boosting is a nono in general. There is one 4868 * exception, when PREEMPT_RT and NOHZ is active: 4869 * 4870 * The idle task calls get_next_timer_interrupt() and holds 4871 * the timer wheel base->lock on the CPU and another CPU wants 4872 * to access the timer (probably to cancel it). We can safely 4873 * ignore the boosting request, as the idle CPU runs this code 4874 * with interrupts disabled and will complete the lock 4875 * protected section without being interrupted. So there is no 4876 * real need to boost. 4877 */ 4878 if (unlikely(p == rq->idle)) { 4879 WARN_ON(p != rq->curr); 4880 WARN_ON(p->pi_blocked_on); 4881 goto out_unlock; 4882 } 4883 4884 trace_sched_pi_setprio(p, pi_task); 4885 oldprio = p->prio; 4886 4887 if (oldprio == prio) 4888 queue_flag &= ~DEQUEUE_MOVE; 4889 4890 prev_class = p->sched_class; 4891 queued = task_on_rq_queued(p); 4892 running = task_current(rq, p); 4893 if (queued) 4894 dequeue_task(rq, p, queue_flag); 4895 if (running) 4896 put_prev_task(rq, p); 4897 4898 /* 4899 * Boosting condition are: 4900 * 1. -rt task is running and holds mutex A 4901 * --> -dl task blocks on mutex A 4902 * 4903 * 2. -dl task is running and holds mutex A 4904 * --> -dl task blocks on mutex A and could preempt the 4905 * running task 4906 */ 4907 if (dl_prio(prio)) { 4908 if (!dl_prio(p->normal_prio) || 4909 (pi_task && dl_prio(pi_task->prio) && 4910 dl_entity_preempt(&pi_task->dl, &p->dl))) { 4911 p->dl.dl_boosted = 1; 4912 queue_flag |= ENQUEUE_REPLENISH; 4913 } else 4914 p->dl.dl_boosted = 0; 4915 p->sched_class = &dl_sched_class; 4916 } else if (rt_prio(prio)) { 4917 if (dl_prio(oldprio)) 4918 p->dl.dl_boosted = 0; 4919 if (oldprio < prio) 4920 queue_flag |= ENQUEUE_HEAD; 4921 p->sched_class = &rt_sched_class; 4922 } else { 4923 if (dl_prio(oldprio)) 4924 p->dl.dl_boosted = 0; 4925 if (rt_prio(oldprio)) 4926 p->rt.timeout = 0; 4927 p->sched_class = &fair_sched_class; 4928 } 4929 4930 p->prio = prio; 4931 4932 if (queued) 4933 enqueue_task(rq, p, queue_flag); 4934 if (running) 4935 set_next_task(rq, p); 4936 4937 check_class_changed(rq, p, prev_class, oldprio); 4938 out_unlock: 4939 /* Avoid rq from going away on us: */ 4940 preempt_disable(); 4941 __task_rq_unlock(rq, &rf); 4942 4943 balance_callback(rq); 4944 preempt_enable(); 4945 } 4946 #else 4947 static inline int rt_effective_prio(struct task_struct *p, int prio) 4948 { 4949 return prio; 4950 } 4951 #endif 4952 4953 void set_user_nice(struct task_struct *p, long nice) 4954 { 4955 bool queued, running; 4956 int old_prio; 4957 struct rq_flags rf; 4958 struct rq *rq; 4959 4960 if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE) 4961 return; 4962 /* 4963 * We have to be careful, if called from sys_setpriority(), 4964 * the task might be in the middle of scheduling on another CPU. 4965 */ 4966 rq = task_rq_lock(p, &rf); 4967 update_rq_clock(rq); 4968 4969 /* 4970 * The RT priorities are set via sched_setscheduler(), but we still 4971 * allow the 'normal' nice value to be set - but as expected 4972 * it wont have any effect on scheduling until the task is 4973 * SCHED_DEADLINE, SCHED_FIFO or SCHED_RR: 4974 */ 4975 if (task_has_dl_policy(p) || task_has_rt_policy(p)) { 4976 p->static_prio = NICE_TO_PRIO(nice); 4977 goto out_unlock; 4978 } 4979 queued = task_on_rq_queued(p); 4980 running = task_current(rq, p); 4981 if (queued) 4982 dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); 4983 if (running) 4984 put_prev_task(rq, p); 4985 4986 p->static_prio = NICE_TO_PRIO(nice); 4987 set_load_weight(p, true); 4988 old_prio = p->prio; 4989 p->prio = effective_prio(p); 4990 4991 if (queued) 4992 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); 4993 if (running) 4994 set_next_task(rq, p); 4995 4996 /* 4997 * If the task increased its priority or is running and 4998 * lowered its priority, then reschedule its CPU: 4999 */ 5000 p->sched_class->prio_changed(rq, p, old_prio); 5001 5002 out_unlock: 5003 task_rq_unlock(rq, p, &rf); 5004 } 5005 EXPORT_SYMBOL(set_user_nice); 5006 5007 /* 5008 * can_nice - check if a task can reduce its nice value 5009 * @p: task 5010 * @nice: nice value 5011 */ 5012 int can_nice(const struct task_struct *p, const int nice) 5013 { 5014 /* Convert nice value [19,-20] to rlimit style value [1,40]: */ 5015 int nice_rlim = nice_to_rlimit(nice); 5016 5017 return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) || 5018 capable(CAP_SYS_NICE)); 5019 } 5020 5021 #ifdef __ARCH_WANT_SYS_NICE 5022 5023 /* 5024 * sys_nice - change the priority of the current process. 5025 * @increment: priority increment 5026 * 5027 * sys_setpriority is a more generic, but much slower function that 5028 * does similar things. 5029 */ 5030 SYSCALL_DEFINE1(nice, int, increment) 5031 { 5032 long nice, retval; 5033 5034 /* 5035 * Setpriority might change our priority at the same moment. 5036 * We don't have to worry. Conceptually one call occurs first 5037 * and we have a single winner. 5038 */ 5039 increment = clamp(increment, -NICE_WIDTH, NICE_WIDTH); 5040 nice = task_nice(current) + increment; 5041 5042 nice = clamp_val(nice, MIN_NICE, MAX_NICE); 5043 if (increment < 0 && !can_nice(current, nice)) 5044 return -EPERM; 5045 5046 retval = security_task_setnice(current, nice); 5047 if (retval) 5048 return retval; 5049 5050 set_user_nice(current, nice); 5051 return 0; 5052 } 5053 5054 #endif 5055 5056 /** 5057 * task_prio - return the priority value of a given task. 5058 * @p: the task in question. 5059 * 5060 * Return: The priority value as seen by users in /proc. 5061 * RT tasks are offset by -200. Normal tasks are centered 5062 * around 0, value goes from -16 to +15. 5063 */ 5064 int task_prio(const struct task_struct *p) 5065 { 5066 return p->prio - MAX_RT_PRIO; 5067 } 5068 5069 /** 5070 * idle_cpu - is a given CPU idle currently? 5071 * @cpu: the processor in question. 5072 * 5073 * Return: 1 if the CPU is currently idle. 0 otherwise. 5074 */ 5075 int idle_cpu(int cpu) 5076 { 5077 struct rq *rq = cpu_rq(cpu); 5078 5079 if (rq->curr != rq->idle) 5080 return 0; 5081 5082 if (rq->nr_running) 5083 return 0; 5084 5085 #ifdef CONFIG_SMP 5086 if (rq->ttwu_pending) 5087 return 0; 5088 #endif 5089 5090 return 1; 5091 } 5092 5093 /** 5094 * available_idle_cpu - is a given CPU idle for enqueuing work. 5095 * @cpu: the CPU in question. 5096 * 5097 * Return: 1 if the CPU is currently idle. 0 otherwise. 5098 */ 5099 int available_idle_cpu(int cpu) 5100 { 5101 if (!idle_cpu(cpu)) 5102 return 0; 5103 5104 if (vcpu_is_preempted(cpu)) 5105 return 0; 5106 5107 return 1; 5108 } 5109 5110 /** 5111 * idle_task - return the idle task for a given CPU. 5112 * @cpu: the processor in question. 5113 * 5114 * Return: The idle task for the CPU @cpu. 5115 */ 5116 struct task_struct *idle_task(int cpu) 5117 { 5118 return cpu_rq(cpu)->idle; 5119 } 5120 5121 /** 5122 * find_process_by_pid - find a process with a matching PID value. 5123 * @pid: the pid in question. 5124 * 5125 * The task of @pid, if found. %NULL otherwise. 5126 */ 5127 static struct task_struct *find_process_by_pid(pid_t pid) 5128 { 5129 return pid ? find_task_by_vpid(pid) : current; 5130 } 5131 5132 /* 5133 * sched_setparam() passes in -1 for its policy, to let the functions 5134 * it calls know not to change it. 5135 */ 5136 #define SETPARAM_POLICY -1 5137 5138 static void __setscheduler_params(struct task_struct *p, 5139 const struct sched_attr *attr) 5140 { 5141 int policy = attr->sched_policy; 5142 5143 if (policy == SETPARAM_POLICY) 5144 policy = p->policy; 5145 5146 p->policy = policy; 5147 5148 if (dl_policy(policy)) 5149 __setparam_dl(p, attr); 5150 else if (fair_policy(policy)) 5151 p->static_prio = NICE_TO_PRIO(attr->sched_nice); 5152 5153 /* 5154 * __sched_setscheduler() ensures attr->sched_priority == 0 when 5155 * !rt_policy. Always setting this ensures that things like 5156 * getparam()/getattr() don't report silly values for !rt tasks. 5157 */ 5158 p->rt_priority = attr->sched_priority; 5159 p->normal_prio = normal_prio(p); 5160 set_load_weight(p, true); 5161 } 5162 5163 /* Actually do priority change: must hold pi & rq lock. */ 5164 static void __setscheduler(struct rq *rq, struct task_struct *p, 5165 const struct sched_attr *attr, bool keep_boost) 5166 { 5167 /* 5168 * If params can't change scheduling class changes aren't allowed 5169 * either. 5170 */ 5171 if (attr->sched_flags & SCHED_FLAG_KEEP_PARAMS) 5172 return; 5173 5174 __setscheduler_params(p, attr); 5175 5176 /* 5177 * Keep a potential priority boosting if called from 5178 * sched_setscheduler(). 5179 */ 5180 p->prio = normal_prio(p); 5181 if (keep_boost) 5182 p->prio = rt_effective_prio(p, p->prio); 5183 5184 if (dl_prio(p->prio)) 5185 p->sched_class = &dl_sched_class; 5186 else if (rt_prio(p->prio)) 5187 p->sched_class = &rt_sched_class; 5188 else 5189 p->sched_class = &fair_sched_class; 5190 } 5191 5192 /* 5193 * Check the target process has a UID that matches the current process's: 5194 */ 5195 static bool check_same_owner(struct task_struct *p) 5196 { 5197 const struct cred *cred = current_cred(), *pcred; 5198 bool match; 5199 5200 rcu_read_lock(); 5201 pcred = __task_cred(p); 5202 match = (uid_eq(cred->euid, pcred->euid) || 5203 uid_eq(cred->euid, pcred->uid)); 5204 rcu_read_unlock(); 5205 return match; 5206 } 5207 5208 static int __sched_setscheduler(struct task_struct *p, 5209 const struct sched_attr *attr, 5210 bool user, bool pi) 5211 { 5212 int newprio = dl_policy(attr->sched_policy) ? MAX_DL_PRIO - 1 : 5213 MAX_RT_PRIO - 1 - attr->sched_priority; 5214 int retval, oldprio, oldpolicy = -1, queued, running; 5215 int new_effective_prio, policy = attr->sched_policy; 5216 const struct sched_class *prev_class; 5217 struct rq_flags rf; 5218 int reset_on_fork; 5219 int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; 5220 struct rq *rq; 5221 5222 /* The pi code expects interrupts enabled */ 5223 BUG_ON(pi && in_interrupt()); 5224 recheck: 5225 /* Double check policy once rq lock held: */ 5226 if (policy < 0) { 5227 reset_on_fork = p->sched_reset_on_fork; 5228 policy = oldpolicy = p->policy; 5229 } else { 5230 reset_on_fork = !!(attr->sched_flags & SCHED_FLAG_RESET_ON_FORK); 5231 5232 if (!valid_policy(policy)) 5233 return -EINVAL; 5234 } 5235 5236 if (attr->sched_flags & ~(SCHED_FLAG_ALL | SCHED_FLAG_SUGOV)) 5237 return -EINVAL; 5238 5239 /* 5240 * Valid priorities for SCHED_FIFO and SCHED_RR are 5241 * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL, 5242 * SCHED_BATCH and SCHED_IDLE is 0. 5243 */ 5244 if ((p->mm && attr->sched_priority > MAX_USER_RT_PRIO-1) || 5245 (!p->mm && attr->sched_priority > MAX_RT_PRIO-1)) 5246 return -EINVAL; 5247 if ((dl_policy(policy) && !__checkparam_dl(attr)) || 5248 (rt_policy(policy) != (attr->sched_priority != 0))) 5249 return -EINVAL; 5250 5251 /* 5252 * Allow unprivileged RT tasks to decrease priority: 5253 */ 5254 if (user && !capable(CAP_SYS_NICE)) { 5255 if (fair_policy(policy)) { 5256 if (attr->sched_nice < task_nice(p) && 5257 !can_nice(p, attr->sched_nice)) 5258 return -EPERM; 5259 } 5260 5261 if (rt_policy(policy)) { 5262 unsigned long rlim_rtprio = 5263 task_rlimit(p, RLIMIT_RTPRIO); 5264 5265 /* Can't set/change the rt policy: */ 5266 if (policy != p->policy && !rlim_rtprio) 5267 return -EPERM; 5268 5269 /* Can't increase priority: */ 5270 if (attr->sched_priority > p->rt_priority && 5271 attr->sched_priority > rlim_rtprio) 5272 return -EPERM; 5273 } 5274 5275 /* 5276 * Can't set/change SCHED_DEADLINE policy at all for now 5277 * (safest behavior); in the future we would like to allow 5278 * unprivileged DL tasks to increase their relative deadline 5279 * or reduce their runtime (both ways reducing utilization) 5280 */ 5281 if (dl_policy(policy)) 5282 return -EPERM; 5283 5284 /* 5285 * Treat SCHED_IDLE as nice 20. Only allow a switch to 5286 * SCHED_NORMAL if the RLIMIT_NICE would normally permit it. 5287 */ 5288 if (task_has_idle_policy(p) && !idle_policy(policy)) { 5289 if (!can_nice(p, task_nice(p))) 5290 return -EPERM; 5291 } 5292 5293 /* Can't change other user's priorities: */ 5294 if (!check_same_owner(p)) 5295 return -EPERM; 5296 5297 /* Normal users shall not reset the sched_reset_on_fork flag: */ 5298 if (p->sched_reset_on_fork && !reset_on_fork) 5299 return -EPERM; 5300 } 5301 5302 if (user) { 5303 if (attr->sched_flags & SCHED_FLAG_SUGOV) 5304 return -EINVAL; 5305 5306 retval = security_task_setscheduler(p); 5307 if (retval) 5308 return retval; 5309 } 5310 5311 /* Update task specific "requested" clamps */ 5312 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP) { 5313 retval = uclamp_validate(p, attr); 5314 if (retval) 5315 return retval; 5316 } 5317 5318 if (pi) 5319 cpuset_read_lock(); 5320 5321 /* 5322 * Make sure no PI-waiters arrive (or leave) while we are 5323 * changing the priority of the task: 5324 * 5325 * To be able to change p->policy safely, the appropriate 5326 * runqueue lock must be held. 5327 */ 5328 rq = task_rq_lock(p, &rf); 5329 update_rq_clock(rq); 5330 5331 /* 5332 * Changing the policy of the stop threads its a very bad idea: 5333 */ 5334 if (p == rq->stop) { 5335 retval = -EINVAL; 5336 goto unlock; 5337 } 5338 5339 /* 5340 * If not changing anything there's no need to proceed further, 5341 * but store a possible modification of reset_on_fork. 5342 */ 5343 if (unlikely(policy == p->policy)) { 5344 if (fair_policy(policy) && attr->sched_nice != task_nice(p)) 5345 goto change; 5346 if (rt_policy(policy) && attr->sched_priority != p->rt_priority) 5347 goto change; 5348 if (dl_policy(policy) && dl_param_changed(p, attr)) 5349 goto change; 5350 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP) 5351 goto change; 5352 5353 p->sched_reset_on_fork = reset_on_fork; 5354 retval = 0; 5355 goto unlock; 5356 } 5357 change: 5358 5359 if (user) { 5360 #ifdef CONFIG_RT_GROUP_SCHED 5361 /* 5362 * Do not allow realtime tasks into groups that have no runtime 5363 * assigned. 5364 */ 5365 if (rt_bandwidth_enabled() && rt_policy(policy) && 5366 task_group(p)->rt_bandwidth.rt_runtime == 0 && 5367 !task_group_is_autogroup(task_group(p))) { 5368 retval = -EPERM; 5369 goto unlock; 5370 } 5371 #endif 5372 #ifdef CONFIG_SMP 5373 if (dl_bandwidth_enabled() && dl_policy(policy) && 5374 !(attr->sched_flags & SCHED_FLAG_SUGOV)) { 5375 cpumask_t *span = rq->rd->span; 5376 5377 /* 5378 * Don't allow tasks with an affinity mask smaller than 5379 * the entire root_domain to become SCHED_DEADLINE. We 5380 * will also fail if there's no bandwidth available. 5381 */ 5382 if (!cpumask_subset(span, p->cpus_ptr) || 5383 rq->rd->dl_bw.bw == 0) { 5384 retval = -EPERM; 5385 goto unlock; 5386 } 5387 } 5388 #endif 5389 } 5390 5391 /* Re-check policy now with rq lock held: */ 5392 if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { 5393 policy = oldpolicy = -1; 5394 task_rq_unlock(rq, p, &rf); 5395 if (pi) 5396 cpuset_read_unlock(); 5397 goto recheck; 5398 } 5399 5400 /* 5401 * If setscheduling to SCHED_DEADLINE (or changing the parameters 5402 * of a SCHED_DEADLINE task) we need to check if enough bandwidth 5403 * is available. 5404 */ 5405 if ((dl_policy(policy) || dl_task(p)) && sched_dl_overflow(p, policy, attr)) { 5406 retval = -EBUSY; 5407 goto unlock; 5408 } 5409 5410 p->sched_reset_on_fork = reset_on_fork; 5411 oldprio = p->prio; 5412 5413 if (pi) { 5414 /* 5415 * Take priority boosted tasks into account. If the new 5416 * effective priority is unchanged, we just store the new 5417 * normal parameters and do not touch the scheduler class and 5418 * the runqueue. This will be done when the task deboost 5419 * itself. 5420 */ 5421 new_effective_prio = rt_effective_prio(p, newprio); 5422 if (new_effective_prio == oldprio) 5423 queue_flags &= ~DEQUEUE_MOVE; 5424 } 5425 5426 queued = task_on_rq_queued(p); 5427 running = task_current(rq, p); 5428 if (queued) 5429 dequeue_task(rq, p, queue_flags); 5430 if (running) 5431 put_prev_task(rq, p); 5432 5433 prev_class = p->sched_class; 5434 5435 __setscheduler(rq, p, attr, pi); 5436 __setscheduler_uclamp(p, attr); 5437 5438 if (queued) { 5439 /* 5440 * We enqueue to tail when the priority of a task is 5441 * increased (user space view). 5442 */ 5443 if (oldprio < p->prio) 5444 queue_flags |= ENQUEUE_HEAD; 5445 5446 enqueue_task(rq, p, queue_flags); 5447 } 5448 if (running) 5449 set_next_task(rq, p); 5450 5451 check_class_changed(rq, p, prev_class, oldprio); 5452 5453 /* Avoid rq from going away on us: */ 5454 preempt_disable(); 5455 task_rq_unlock(rq, p, &rf); 5456 5457 if (pi) { 5458 cpuset_read_unlock(); 5459 rt_mutex_adjust_pi(p); 5460 } 5461 5462 /* Run balance callbacks after we've adjusted the PI chain: */ 5463 balance_callback(rq); 5464 preempt_enable(); 5465 5466 return 0; 5467 5468 unlock: 5469 task_rq_unlock(rq, p, &rf); 5470 if (pi) 5471 cpuset_read_unlock(); 5472 return retval; 5473 } 5474 5475 static int _sched_setscheduler(struct task_struct *p, int policy, 5476 const struct sched_param *param, bool check) 5477 { 5478 struct sched_attr attr = { 5479 .sched_policy = policy, 5480 .sched_priority = param->sched_priority, 5481 .sched_nice = PRIO_TO_NICE(p->static_prio), 5482 }; 5483 5484 /* Fixup the legacy SCHED_RESET_ON_FORK hack. */ 5485 if ((policy != SETPARAM_POLICY) && (policy & SCHED_RESET_ON_FORK)) { 5486 attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK; 5487 policy &= ~SCHED_RESET_ON_FORK; 5488 attr.sched_policy = policy; 5489 } 5490 5491 return __sched_setscheduler(p, &attr, check, true); 5492 } 5493 /** 5494 * sched_setscheduler - change the scheduling policy and/or RT priority of a thread. 5495 * @p: the task in question. 5496 * @policy: new policy. 5497 * @param: structure containing the new RT priority. 5498 * 5499 * Use sched_set_fifo(), read its comment. 5500 * 5501 * Return: 0 on success. An error code otherwise. 5502 * 5503 * NOTE that the task may be already dead. 5504 */ 5505 int sched_setscheduler(struct task_struct *p, int policy, 5506 const struct sched_param *param) 5507 { 5508 return _sched_setscheduler(p, policy, param, true); 5509 } 5510 5511 int sched_setattr(struct task_struct *p, const struct sched_attr *attr) 5512 { 5513 return __sched_setscheduler(p, attr, true, true); 5514 } 5515 5516 int sched_setattr_nocheck(struct task_struct *p, const struct sched_attr *attr) 5517 { 5518 return __sched_setscheduler(p, attr, false, true); 5519 } 5520 5521 /** 5522 * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace. 5523 * @p: the task in question. 5524 * @policy: new policy. 5525 * @param: structure containing the new RT priority. 5526 * 5527 * Just like sched_setscheduler, only don't bother checking if the 5528 * current context has permission. For example, this is needed in 5529 * stop_machine(): we create temporary high priority worker threads, 5530 * but our caller might not have that capability. 5531 * 5532 * Return: 0 on success. An error code otherwise. 5533 */ 5534 int sched_setscheduler_nocheck(struct task_struct *p, int policy, 5535 const struct sched_param *param) 5536 { 5537 return _sched_setscheduler(p, policy, param, false); 5538 } 5539 5540 /* 5541 * SCHED_FIFO is a broken scheduler model; that is, it is fundamentally 5542 * incapable of resource management, which is the one thing an OS really should 5543 * be doing. 5544 * 5545 * This is of course the reason it is limited to privileged users only. 5546 * 5547 * Worse still; it is fundamentally impossible to compose static priority 5548 * workloads. You cannot take two correctly working static prio workloads 5549 * and smash them together and still expect them to work. 5550 * 5551 * For this reason 'all' FIFO tasks the kernel creates are basically at: 5552 * 5553 * MAX_RT_PRIO / 2 5554 * 5555 * The administrator _MUST_ configure the system, the kernel simply doesn't 5556 * know enough information to make a sensible choice. 5557 */ 5558 void sched_set_fifo(struct task_struct *p) 5559 { 5560 struct sched_param sp = { .sched_priority = MAX_RT_PRIO / 2 }; 5561 WARN_ON_ONCE(sched_setscheduler_nocheck(p, SCHED_FIFO, &sp) != 0); 5562 } 5563 EXPORT_SYMBOL_GPL(sched_set_fifo); 5564 5565 /* 5566 * For when you don't much care about FIFO, but want to be above SCHED_NORMAL. 5567 */ 5568 void sched_set_fifo_low(struct task_struct *p) 5569 { 5570 struct sched_param sp = { .sched_priority = 1 }; 5571 WARN_ON_ONCE(sched_setscheduler_nocheck(p, SCHED_FIFO, &sp) != 0); 5572 } 5573 EXPORT_SYMBOL_GPL(sched_set_fifo_low); 5574 5575 void sched_set_normal(struct task_struct *p, int nice) 5576 { 5577 struct sched_attr attr = { 5578 .sched_policy = SCHED_NORMAL, 5579 .sched_nice = nice, 5580 }; 5581 WARN_ON_ONCE(sched_setattr_nocheck(p, &attr) != 0); 5582 } 5583 EXPORT_SYMBOL_GPL(sched_set_normal); 5584 5585 static int 5586 do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param) 5587 { 5588 struct sched_param lparam; 5589 struct task_struct *p; 5590 int retval; 5591 5592 if (!param || pid < 0) 5593 return -EINVAL; 5594 if (copy_from_user(&lparam, param, sizeof(struct sched_param))) 5595 return -EFAULT; 5596 5597 rcu_read_lock(); 5598 retval = -ESRCH; 5599 p = find_process_by_pid(pid); 5600 if (likely(p)) 5601 get_task_struct(p); 5602 rcu_read_unlock(); 5603 5604 if (likely(p)) { 5605 retval = sched_setscheduler(p, policy, &lparam); 5606 put_task_struct(p); 5607 } 5608 5609 return retval; 5610 } 5611 5612 /* 5613 * Mimics kernel/events/core.c perf_copy_attr(). 5614 */ 5615 static int sched_copy_attr(struct sched_attr __user *uattr, struct sched_attr *attr) 5616 { 5617 u32 size; 5618 int ret; 5619 5620 /* Zero the full structure, so that a short copy will be nice: */ 5621 memset(attr, 0, sizeof(*attr)); 5622 5623 ret = get_user(size, &uattr->size); 5624 if (ret) 5625 return ret; 5626 5627 /* ABI compatibility quirk: */ 5628 if (!size) 5629 size = SCHED_ATTR_SIZE_VER0; 5630 if (size < SCHED_ATTR_SIZE_VER0 || size > PAGE_SIZE) 5631 goto err_size; 5632 5633 ret = copy_struct_from_user(attr, sizeof(*attr), uattr, size); 5634 if (ret) { 5635 if (ret == -E2BIG) 5636 goto err_size; 5637 return ret; 5638 } 5639 5640 if ((attr->sched_flags & SCHED_FLAG_UTIL_CLAMP) && 5641 size < SCHED_ATTR_SIZE_VER1) 5642 return -EINVAL; 5643 5644 /* 5645 * XXX: Do we want to be lenient like existing syscalls; or do we want 5646 * to be strict and return an error on out-of-bounds values? 5647 */ 5648 attr->sched_nice = clamp(attr->sched_nice, MIN_NICE, MAX_NICE); 5649 5650 return 0; 5651 5652 err_size: 5653 put_user(sizeof(*attr), &uattr->size); 5654 return -E2BIG; 5655 } 5656 5657 /** 5658 * sys_sched_setscheduler - set/change the scheduler policy and RT priority 5659 * @pid: the pid in question. 5660 * @policy: new policy. 5661 * @param: structure containing the new RT priority. 5662 * 5663 * Return: 0 on success. An error code otherwise. 5664 */ 5665 SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy, struct sched_param __user *, param) 5666 { 5667 if (policy < 0) 5668 return -EINVAL; 5669 5670 return do_sched_setscheduler(pid, policy, param); 5671 } 5672 5673 /** 5674 * sys_sched_setparam - set/change the RT priority of a thread 5675 * @pid: the pid in question. 5676 * @param: structure containing the new RT priority. 5677 * 5678 * Return: 0 on success. An error code otherwise. 5679 */ 5680 SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param) 5681 { 5682 return do_sched_setscheduler(pid, SETPARAM_POLICY, param); 5683 } 5684 5685 /** 5686 * sys_sched_setattr - same as above, but with extended sched_attr 5687 * @pid: the pid in question. 5688 * @uattr: structure containing the extended parameters. 5689 * @flags: for future extension. 5690 */ 5691 SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr, 5692 unsigned int, flags) 5693 { 5694 struct sched_attr attr; 5695 struct task_struct *p; 5696 int retval; 5697 5698 if (!uattr || pid < 0 || flags) 5699 return -EINVAL; 5700 5701 retval = sched_copy_attr(uattr, &attr); 5702 if (retval) 5703 return retval; 5704 5705 if ((int)attr.sched_policy < 0) 5706 return -EINVAL; 5707 if (attr.sched_flags & SCHED_FLAG_KEEP_POLICY) 5708 attr.sched_policy = SETPARAM_POLICY; 5709 5710 rcu_read_lock(); 5711 retval = -ESRCH; 5712 p = find_process_by_pid(pid); 5713 if (likely(p)) 5714 get_task_struct(p); 5715 rcu_read_unlock(); 5716 5717 if (likely(p)) { 5718 retval = sched_setattr(p, &attr); 5719 put_task_struct(p); 5720 } 5721 5722 return retval; 5723 } 5724 5725 /** 5726 * sys_sched_getscheduler - get the policy (scheduling class) of a thread 5727 * @pid: the pid in question. 5728 * 5729 * Return: On success, the policy of the thread. Otherwise, a negative error 5730 * code. 5731 */ 5732 SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid) 5733 { 5734 struct task_struct *p; 5735 int retval; 5736 5737 if (pid < 0) 5738 return -EINVAL; 5739 5740 retval = -ESRCH; 5741 rcu_read_lock(); 5742 p = find_process_by_pid(pid); 5743 if (p) { 5744 retval = security_task_getscheduler(p); 5745 if (!retval) 5746 retval = p->policy 5747 | (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0); 5748 } 5749 rcu_read_unlock(); 5750 return retval; 5751 } 5752 5753 /** 5754 * sys_sched_getparam - get the RT priority of a thread 5755 * @pid: the pid in question. 5756 * @param: structure containing the RT priority. 5757 * 5758 * Return: On success, 0 and the RT priority is in @param. Otherwise, an error 5759 * code. 5760 */ 5761 SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param) 5762 { 5763 struct sched_param lp = { .sched_priority = 0 }; 5764 struct task_struct *p; 5765 int retval; 5766 5767 if (!param || pid < 0) 5768 return -EINVAL; 5769 5770 rcu_read_lock(); 5771 p = find_process_by_pid(pid); 5772 retval = -ESRCH; 5773 if (!p) 5774 goto out_unlock; 5775 5776 retval = security_task_getscheduler(p); 5777 if (retval) 5778 goto out_unlock; 5779 5780 if (task_has_rt_policy(p)) 5781 lp.sched_priority = p->rt_priority; 5782 rcu_read_unlock(); 5783 5784 /* 5785 * This one might sleep, we cannot do it with a spinlock held ... 5786 */ 5787 retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0; 5788 5789 return retval; 5790 5791 out_unlock: 5792 rcu_read_unlock(); 5793 return retval; 5794 } 5795 5796 /* 5797 * Copy the kernel size attribute structure (which might be larger 5798 * than what user-space knows about) to user-space. 5799 * 5800 * Note that all cases are valid: user-space buffer can be larger or 5801 * smaller than the kernel-space buffer. The usual case is that both 5802 * have the same size. 5803 */ 5804 static int 5805 sched_attr_copy_to_user(struct sched_attr __user *uattr, 5806 struct sched_attr *kattr, 5807 unsigned int usize) 5808 { 5809 unsigned int ksize = sizeof(*kattr); 5810 5811 if (!access_ok(uattr, usize)) 5812 return -EFAULT; 5813 5814 /* 5815 * sched_getattr() ABI forwards and backwards compatibility: 5816 * 5817 * If usize == ksize then we just copy everything to user-space and all is good. 5818 * 5819 * If usize < ksize then we only copy as much as user-space has space for, 5820 * this keeps ABI compatibility as well. We skip the rest. 5821 * 5822 * If usize > ksize then user-space is using a newer version of the ABI, 5823 * which part the kernel doesn't know about. Just ignore it - tooling can 5824 * detect the kernel's knowledge of attributes from the attr->size value 5825 * which is set to ksize in this case. 5826 */ 5827 kattr->size = min(usize, ksize); 5828 5829 if (copy_to_user(uattr, kattr, kattr->size)) 5830 return -EFAULT; 5831 5832 return 0; 5833 } 5834 5835 /** 5836 * sys_sched_getattr - similar to sched_getparam, but with sched_attr 5837 * @pid: the pid in question. 5838 * @uattr: structure containing the extended parameters. 5839 * @usize: sizeof(attr) for fwd/bwd comp. 5840 * @flags: for future extension. 5841 */ 5842 SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr, 5843 unsigned int, usize, unsigned int, flags) 5844 { 5845 struct sched_attr kattr = { }; 5846 struct task_struct *p; 5847 int retval; 5848 5849 if (!uattr || pid < 0 || usize > PAGE_SIZE || 5850 usize < SCHED_ATTR_SIZE_VER0 || flags) 5851 return -EINVAL; 5852 5853 rcu_read_lock(); 5854 p = find_process_by_pid(pid); 5855 retval = -ESRCH; 5856 if (!p) 5857 goto out_unlock; 5858 5859 retval = security_task_getscheduler(p); 5860 if (retval) 5861 goto out_unlock; 5862 5863 kattr.sched_policy = p->policy; 5864 if (p->sched_reset_on_fork) 5865 kattr.sched_flags |= SCHED_FLAG_RESET_ON_FORK; 5866 if (task_has_dl_policy(p)) 5867 __getparam_dl(p, &kattr); 5868 else if (task_has_rt_policy(p)) 5869 kattr.sched_priority = p->rt_priority; 5870 else 5871 kattr.sched_nice = task_nice(p); 5872 5873 #ifdef CONFIG_UCLAMP_TASK 5874 /* 5875 * This could race with another potential updater, but this is fine 5876 * because it'll correctly read the old or the new value. We don't need 5877 * to guarantee who wins the race as long as it doesn't return garbage. 5878 */ 5879 kattr.sched_util_min = p->uclamp_req[UCLAMP_MIN].value; 5880 kattr.sched_util_max = p->uclamp_req[UCLAMP_MAX].value; 5881 #endif 5882 5883 rcu_read_unlock(); 5884 5885 return sched_attr_copy_to_user(uattr, &kattr, usize); 5886 5887 out_unlock: 5888 rcu_read_unlock(); 5889 return retval; 5890 } 5891 5892 long sched_setaffinity(pid_t pid, const struct cpumask *in_mask) 5893 { 5894 cpumask_var_t cpus_allowed, new_mask; 5895 struct task_struct *p; 5896 int retval; 5897 5898 rcu_read_lock(); 5899 5900 p = find_process_by_pid(pid); 5901 if (!p) { 5902 rcu_read_unlock(); 5903 return -ESRCH; 5904 } 5905 5906 /* Prevent p going away */ 5907 get_task_struct(p); 5908 rcu_read_unlock(); 5909 5910 if (p->flags & PF_NO_SETAFFINITY) { 5911 retval = -EINVAL; 5912 goto out_put_task; 5913 } 5914 if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) { 5915 retval = -ENOMEM; 5916 goto out_put_task; 5917 } 5918 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) { 5919 retval = -ENOMEM; 5920 goto out_free_cpus_allowed; 5921 } 5922 retval = -EPERM; 5923 if (!check_same_owner(p)) { 5924 rcu_read_lock(); 5925 if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) { 5926 rcu_read_unlock(); 5927 goto out_free_new_mask; 5928 } 5929 rcu_read_unlock(); 5930 } 5931 5932 retval = security_task_setscheduler(p); 5933 if (retval) 5934 goto out_free_new_mask; 5935 5936 5937 cpuset_cpus_allowed(p, cpus_allowed); 5938 cpumask_and(new_mask, in_mask, cpus_allowed); 5939 5940 /* 5941 * Since bandwidth control happens on root_domain basis, 5942 * if admission test is enabled, we only admit -deadline 5943 * tasks allowed to run on all the CPUs in the task's 5944 * root_domain. 5945 */ 5946 #ifdef CONFIG_SMP 5947 if (task_has_dl_policy(p) && dl_bandwidth_enabled()) { 5948 rcu_read_lock(); 5949 if (!cpumask_subset(task_rq(p)->rd->span, new_mask)) { 5950 retval = -EBUSY; 5951 rcu_read_unlock(); 5952 goto out_free_new_mask; 5953 } 5954 rcu_read_unlock(); 5955 } 5956 #endif 5957 again: 5958 retval = __set_cpus_allowed_ptr(p, new_mask, true); 5959 5960 if (!retval) { 5961 cpuset_cpus_allowed(p, cpus_allowed); 5962 if (!cpumask_subset(new_mask, cpus_allowed)) { 5963 /* 5964 * We must have raced with a concurrent cpuset 5965 * update. Just reset the cpus_allowed to the 5966 * cpuset's cpus_allowed 5967 */ 5968 cpumask_copy(new_mask, cpus_allowed); 5969 goto again; 5970 } 5971 } 5972 out_free_new_mask: 5973 free_cpumask_var(new_mask); 5974 out_free_cpus_allowed: 5975 free_cpumask_var(cpus_allowed); 5976 out_put_task: 5977 put_task_struct(p); 5978 return retval; 5979 } 5980 5981 static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len, 5982 struct cpumask *new_mask) 5983 { 5984 if (len < cpumask_size()) 5985 cpumask_clear(new_mask); 5986 else if (len > cpumask_size()) 5987 len = cpumask_size(); 5988 5989 return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0; 5990 } 5991 5992 /** 5993 * sys_sched_setaffinity - set the CPU affinity of a process 5994 * @pid: pid of the process 5995 * @len: length in bytes of the bitmask pointed to by user_mask_ptr 5996 * @user_mask_ptr: user-space pointer to the new CPU mask 5997 * 5998 * Return: 0 on success. An error code otherwise. 5999 */ 6000 SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len, 6001 unsigned long __user *, user_mask_ptr) 6002 { 6003 cpumask_var_t new_mask; 6004 int retval; 6005 6006 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) 6007 return -ENOMEM; 6008 6009 retval = get_user_cpu_mask(user_mask_ptr, len, new_mask); 6010 if (retval == 0) 6011 retval = sched_setaffinity(pid, new_mask); 6012 free_cpumask_var(new_mask); 6013 return retval; 6014 } 6015 6016 long sched_getaffinity(pid_t pid, struct cpumask *mask) 6017 { 6018 struct task_struct *p; 6019 unsigned long flags; 6020 int retval; 6021 6022 rcu_read_lock(); 6023 6024 retval = -ESRCH; 6025 p = find_process_by_pid(pid); 6026 if (!p) 6027 goto out_unlock; 6028 6029 retval = security_task_getscheduler(p); 6030 if (retval) 6031 goto out_unlock; 6032 6033 raw_spin_lock_irqsave(&p->pi_lock, flags); 6034 cpumask_and(mask, &p->cpus_mask, cpu_active_mask); 6035 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 6036 6037 out_unlock: 6038 rcu_read_unlock(); 6039 6040 return retval; 6041 } 6042 6043 /** 6044 * sys_sched_getaffinity - get the CPU affinity of a process 6045 * @pid: pid of the process 6046 * @len: length in bytes of the bitmask pointed to by user_mask_ptr 6047 * @user_mask_ptr: user-space pointer to hold the current CPU mask 6048 * 6049 * Return: size of CPU mask copied to user_mask_ptr on success. An 6050 * error code otherwise. 6051 */ 6052 SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len, 6053 unsigned long __user *, user_mask_ptr) 6054 { 6055 int ret; 6056 cpumask_var_t mask; 6057 6058 if ((len * BITS_PER_BYTE) < nr_cpu_ids) 6059 return -EINVAL; 6060 if (len & (sizeof(unsigned long)-1)) 6061 return -EINVAL; 6062 6063 if (!alloc_cpumask_var(&mask, GFP_KERNEL)) 6064 return -ENOMEM; 6065 6066 ret = sched_getaffinity(pid, mask); 6067 if (ret == 0) { 6068 unsigned int retlen = min(len, cpumask_size()); 6069 6070 if (copy_to_user(user_mask_ptr, mask, retlen)) 6071 ret = -EFAULT; 6072 else 6073 ret = retlen; 6074 } 6075 free_cpumask_var(mask); 6076 6077 return ret; 6078 } 6079 6080 /** 6081 * sys_sched_yield - yield the current processor to other threads. 6082 * 6083 * This function yields the current CPU to other tasks. If there are no 6084 * other threads running on this CPU then this function will return. 6085 * 6086 * Return: 0. 6087 */ 6088 static void do_sched_yield(void) 6089 { 6090 struct rq_flags rf; 6091 struct rq *rq; 6092 6093 rq = this_rq_lock_irq(&rf); 6094 6095 schedstat_inc(rq->yld_count); 6096 current->sched_class->yield_task(rq); 6097 6098 /* 6099 * Since we are going to call schedule() anyway, there's 6100 * no need to preempt or enable interrupts: 6101 */ 6102 preempt_disable(); 6103 rq_unlock(rq, &rf); 6104 sched_preempt_enable_no_resched(); 6105 6106 schedule(); 6107 } 6108 6109 SYSCALL_DEFINE0(sched_yield) 6110 { 6111 do_sched_yield(); 6112 return 0; 6113 } 6114 6115 #ifndef CONFIG_PREEMPTION 6116 int __sched _cond_resched(void) 6117 { 6118 if (should_resched(0)) { 6119 preempt_schedule_common(); 6120 return 1; 6121 } 6122 rcu_all_qs(); 6123 return 0; 6124 } 6125 EXPORT_SYMBOL(_cond_resched); 6126 #endif 6127 6128 /* 6129 * __cond_resched_lock() - if a reschedule is pending, drop the given lock, 6130 * call schedule, and on return reacquire the lock. 6131 * 6132 * This works OK both with and without CONFIG_PREEMPTION. We do strange low-level 6133 * operations here to prevent schedule() from being called twice (once via 6134 * spin_unlock(), once by hand). 6135 */ 6136 int __cond_resched_lock(spinlock_t *lock) 6137 { 6138 int resched = should_resched(PREEMPT_LOCK_OFFSET); 6139 int ret = 0; 6140 6141 lockdep_assert_held(lock); 6142 6143 if (spin_needbreak(lock) || resched) { 6144 spin_unlock(lock); 6145 if (resched) 6146 preempt_schedule_common(); 6147 else 6148 cpu_relax(); 6149 ret = 1; 6150 spin_lock(lock); 6151 } 6152 return ret; 6153 } 6154 EXPORT_SYMBOL(__cond_resched_lock); 6155 6156 /** 6157 * yield - yield the current processor to other threads. 6158 * 6159 * Do not ever use this function, there's a 99% chance you're doing it wrong. 6160 * 6161 * The scheduler is at all times free to pick the calling task as the most 6162 * eligible task to run, if removing the yield() call from your code breaks 6163 * it, its already broken. 6164 * 6165 * Typical broken usage is: 6166 * 6167 * while (!event) 6168 * yield(); 6169 * 6170 * where one assumes that yield() will let 'the other' process run that will 6171 * make event true. If the current task is a SCHED_FIFO task that will never 6172 * happen. Never use yield() as a progress guarantee!! 6173 * 6174 * If you want to use yield() to wait for something, use wait_event(). 6175 * If you want to use yield() to be 'nice' for others, use cond_resched(). 6176 * If you still want to use yield(), do not! 6177 */ 6178 void __sched yield(void) 6179 { 6180 set_current_state(TASK_RUNNING); 6181 do_sched_yield(); 6182 } 6183 EXPORT_SYMBOL(yield); 6184 6185 /** 6186 * yield_to - yield the current processor to another thread in 6187 * your thread group, or accelerate that thread toward the 6188 * processor it's on. 6189 * @p: target task 6190 * @preempt: whether task preemption is allowed or not 6191 * 6192 * It's the caller's job to ensure that the target task struct 6193 * can't go away on us before we can do any checks. 6194 * 6195 * Return: 6196 * true (>0) if we indeed boosted the target task. 6197 * false (0) if we failed to boost the target. 6198 * -ESRCH if there's no task to yield to. 6199 */ 6200 int __sched yield_to(struct task_struct *p, bool preempt) 6201 { 6202 struct task_struct *curr = current; 6203 struct rq *rq, *p_rq; 6204 unsigned long flags; 6205 int yielded = 0; 6206 6207 local_irq_save(flags); 6208 rq = this_rq(); 6209 6210 again: 6211 p_rq = task_rq(p); 6212 /* 6213 * If we're the only runnable task on the rq and target rq also 6214 * has only one task, there's absolutely no point in yielding. 6215 */ 6216 if (rq->nr_running == 1 && p_rq->nr_running == 1) { 6217 yielded = -ESRCH; 6218 goto out_irq; 6219 } 6220 6221 double_rq_lock(rq, p_rq); 6222 if (task_rq(p) != p_rq) { 6223 double_rq_unlock(rq, p_rq); 6224 goto again; 6225 } 6226 6227 if (!curr->sched_class->yield_to_task) 6228 goto out_unlock; 6229 6230 if (curr->sched_class != p->sched_class) 6231 goto out_unlock; 6232 6233 if (task_running(p_rq, p) || p->state) 6234 goto out_unlock; 6235 6236 yielded = curr->sched_class->yield_to_task(rq, p); 6237 if (yielded) { 6238 schedstat_inc(rq->yld_count); 6239 /* 6240 * Make p's CPU reschedule; pick_next_entity takes care of 6241 * fairness. 6242 */ 6243 if (preempt && rq != p_rq) 6244 resched_curr(p_rq); 6245 } 6246 6247 out_unlock: 6248 double_rq_unlock(rq, p_rq); 6249 out_irq: 6250 local_irq_restore(flags); 6251 6252 if (yielded > 0) 6253 schedule(); 6254 6255 return yielded; 6256 } 6257 EXPORT_SYMBOL_GPL(yield_to); 6258 6259 int io_schedule_prepare(void) 6260 { 6261 int old_iowait = current->in_iowait; 6262 6263 current->in_iowait = 1; 6264 blk_schedule_flush_plug(current); 6265 6266 return old_iowait; 6267 } 6268 6269 void io_schedule_finish(int token) 6270 { 6271 current->in_iowait = token; 6272 } 6273 6274 /* 6275 * This task is about to go to sleep on IO. Increment rq->nr_iowait so 6276 * that process accounting knows that this is a task in IO wait state. 6277 */ 6278 long __sched io_schedule_timeout(long timeout) 6279 { 6280 int token; 6281 long ret; 6282 6283 token = io_schedule_prepare(); 6284 ret = schedule_timeout(timeout); 6285 io_schedule_finish(token); 6286 6287 return ret; 6288 } 6289 EXPORT_SYMBOL(io_schedule_timeout); 6290 6291 void __sched io_schedule(void) 6292 { 6293 int token; 6294 6295 token = io_schedule_prepare(); 6296 schedule(); 6297 io_schedule_finish(token); 6298 } 6299 EXPORT_SYMBOL(io_schedule); 6300 6301 /** 6302 * sys_sched_get_priority_max - return maximum RT priority. 6303 * @policy: scheduling class. 6304 * 6305 * Return: On success, this syscall returns the maximum 6306 * rt_priority that can be used by a given scheduling class. 6307 * On failure, a negative error code is returned. 6308 */ 6309 SYSCALL_DEFINE1(sched_get_priority_max, int, policy) 6310 { 6311 int ret = -EINVAL; 6312 6313 switch (policy) { 6314 case SCHED_FIFO: 6315 case SCHED_RR: 6316 ret = MAX_USER_RT_PRIO-1; 6317 break; 6318 case SCHED_DEADLINE: 6319 case SCHED_NORMAL: 6320 case SCHED_BATCH: 6321 case SCHED_IDLE: 6322 ret = 0; 6323 break; 6324 } 6325 return ret; 6326 } 6327 6328 /** 6329 * sys_sched_get_priority_min - return minimum RT priority. 6330 * @policy: scheduling class. 6331 * 6332 * Return: On success, this syscall returns the minimum 6333 * rt_priority that can be used by a given scheduling class. 6334 * On failure, a negative error code is returned. 6335 */ 6336 SYSCALL_DEFINE1(sched_get_priority_min, int, policy) 6337 { 6338 int ret = -EINVAL; 6339 6340 switch (policy) { 6341 case SCHED_FIFO: 6342 case SCHED_RR: 6343 ret = 1; 6344 break; 6345 case SCHED_DEADLINE: 6346 case SCHED_NORMAL: 6347 case SCHED_BATCH: 6348 case SCHED_IDLE: 6349 ret = 0; 6350 } 6351 return ret; 6352 } 6353 6354 static int sched_rr_get_interval(pid_t pid, struct timespec64 *t) 6355 { 6356 struct task_struct *p; 6357 unsigned int time_slice; 6358 struct rq_flags rf; 6359 struct rq *rq; 6360 int retval; 6361 6362 if (pid < 0) 6363 return -EINVAL; 6364 6365 retval = -ESRCH; 6366 rcu_read_lock(); 6367 p = find_process_by_pid(pid); 6368 if (!p) 6369 goto out_unlock; 6370 6371 retval = security_task_getscheduler(p); 6372 if (retval) 6373 goto out_unlock; 6374 6375 rq = task_rq_lock(p, &rf); 6376 time_slice = 0; 6377 if (p->sched_class->get_rr_interval) 6378 time_slice = p->sched_class->get_rr_interval(rq, p); 6379 task_rq_unlock(rq, p, &rf); 6380 6381 rcu_read_unlock(); 6382 jiffies_to_timespec64(time_slice, t); 6383 return 0; 6384 6385 out_unlock: 6386 rcu_read_unlock(); 6387 return retval; 6388 } 6389 6390 /** 6391 * sys_sched_rr_get_interval - return the default timeslice of a process. 6392 * @pid: pid of the process. 6393 * @interval: userspace pointer to the timeslice value. 6394 * 6395 * this syscall writes the default timeslice value of a given process 6396 * into the user-space timespec buffer. A value of '0' means infinity. 6397 * 6398 * Return: On success, 0 and the timeslice is in @interval. Otherwise, 6399 * an error code. 6400 */ 6401 SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid, 6402 struct __kernel_timespec __user *, interval) 6403 { 6404 struct timespec64 t; 6405 int retval = sched_rr_get_interval(pid, &t); 6406 6407 if (retval == 0) 6408 retval = put_timespec64(&t, interval); 6409 6410 return retval; 6411 } 6412 6413 #ifdef CONFIG_COMPAT_32BIT_TIME 6414 SYSCALL_DEFINE2(sched_rr_get_interval_time32, pid_t, pid, 6415 struct old_timespec32 __user *, interval) 6416 { 6417 struct timespec64 t; 6418 int retval = sched_rr_get_interval(pid, &t); 6419 6420 if (retval == 0) 6421 retval = put_old_timespec32(&t, interval); 6422 return retval; 6423 } 6424 #endif 6425 6426 void sched_show_task(struct task_struct *p) 6427 { 6428 unsigned long free = 0; 6429 int ppid; 6430 6431 if (!try_get_task_stack(p)) 6432 return; 6433 6434 pr_info("task:%-15.15s state:%c", p->comm, task_state_to_char(p)); 6435 6436 if (p->state == TASK_RUNNING) 6437 pr_cont(" running task "); 6438 #ifdef CONFIG_DEBUG_STACK_USAGE 6439 free = stack_not_used(p); 6440 #endif 6441 ppid = 0; 6442 rcu_read_lock(); 6443 if (pid_alive(p)) 6444 ppid = task_pid_nr(rcu_dereference(p->real_parent)); 6445 rcu_read_unlock(); 6446 pr_cont(" stack:%5lu pid:%5d ppid:%6d flags:0x%08lx\n", 6447 free, task_pid_nr(p), ppid, 6448 (unsigned long)task_thread_info(p)->flags); 6449 6450 print_worker_info(KERN_INFO, p); 6451 show_stack(p, NULL, KERN_INFO); 6452 put_task_stack(p); 6453 } 6454 EXPORT_SYMBOL_GPL(sched_show_task); 6455 6456 static inline bool 6457 state_filter_match(unsigned long state_filter, struct task_struct *p) 6458 { 6459 /* no filter, everything matches */ 6460 if (!state_filter) 6461 return true; 6462 6463 /* filter, but doesn't match */ 6464 if (!(p->state & state_filter)) 6465 return false; 6466 6467 /* 6468 * When looking for TASK_UNINTERRUPTIBLE skip TASK_IDLE (allows 6469 * TASK_KILLABLE). 6470 */ 6471 if (state_filter == TASK_UNINTERRUPTIBLE && p->state == TASK_IDLE) 6472 return false; 6473 6474 return true; 6475 } 6476 6477 6478 void show_state_filter(unsigned long state_filter) 6479 { 6480 struct task_struct *g, *p; 6481 6482 rcu_read_lock(); 6483 for_each_process_thread(g, p) { 6484 /* 6485 * reset the NMI-timeout, listing all files on a slow 6486 * console might take a lot of time: 6487 * Also, reset softlockup watchdogs on all CPUs, because 6488 * another CPU might be blocked waiting for us to process 6489 * an IPI. 6490 */ 6491 touch_nmi_watchdog(); 6492 touch_all_softlockup_watchdogs(); 6493 if (state_filter_match(state_filter, p)) 6494 sched_show_task(p); 6495 } 6496 6497 #ifdef CONFIG_SCHED_DEBUG 6498 if (!state_filter) 6499 sysrq_sched_debug_show(); 6500 #endif 6501 rcu_read_unlock(); 6502 /* 6503 * Only show locks if all tasks are dumped: 6504 */ 6505 if (!state_filter) 6506 debug_show_all_locks(); 6507 } 6508 6509 /** 6510 * init_idle - set up an idle thread for a given CPU 6511 * @idle: task in question 6512 * @cpu: CPU the idle task belongs to 6513 * 6514 * NOTE: this function does not set the idle thread's NEED_RESCHED 6515 * flag, to make booting more robust. 6516 */ 6517 void init_idle(struct task_struct *idle, int cpu) 6518 { 6519 struct rq *rq = cpu_rq(cpu); 6520 unsigned long flags; 6521 6522 __sched_fork(0, idle); 6523 6524 raw_spin_lock_irqsave(&idle->pi_lock, flags); 6525 raw_spin_lock(&rq->lock); 6526 6527 idle->state = TASK_RUNNING; 6528 idle->se.exec_start = sched_clock(); 6529 idle->flags |= PF_IDLE; 6530 6531 scs_task_reset(idle); 6532 kasan_unpoison_task_stack(idle); 6533 6534 #ifdef CONFIG_SMP 6535 /* 6536 * Its possible that init_idle() gets called multiple times on a task, 6537 * in that case do_set_cpus_allowed() will not do the right thing. 6538 * 6539 * And since this is boot we can forgo the serialization. 6540 */ 6541 set_cpus_allowed_common(idle, cpumask_of(cpu)); 6542 #endif 6543 /* 6544 * We're having a chicken and egg problem, even though we are 6545 * holding rq->lock, the CPU isn't yet set to this CPU so the 6546 * lockdep check in task_group() will fail. 6547 * 6548 * Similar case to sched_fork(). / Alternatively we could 6549 * use task_rq_lock() here and obtain the other rq->lock. 6550 * 6551 * Silence PROVE_RCU 6552 */ 6553 rcu_read_lock(); 6554 __set_task_cpu(idle, cpu); 6555 rcu_read_unlock(); 6556 6557 rq->idle = idle; 6558 rcu_assign_pointer(rq->curr, idle); 6559 idle->on_rq = TASK_ON_RQ_QUEUED; 6560 #ifdef CONFIG_SMP 6561 idle->on_cpu = 1; 6562 #endif 6563 raw_spin_unlock(&rq->lock); 6564 raw_spin_unlock_irqrestore(&idle->pi_lock, flags); 6565 6566 /* Set the preempt count _outside_ the spinlocks! */ 6567 init_idle_preempt_count(idle, cpu); 6568 6569 /* 6570 * The idle tasks have their own, simple scheduling class: 6571 */ 6572 idle->sched_class = &idle_sched_class; 6573 ftrace_graph_init_idle_task(idle, cpu); 6574 vtime_init_idle(idle, cpu); 6575 #ifdef CONFIG_SMP 6576 sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu); 6577 #endif 6578 } 6579 6580 #ifdef CONFIG_SMP 6581 6582 int cpuset_cpumask_can_shrink(const struct cpumask *cur, 6583 const struct cpumask *trial) 6584 { 6585 int ret = 1; 6586 6587 if (!cpumask_weight(cur)) 6588 return ret; 6589 6590 ret = dl_cpuset_cpumask_can_shrink(cur, trial); 6591 6592 return ret; 6593 } 6594 6595 int task_can_attach(struct task_struct *p, 6596 const struct cpumask *cs_cpus_allowed) 6597 { 6598 int ret = 0; 6599 6600 /* 6601 * Kthreads which disallow setaffinity shouldn't be moved 6602 * to a new cpuset; we don't want to change their CPU 6603 * affinity and isolating such threads by their set of 6604 * allowed nodes is unnecessary. Thus, cpusets are not 6605 * applicable for such threads. This prevents checking for 6606 * success of set_cpus_allowed_ptr() on all attached tasks 6607 * before cpus_mask may be changed. 6608 */ 6609 if (p->flags & PF_NO_SETAFFINITY) { 6610 ret = -EINVAL; 6611 goto out; 6612 } 6613 6614 if (dl_task(p) && !cpumask_intersects(task_rq(p)->rd->span, 6615 cs_cpus_allowed)) 6616 ret = dl_task_can_attach(p, cs_cpus_allowed); 6617 6618 out: 6619 return ret; 6620 } 6621 6622 bool sched_smp_initialized __read_mostly; 6623 6624 #ifdef CONFIG_NUMA_BALANCING 6625 /* Migrate current task p to target_cpu */ 6626 int migrate_task_to(struct task_struct *p, int target_cpu) 6627 { 6628 struct migration_arg arg = { p, target_cpu }; 6629 int curr_cpu = task_cpu(p); 6630 6631 if (curr_cpu == target_cpu) 6632 return 0; 6633 6634 if (!cpumask_test_cpu(target_cpu, p->cpus_ptr)) 6635 return -EINVAL; 6636 6637 /* TODO: This is not properly updating schedstats */ 6638 6639 trace_sched_move_numa(p, curr_cpu, target_cpu); 6640 return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg); 6641 } 6642 6643 /* 6644 * Requeue a task on a given node and accurately track the number of NUMA 6645 * tasks on the runqueues 6646 */ 6647 void sched_setnuma(struct task_struct *p, int nid) 6648 { 6649 bool queued, running; 6650 struct rq_flags rf; 6651 struct rq *rq; 6652 6653 rq = task_rq_lock(p, &rf); 6654 queued = task_on_rq_queued(p); 6655 running = task_current(rq, p); 6656 6657 if (queued) 6658 dequeue_task(rq, p, DEQUEUE_SAVE); 6659 if (running) 6660 put_prev_task(rq, p); 6661 6662 p->numa_preferred_nid = nid; 6663 6664 if (queued) 6665 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); 6666 if (running) 6667 set_next_task(rq, p); 6668 task_rq_unlock(rq, p, &rf); 6669 } 6670 #endif /* CONFIG_NUMA_BALANCING */ 6671 6672 #ifdef CONFIG_HOTPLUG_CPU 6673 /* 6674 * Ensure that the idle task is using init_mm right before its CPU goes 6675 * offline. 6676 */ 6677 void idle_task_exit(void) 6678 { 6679 struct mm_struct *mm = current->active_mm; 6680 6681 BUG_ON(cpu_online(smp_processor_id())); 6682 BUG_ON(current != this_rq()->idle); 6683 6684 if (mm != &init_mm) { 6685 switch_mm(mm, &init_mm, current); 6686 finish_arch_post_lock_switch(); 6687 } 6688 6689 /* finish_cpu(), as ran on the BP, will clean up the active_mm state */ 6690 } 6691 6692 /* 6693 * Since this CPU is going 'away' for a while, fold any nr_active delta 6694 * we might have. Assumes we're called after migrate_tasks() so that the 6695 * nr_active count is stable. We need to take the teardown thread which 6696 * is calling this into account, so we hand in adjust = 1 to the load 6697 * calculation. 6698 * 6699 * Also see the comment "Global load-average calculations". 6700 */ 6701 static void calc_load_migrate(struct rq *rq) 6702 { 6703 long delta = calc_load_fold_active(rq, 1); 6704 if (delta) 6705 atomic_long_add(delta, &calc_load_tasks); 6706 } 6707 6708 static struct task_struct *__pick_migrate_task(struct rq *rq) 6709 { 6710 const struct sched_class *class; 6711 struct task_struct *next; 6712 6713 for_each_class(class) { 6714 next = class->pick_next_task(rq); 6715 if (next) { 6716 next->sched_class->put_prev_task(rq, next); 6717 return next; 6718 } 6719 } 6720 6721 /* The idle class should always have a runnable task */ 6722 BUG(); 6723 } 6724 6725 /* 6726 * Migrate all tasks from the rq, sleeping tasks will be migrated by 6727 * try_to_wake_up()->select_task_rq(). 6728 * 6729 * Called with rq->lock held even though we'er in stop_machine() and 6730 * there's no concurrency possible, we hold the required locks anyway 6731 * because of lock validation efforts. 6732 */ 6733 static void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf) 6734 { 6735 struct rq *rq = dead_rq; 6736 struct task_struct *next, *stop = rq->stop; 6737 struct rq_flags orf = *rf; 6738 int dest_cpu; 6739 6740 /* 6741 * Fudge the rq selection such that the below task selection loop 6742 * doesn't get stuck on the currently eligible stop task. 6743 * 6744 * We're currently inside stop_machine() and the rq is either stuck 6745 * in the stop_machine_cpu_stop() loop, or we're executing this code, 6746 * either way we should never end up calling schedule() until we're 6747 * done here. 6748 */ 6749 rq->stop = NULL; 6750 6751 /* 6752 * put_prev_task() and pick_next_task() sched 6753 * class method both need to have an up-to-date 6754 * value of rq->clock[_task] 6755 */ 6756 update_rq_clock(rq); 6757 6758 for (;;) { 6759 /* 6760 * There's this thread running, bail when that's the only 6761 * remaining thread: 6762 */ 6763 if (rq->nr_running == 1) 6764 break; 6765 6766 next = __pick_migrate_task(rq); 6767 6768 /* 6769 * Rules for changing task_struct::cpus_mask are holding 6770 * both pi_lock and rq->lock, such that holding either 6771 * stabilizes the mask. 6772 * 6773 * Drop rq->lock is not quite as disastrous as it usually is 6774 * because !cpu_active at this point, which means load-balance 6775 * will not interfere. Also, stop-machine. 6776 */ 6777 rq_unlock(rq, rf); 6778 raw_spin_lock(&next->pi_lock); 6779 rq_relock(rq, rf); 6780 6781 /* 6782 * Since we're inside stop-machine, _nothing_ should have 6783 * changed the task, WARN if weird stuff happened, because in 6784 * that case the above rq->lock drop is a fail too. 6785 */ 6786 if (WARN_ON(task_rq(next) != rq || !task_on_rq_queued(next))) { 6787 raw_spin_unlock(&next->pi_lock); 6788 continue; 6789 } 6790 6791 /* Find suitable destination for @next, with force if needed. */ 6792 dest_cpu = select_fallback_rq(dead_rq->cpu, next); 6793 rq = __migrate_task(rq, rf, next, dest_cpu); 6794 if (rq != dead_rq) { 6795 rq_unlock(rq, rf); 6796 rq = dead_rq; 6797 *rf = orf; 6798 rq_relock(rq, rf); 6799 } 6800 raw_spin_unlock(&next->pi_lock); 6801 } 6802 6803 rq->stop = stop; 6804 } 6805 #endif /* CONFIG_HOTPLUG_CPU */ 6806 6807 void set_rq_online(struct rq *rq) 6808 { 6809 if (!rq->online) { 6810 const struct sched_class *class; 6811 6812 cpumask_set_cpu(rq->cpu, rq->rd->online); 6813 rq->online = 1; 6814 6815 for_each_class(class) { 6816 if (class->rq_online) 6817 class->rq_online(rq); 6818 } 6819 } 6820 } 6821 6822 void set_rq_offline(struct rq *rq) 6823 { 6824 if (rq->online) { 6825 const struct sched_class *class; 6826 6827 for_each_class(class) { 6828 if (class->rq_offline) 6829 class->rq_offline(rq); 6830 } 6831 6832 cpumask_clear_cpu(rq->cpu, rq->rd->online); 6833 rq->online = 0; 6834 } 6835 } 6836 6837 /* 6838 * used to mark begin/end of suspend/resume: 6839 */ 6840 static int num_cpus_frozen; 6841 6842 /* 6843 * Update cpusets according to cpu_active mask. If cpusets are 6844 * disabled, cpuset_update_active_cpus() becomes a simple wrapper 6845 * around partition_sched_domains(). 6846 * 6847 * If we come here as part of a suspend/resume, don't touch cpusets because we 6848 * want to restore it back to its original state upon resume anyway. 6849 */ 6850 static void cpuset_cpu_active(void) 6851 { 6852 if (cpuhp_tasks_frozen) { 6853 /* 6854 * num_cpus_frozen tracks how many CPUs are involved in suspend 6855 * resume sequence. As long as this is not the last online 6856 * operation in the resume sequence, just build a single sched 6857 * domain, ignoring cpusets. 6858 */ 6859 partition_sched_domains(1, NULL, NULL); 6860 if (--num_cpus_frozen) 6861 return; 6862 /* 6863 * This is the last CPU online operation. So fall through and 6864 * restore the original sched domains by considering the 6865 * cpuset configurations. 6866 */ 6867 cpuset_force_rebuild(); 6868 } 6869 cpuset_update_active_cpus(); 6870 } 6871 6872 static int cpuset_cpu_inactive(unsigned int cpu) 6873 { 6874 if (!cpuhp_tasks_frozen) { 6875 if (dl_cpu_busy(cpu)) 6876 return -EBUSY; 6877 cpuset_update_active_cpus(); 6878 } else { 6879 num_cpus_frozen++; 6880 partition_sched_domains(1, NULL, NULL); 6881 } 6882 return 0; 6883 } 6884 6885 int sched_cpu_activate(unsigned int cpu) 6886 { 6887 struct rq *rq = cpu_rq(cpu); 6888 struct rq_flags rf; 6889 6890 #ifdef CONFIG_SCHED_SMT 6891 /* 6892 * When going up, increment the number of cores with SMT present. 6893 */ 6894 if (cpumask_weight(cpu_smt_mask(cpu)) == 2) 6895 static_branch_inc_cpuslocked(&sched_smt_present); 6896 #endif 6897 set_cpu_active(cpu, true); 6898 6899 if (sched_smp_initialized) { 6900 sched_domains_numa_masks_set(cpu); 6901 cpuset_cpu_active(); 6902 } 6903 6904 /* 6905 * Put the rq online, if not already. This happens: 6906 * 6907 * 1) In the early boot process, because we build the real domains 6908 * after all CPUs have been brought up. 6909 * 6910 * 2) At runtime, if cpuset_cpu_active() fails to rebuild the 6911 * domains. 6912 */ 6913 rq_lock_irqsave(rq, &rf); 6914 if (rq->rd) { 6915 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); 6916 set_rq_online(rq); 6917 } 6918 rq_unlock_irqrestore(rq, &rf); 6919 6920 return 0; 6921 } 6922 6923 int sched_cpu_deactivate(unsigned int cpu) 6924 { 6925 int ret; 6926 6927 set_cpu_active(cpu, false); 6928 /* 6929 * We've cleared cpu_active_mask, wait for all preempt-disabled and RCU 6930 * users of this state to go away such that all new such users will 6931 * observe it. 6932 * 6933 * Do sync before park smpboot threads to take care the rcu boost case. 6934 */ 6935 synchronize_rcu(); 6936 6937 #ifdef CONFIG_SCHED_SMT 6938 /* 6939 * When going down, decrement the number of cores with SMT present. 6940 */ 6941 if (cpumask_weight(cpu_smt_mask(cpu)) == 2) 6942 static_branch_dec_cpuslocked(&sched_smt_present); 6943 #endif 6944 6945 if (!sched_smp_initialized) 6946 return 0; 6947 6948 ret = cpuset_cpu_inactive(cpu); 6949 if (ret) { 6950 set_cpu_active(cpu, true); 6951 return ret; 6952 } 6953 sched_domains_numa_masks_clear(cpu); 6954 return 0; 6955 } 6956 6957 static void sched_rq_cpu_starting(unsigned int cpu) 6958 { 6959 struct rq *rq = cpu_rq(cpu); 6960 6961 rq->calc_load_update = calc_load_update; 6962 update_max_interval(); 6963 } 6964 6965 int sched_cpu_starting(unsigned int cpu) 6966 { 6967 sched_rq_cpu_starting(cpu); 6968 sched_tick_start(cpu); 6969 return 0; 6970 } 6971 6972 #ifdef CONFIG_HOTPLUG_CPU 6973 int sched_cpu_dying(unsigned int cpu) 6974 { 6975 struct rq *rq = cpu_rq(cpu); 6976 struct rq_flags rf; 6977 6978 /* Handle pending wakeups and then migrate everything off */ 6979 sched_tick_stop(cpu); 6980 6981 rq_lock_irqsave(rq, &rf); 6982 if (rq->rd) { 6983 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); 6984 set_rq_offline(rq); 6985 } 6986 migrate_tasks(rq, &rf); 6987 BUG_ON(rq->nr_running != 1); 6988 rq_unlock_irqrestore(rq, &rf); 6989 6990 calc_load_migrate(rq); 6991 update_max_interval(); 6992 nohz_balance_exit_idle(rq); 6993 hrtick_clear(rq); 6994 return 0; 6995 } 6996 #endif 6997 6998 void __init sched_init_smp(void) 6999 { 7000 sched_init_numa(); 7001 7002 /* 7003 * There's no userspace yet to cause hotplug operations; hence all the 7004 * CPU masks are stable and all blatant races in the below code cannot 7005 * happen. 7006 */ 7007 mutex_lock(&sched_domains_mutex); 7008 sched_init_domains(cpu_active_mask); 7009 mutex_unlock(&sched_domains_mutex); 7010 7011 /* Move init over to a non-isolated CPU */ 7012 if (set_cpus_allowed_ptr(current, housekeeping_cpumask(HK_FLAG_DOMAIN)) < 0) 7013 BUG(); 7014 sched_init_granularity(); 7015 7016 init_sched_rt_class(); 7017 init_sched_dl_class(); 7018 7019 sched_smp_initialized = true; 7020 } 7021 7022 static int __init migration_init(void) 7023 { 7024 sched_cpu_starting(smp_processor_id()); 7025 return 0; 7026 } 7027 early_initcall(migration_init); 7028 7029 #else 7030 void __init sched_init_smp(void) 7031 { 7032 sched_init_granularity(); 7033 } 7034 #endif /* CONFIG_SMP */ 7035 7036 int in_sched_functions(unsigned long addr) 7037 { 7038 return in_lock_functions(addr) || 7039 (addr >= (unsigned long)__sched_text_start 7040 && addr < (unsigned long)__sched_text_end); 7041 } 7042 7043 #ifdef CONFIG_CGROUP_SCHED 7044 /* 7045 * Default task group. 7046 * Every task in system belongs to this group at bootup. 7047 */ 7048 struct task_group root_task_group; 7049 LIST_HEAD(task_groups); 7050 7051 /* Cacheline aligned slab cache for task_group */ 7052 static struct kmem_cache *task_group_cache __read_mostly; 7053 #endif 7054 7055 DECLARE_PER_CPU(cpumask_var_t, load_balance_mask); 7056 DECLARE_PER_CPU(cpumask_var_t, select_idle_mask); 7057 7058 void __init sched_init(void) 7059 { 7060 unsigned long ptr = 0; 7061 int i; 7062 7063 /* Make sure the linker didn't screw up */ 7064 BUG_ON(&idle_sched_class + 1 != &fair_sched_class || 7065 &fair_sched_class + 1 != &rt_sched_class || 7066 &rt_sched_class + 1 != &dl_sched_class); 7067 #ifdef CONFIG_SMP 7068 BUG_ON(&dl_sched_class + 1 != &stop_sched_class); 7069 #endif 7070 7071 wait_bit_init(); 7072 7073 #ifdef CONFIG_FAIR_GROUP_SCHED 7074 ptr += 2 * nr_cpu_ids * sizeof(void **); 7075 #endif 7076 #ifdef CONFIG_RT_GROUP_SCHED 7077 ptr += 2 * nr_cpu_ids * sizeof(void **); 7078 #endif 7079 if (ptr) { 7080 ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT); 7081 7082 #ifdef CONFIG_FAIR_GROUP_SCHED 7083 root_task_group.se = (struct sched_entity **)ptr; 7084 ptr += nr_cpu_ids * sizeof(void **); 7085 7086 root_task_group.cfs_rq = (struct cfs_rq **)ptr; 7087 ptr += nr_cpu_ids * sizeof(void **); 7088 7089 root_task_group.shares = ROOT_TASK_GROUP_LOAD; 7090 init_cfs_bandwidth(&root_task_group.cfs_bandwidth); 7091 #endif /* CONFIG_FAIR_GROUP_SCHED */ 7092 #ifdef CONFIG_RT_GROUP_SCHED 7093 root_task_group.rt_se = (struct sched_rt_entity **)ptr; 7094 ptr += nr_cpu_ids * sizeof(void **); 7095 7096 root_task_group.rt_rq = (struct rt_rq **)ptr; 7097 ptr += nr_cpu_ids * sizeof(void **); 7098 7099 #endif /* CONFIG_RT_GROUP_SCHED */ 7100 } 7101 #ifdef CONFIG_CPUMASK_OFFSTACK 7102 for_each_possible_cpu(i) { 7103 per_cpu(load_balance_mask, i) = (cpumask_var_t)kzalloc_node( 7104 cpumask_size(), GFP_KERNEL, cpu_to_node(i)); 7105 per_cpu(select_idle_mask, i) = (cpumask_var_t)kzalloc_node( 7106 cpumask_size(), GFP_KERNEL, cpu_to_node(i)); 7107 } 7108 #endif /* CONFIG_CPUMASK_OFFSTACK */ 7109 7110 init_rt_bandwidth(&def_rt_bandwidth, global_rt_period(), global_rt_runtime()); 7111 init_dl_bandwidth(&def_dl_bandwidth, global_rt_period(), global_rt_runtime()); 7112 7113 #ifdef CONFIG_SMP 7114 init_defrootdomain(); 7115 #endif 7116 7117 #ifdef CONFIG_RT_GROUP_SCHED 7118 init_rt_bandwidth(&root_task_group.rt_bandwidth, 7119 global_rt_period(), global_rt_runtime()); 7120 #endif /* CONFIG_RT_GROUP_SCHED */ 7121 7122 #ifdef CONFIG_CGROUP_SCHED 7123 task_group_cache = KMEM_CACHE(task_group, 0); 7124 7125 list_add(&root_task_group.list, &task_groups); 7126 INIT_LIST_HEAD(&root_task_group.children); 7127 INIT_LIST_HEAD(&root_task_group.siblings); 7128 autogroup_init(&init_task); 7129 #endif /* CONFIG_CGROUP_SCHED */ 7130 7131 for_each_possible_cpu(i) { 7132 struct rq *rq; 7133 7134 rq = cpu_rq(i); 7135 raw_spin_lock_init(&rq->lock); 7136 rq->nr_running = 0; 7137 rq->calc_load_active = 0; 7138 rq->calc_load_update = jiffies + LOAD_FREQ; 7139 init_cfs_rq(&rq->cfs); 7140 init_rt_rq(&rq->rt); 7141 init_dl_rq(&rq->dl); 7142 #ifdef CONFIG_FAIR_GROUP_SCHED 7143 INIT_LIST_HEAD(&rq->leaf_cfs_rq_list); 7144 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 7145 /* 7146 * How much CPU bandwidth does root_task_group get? 7147 * 7148 * In case of task-groups formed thr' the cgroup filesystem, it 7149 * gets 100% of the CPU resources in the system. This overall 7150 * system CPU resource is divided among the tasks of 7151 * root_task_group and its child task-groups in a fair manner, 7152 * based on each entity's (task or task-group's) weight 7153 * (se->load.weight). 7154 * 7155 * In other words, if root_task_group has 10 tasks of weight 7156 * 1024) and two child groups A0 and A1 (of weight 1024 each), 7157 * then A0's share of the CPU resource is: 7158 * 7159 * A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33% 7160 * 7161 * We achieve this by letting root_task_group's tasks sit 7162 * directly in rq->cfs (i.e root_task_group->se[] = NULL). 7163 */ 7164 init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL); 7165 #endif /* CONFIG_FAIR_GROUP_SCHED */ 7166 7167 rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime; 7168 #ifdef CONFIG_RT_GROUP_SCHED 7169 init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL); 7170 #endif 7171 #ifdef CONFIG_SMP 7172 rq->sd = NULL; 7173 rq->rd = NULL; 7174 rq->cpu_capacity = rq->cpu_capacity_orig = SCHED_CAPACITY_SCALE; 7175 rq->balance_callback = NULL; 7176 rq->active_balance = 0; 7177 rq->next_balance = jiffies; 7178 rq->push_cpu = 0; 7179 rq->cpu = i; 7180 rq->online = 0; 7181 rq->idle_stamp = 0; 7182 rq->avg_idle = 2*sysctl_sched_migration_cost; 7183 rq->max_idle_balance_cost = sysctl_sched_migration_cost; 7184 7185 INIT_LIST_HEAD(&rq->cfs_tasks); 7186 7187 rq_attach_root(rq, &def_root_domain); 7188 #ifdef CONFIG_NO_HZ_COMMON 7189 rq->last_blocked_load_update_tick = jiffies; 7190 atomic_set(&rq->nohz_flags, 0); 7191 7192 rq_csd_init(rq, &rq->nohz_csd, nohz_csd_func); 7193 #endif 7194 #endif /* CONFIG_SMP */ 7195 hrtick_rq_init(rq); 7196 atomic_set(&rq->nr_iowait, 0); 7197 } 7198 7199 set_load_weight(&init_task, false); 7200 7201 /* 7202 * The boot idle thread does lazy MMU switching as well: 7203 */ 7204 mmgrab(&init_mm); 7205 enter_lazy_tlb(&init_mm, current); 7206 7207 /* 7208 * Make us the idle thread. Technically, schedule() should not be 7209 * called from this thread, however somewhere below it might be, 7210 * but because we are the idle thread, we just pick up running again 7211 * when this runqueue becomes "idle". 7212 */ 7213 init_idle(current, smp_processor_id()); 7214 7215 calc_load_update = jiffies + LOAD_FREQ; 7216 7217 #ifdef CONFIG_SMP 7218 idle_thread_set_boot_cpu(); 7219 #endif 7220 init_sched_fair_class(); 7221 7222 init_schedstats(); 7223 7224 psi_init(); 7225 7226 init_uclamp(); 7227 7228 scheduler_running = 1; 7229 } 7230 7231 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP 7232 static inline int preempt_count_equals(int preempt_offset) 7233 { 7234 int nested = preempt_count() + rcu_preempt_depth(); 7235 7236 return (nested == preempt_offset); 7237 } 7238 7239 void __might_sleep(const char *file, int line, int preempt_offset) 7240 { 7241 /* 7242 * Blocking primitives will set (and therefore destroy) current->state, 7243 * since we will exit with TASK_RUNNING make sure we enter with it, 7244 * otherwise we will destroy state. 7245 */ 7246 WARN_ONCE(current->state != TASK_RUNNING && current->task_state_change, 7247 "do not call blocking ops when !TASK_RUNNING; " 7248 "state=%lx set at [<%p>] %pS\n", 7249 current->state, 7250 (void *)current->task_state_change, 7251 (void *)current->task_state_change); 7252 7253 ___might_sleep(file, line, preempt_offset); 7254 } 7255 EXPORT_SYMBOL(__might_sleep); 7256 7257 void ___might_sleep(const char *file, int line, int preempt_offset) 7258 { 7259 /* Ratelimiting timestamp: */ 7260 static unsigned long prev_jiffy; 7261 7262 unsigned long preempt_disable_ip; 7263 7264 /* WARN_ON_ONCE() by default, no rate limit required: */ 7265 rcu_sleep_check(); 7266 7267 if ((preempt_count_equals(preempt_offset) && !irqs_disabled() && 7268 !is_idle_task(current) && !current->non_block_count) || 7269 system_state == SYSTEM_BOOTING || system_state > SYSTEM_RUNNING || 7270 oops_in_progress) 7271 return; 7272 7273 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) 7274 return; 7275 prev_jiffy = jiffies; 7276 7277 /* Save this before calling printk(), since that will clobber it: */ 7278 preempt_disable_ip = get_preempt_disable_ip(current); 7279 7280 printk(KERN_ERR 7281 "BUG: sleeping function called from invalid context at %s:%d\n", 7282 file, line); 7283 printk(KERN_ERR 7284 "in_atomic(): %d, irqs_disabled(): %d, non_block: %d, pid: %d, name: %s\n", 7285 in_atomic(), irqs_disabled(), current->non_block_count, 7286 current->pid, current->comm); 7287 7288 if (task_stack_end_corrupted(current)) 7289 printk(KERN_EMERG "Thread overran stack, or stack corrupted\n"); 7290 7291 debug_show_held_locks(current); 7292 if (irqs_disabled()) 7293 print_irqtrace_events(current); 7294 if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) 7295 && !preempt_count_equals(preempt_offset)) { 7296 pr_err("Preemption disabled at:"); 7297 print_ip_sym(KERN_ERR, preempt_disable_ip); 7298 } 7299 dump_stack(); 7300 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 7301 } 7302 EXPORT_SYMBOL(___might_sleep); 7303 7304 void __cant_sleep(const char *file, int line, int preempt_offset) 7305 { 7306 static unsigned long prev_jiffy; 7307 7308 if (irqs_disabled()) 7309 return; 7310 7311 if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) 7312 return; 7313 7314 if (preempt_count() > preempt_offset) 7315 return; 7316 7317 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) 7318 return; 7319 prev_jiffy = jiffies; 7320 7321 printk(KERN_ERR "BUG: assuming atomic context at %s:%d\n", file, line); 7322 printk(KERN_ERR "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n", 7323 in_atomic(), irqs_disabled(), 7324 current->pid, current->comm); 7325 7326 debug_show_held_locks(current); 7327 dump_stack(); 7328 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 7329 } 7330 EXPORT_SYMBOL_GPL(__cant_sleep); 7331 #endif 7332 7333 #ifdef CONFIG_MAGIC_SYSRQ 7334 void normalize_rt_tasks(void) 7335 { 7336 struct task_struct *g, *p; 7337 struct sched_attr attr = { 7338 .sched_policy = SCHED_NORMAL, 7339 }; 7340 7341 read_lock(&tasklist_lock); 7342 for_each_process_thread(g, p) { 7343 /* 7344 * Only normalize user tasks: 7345 */ 7346 if (p->flags & PF_KTHREAD) 7347 continue; 7348 7349 p->se.exec_start = 0; 7350 schedstat_set(p->se.statistics.wait_start, 0); 7351 schedstat_set(p->se.statistics.sleep_start, 0); 7352 schedstat_set(p->se.statistics.block_start, 0); 7353 7354 if (!dl_task(p) && !rt_task(p)) { 7355 /* 7356 * Renice negative nice level userspace 7357 * tasks back to 0: 7358 */ 7359 if (task_nice(p) < 0) 7360 set_user_nice(p, 0); 7361 continue; 7362 } 7363 7364 __sched_setscheduler(p, &attr, false, false); 7365 } 7366 read_unlock(&tasklist_lock); 7367 } 7368 7369 #endif /* CONFIG_MAGIC_SYSRQ */ 7370 7371 #if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) 7372 /* 7373 * These functions are only useful for the IA64 MCA handling, or kdb. 7374 * 7375 * They can only be called when the whole system has been 7376 * stopped - every CPU needs to be quiescent, and no scheduling 7377 * activity can take place. Using them for anything else would 7378 * be a serious bug, and as a result, they aren't even visible 7379 * under any other configuration. 7380 */ 7381 7382 /** 7383 * curr_task - return the current task for a given CPU. 7384 * @cpu: the processor in question. 7385 * 7386 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! 7387 * 7388 * Return: The current task for @cpu. 7389 */ 7390 struct task_struct *curr_task(int cpu) 7391 { 7392 return cpu_curr(cpu); 7393 } 7394 7395 #endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */ 7396 7397 #ifdef CONFIG_IA64 7398 /** 7399 * ia64_set_curr_task - set the current task for a given CPU. 7400 * @cpu: the processor in question. 7401 * @p: the task pointer to set. 7402 * 7403 * Description: This function must only be used when non-maskable interrupts 7404 * are serviced on a separate stack. It allows the architecture to switch the 7405 * notion of the current task on a CPU in a non-blocking manner. This function 7406 * must be called with all CPU's synchronized, and interrupts disabled, the 7407 * and caller must save the original value of the current task (see 7408 * curr_task() above) and restore that value before reenabling interrupts and 7409 * re-starting the system. 7410 * 7411 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! 7412 */ 7413 void ia64_set_curr_task(int cpu, struct task_struct *p) 7414 { 7415 cpu_curr(cpu) = p; 7416 } 7417 7418 #endif 7419 7420 #ifdef CONFIG_CGROUP_SCHED 7421 /* task_group_lock serializes the addition/removal of task groups */ 7422 static DEFINE_SPINLOCK(task_group_lock); 7423 7424 static inline void alloc_uclamp_sched_group(struct task_group *tg, 7425 struct task_group *parent) 7426 { 7427 #ifdef CONFIG_UCLAMP_TASK_GROUP 7428 enum uclamp_id clamp_id; 7429 7430 for_each_clamp_id(clamp_id) { 7431 uclamp_se_set(&tg->uclamp_req[clamp_id], 7432 uclamp_none(clamp_id), false); 7433 tg->uclamp[clamp_id] = parent->uclamp[clamp_id]; 7434 } 7435 #endif 7436 } 7437 7438 static void sched_free_group(struct task_group *tg) 7439 { 7440 free_fair_sched_group(tg); 7441 free_rt_sched_group(tg); 7442 autogroup_free(tg); 7443 kmem_cache_free(task_group_cache, tg); 7444 } 7445 7446 /* allocate runqueue etc for a new task group */ 7447 struct task_group *sched_create_group(struct task_group *parent) 7448 { 7449 struct task_group *tg; 7450 7451 tg = kmem_cache_alloc(task_group_cache, GFP_KERNEL | __GFP_ZERO); 7452 if (!tg) 7453 return ERR_PTR(-ENOMEM); 7454 7455 if (!alloc_fair_sched_group(tg, parent)) 7456 goto err; 7457 7458 if (!alloc_rt_sched_group(tg, parent)) 7459 goto err; 7460 7461 alloc_uclamp_sched_group(tg, parent); 7462 7463 return tg; 7464 7465 err: 7466 sched_free_group(tg); 7467 return ERR_PTR(-ENOMEM); 7468 } 7469 7470 void sched_online_group(struct task_group *tg, struct task_group *parent) 7471 { 7472 unsigned long flags; 7473 7474 spin_lock_irqsave(&task_group_lock, flags); 7475 list_add_rcu(&tg->list, &task_groups); 7476 7477 /* Root should already exist: */ 7478 WARN_ON(!parent); 7479 7480 tg->parent = parent; 7481 INIT_LIST_HEAD(&tg->children); 7482 list_add_rcu(&tg->siblings, &parent->children); 7483 spin_unlock_irqrestore(&task_group_lock, flags); 7484 7485 online_fair_sched_group(tg); 7486 } 7487 7488 /* rcu callback to free various structures associated with a task group */ 7489 static void sched_free_group_rcu(struct rcu_head *rhp) 7490 { 7491 /* Now it should be safe to free those cfs_rqs: */ 7492 sched_free_group(container_of(rhp, struct task_group, rcu)); 7493 } 7494 7495 void sched_destroy_group(struct task_group *tg) 7496 { 7497 /* Wait for possible concurrent references to cfs_rqs complete: */ 7498 call_rcu(&tg->rcu, sched_free_group_rcu); 7499 } 7500 7501 void sched_offline_group(struct task_group *tg) 7502 { 7503 unsigned long flags; 7504 7505 /* End participation in shares distribution: */ 7506 unregister_fair_sched_group(tg); 7507 7508 spin_lock_irqsave(&task_group_lock, flags); 7509 list_del_rcu(&tg->list); 7510 list_del_rcu(&tg->siblings); 7511 spin_unlock_irqrestore(&task_group_lock, flags); 7512 } 7513 7514 static void sched_change_group(struct task_struct *tsk, int type) 7515 { 7516 struct task_group *tg; 7517 7518 /* 7519 * All callers are synchronized by task_rq_lock(); we do not use RCU 7520 * which is pointless here. Thus, we pass "true" to task_css_check() 7521 * to prevent lockdep warnings. 7522 */ 7523 tg = container_of(task_css_check(tsk, cpu_cgrp_id, true), 7524 struct task_group, css); 7525 tg = autogroup_task_group(tsk, tg); 7526 tsk->sched_task_group = tg; 7527 7528 #ifdef CONFIG_FAIR_GROUP_SCHED 7529 if (tsk->sched_class->task_change_group) 7530 tsk->sched_class->task_change_group(tsk, type); 7531 else 7532 #endif 7533 set_task_rq(tsk, task_cpu(tsk)); 7534 } 7535 7536 /* 7537 * Change task's runqueue when it moves between groups. 7538 * 7539 * The caller of this function should have put the task in its new group by 7540 * now. This function just updates tsk->se.cfs_rq and tsk->se.parent to reflect 7541 * its new group. 7542 */ 7543 void sched_move_task(struct task_struct *tsk) 7544 { 7545 int queued, running, queue_flags = 7546 DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; 7547 struct rq_flags rf; 7548 struct rq *rq; 7549 7550 rq = task_rq_lock(tsk, &rf); 7551 update_rq_clock(rq); 7552 7553 running = task_current(rq, tsk); 7554 queued = task_on_rq_queued(tsk); 7555 7556 if (queued) 7557 dequeue_task(rq, tsk, queue_flags); 7558 if (running) 7559 put_prev_task(rq, tsk); 7560 7561 sched_change_group(tsk, TASK_MOVE_GROUP); 7562 7563 if (queued) 7564 enqueue_task(rq, tsk, queue_flags); 7565 if (running) { 7566 set_next_task(rq, tsk); 7567 /* 7568 * After changing group, the running task may have joined a 7569 * throttled one but it's still the running task. Trigger a 7570 * resched to make sure that task can still run. 7571 */ 7572 resched_curr(rq); 7573 } 7574 7575 task_rq_unlock(rq, tsk, &rf); 7576 } 7577 7578 static inline struct task_group *css_tg(struct cgroup_subsys_state *css) 7579 { 7580 return css ? container_of(css, struct task_group, css) : NULL; 7581 } 7582 7583 static struct cgroup_subsys_state * 7584 cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) 7585 { 7586 struct task_group *parent = css_tg(parent_css); 7587 struct task_group *tg; 7588 7589 if (!parent) { 7590 /* This is early initialization for the top cgroup */ 7591 return &root_task_group.css; 7592 } 7593 7594 tg = sched_create_group(parent); 7595 if (IS_ERR(tg)) 7596 return ERR_PTR(-ENOMEM); 7597 7598 return &tg->css; 7599 } 7600 7601 /* Expose task group only after completing cgroup initialization */ 7602 static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) 7603 { 7604 struct task_group *tg = css_tg(css); 7605 struct task_group *parent = css_tg(css->parent); 7606 7607 if (parent) 7608 sched_online_group(tg, parent); 7609 7610 #ifdef CONFIG_UCLAMP_TASK_GROUP 7611 /* Propagate the effective uclamp value for the new group */ 7612 cpu_util_update_eff(css); 7613 #endif 7614 7615 return 0; 7616 } 7617 7618 static void cpu_cgroup_css_released(struct cgroup_subsys_state *css) 7619 { 7620 struct task_group *tg = css_tg(css); 7621 7622 sched_offline_group(tg); 7623 } 7624 7625 static void cpu_cgroup_css_free(struct cgroup_subsys_state *css) 7626 { 7627 struct task_group *tg = css_tg(css); 7628 7629 /* 7630 * Relies on the RCU grace period between css_released() and this. 7631 */ 7632 sched_free_group(tg); 7633 } 7634 7635 /* 7636 * This is called before wake_up_new_task(), therefore we really only 7637 * have to set its group bits, all the other stuff does not apply. 7638 */ 7639 static void cpu_cgroup_fork(struct task_struct *task) 7640 { 7641 struct rq_flags rf; 7642 struct rq *rq; 7643 7644 rq = task_rq_lock(task, &rf); 7645 7646 update_rq_clock(rq); 7647 sched_change_group(task, TASK_SET_GROUP); 7648 7649 task_rq_unlock(rq, task, &rf); 7650 } 7651 7652 static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) 7653 { 7654 struct task_struct *task; 7655 struct cgroup_subsys_state *css; 7656 int ret = 0; 7657 7658 cgroup_taskset_for_each(task, css, tset) { 7659 #ifdef CONFIG_RT_GROUP_SCHED 7660 if (!sched_rt_can_attach(css_tg(css), task)) 7661 return -EINVAL; 7662 #endif 7663 /* 7664 * Serialize against wake_up_new_task() such that if its 7665 * running, we're sure to observe its full state. 7666 */ 7667 raw_spin_lock_irq(&task->pi_lock); 7668 /* 7669 * Avoid calling sched_move_task() before wake_up_new_task() 7670 * has happened. This would lead to problems with PELT, due to 7671 * move wanting to detach+attach while we're not attached yet. 7672 */ 7673 if (task->state == TASK_NEW) 7674 ret = -EINVAL; 7675 raw_spin_unlock_irq(&task->pi_lock); 7676 7677 if (ret) 7678 break; 7679 } 7680 return ret; 7681 } 7682 7683 static void cpu_cgroup_attach(struct cgroup_taskset *tset) 7684 { 7685 struct task_struct *task; 7686 struct cgroup_subsys_state *css; 7687 7688 cgroup_taskset_for_each(task, css, tset) 7689 sched_move_task(task); 7690 } 7691 7692 #ifdef CONFIG_UCLAMP_TASK_GROUP 7693 static void cpu_util_update_eff(struct cgroup_subsys_state *css) 7694 { 7695 struct cgroup_subsys_state *top_css = css; 7696 struct uclamp_se *uc_parent = NULL; 7697 struct uclamp_se *uc_se = NULL; 7698 unsigned int eff[UCLAMP_CNT]; 7699 enum uclamp_id clamp_id; 7700 unsigned int clamps; 7701 7702 css_for_each_descendant_pre(css, top_css) { 7703 uc_parent = css_tg(css)->parent 7704 ? css_tg(css)->parent->uclamp : NULL; 7705 7706 for_each_clamp_id(clamp_id) { 7707 /* Assume effective clamps matches requested clamps */ 7708 eff[clamp_id] = css_tg(css)->uclamp_req[clamp_id].value; 7709 /* Cap effective clamps with parent's effective clamps */ 7710 if (uc_parent && 7711 eff[clamp_id] > uc_parent[clamp_id].value) { 7712 eff[clamp_id] = uc_parent[clamp_id].value; 7713 } 7714 } 7715 /* Ensure protection is always capped by limit */ 7716 eff[UCLAMP_MIN] = min(eff[UCLAMP_MIN], eff[UCLAMP_MAX]); 7717 7718 /* Propagate most restrictive effective clamps */ 7719 clamps = 0x0; 7720 uc_se = css_tg(css)->uclamp; 7721 for_each_clamp_id(clamp_id) { 7722 if (eff[clamp_id] == uc_se[clamp_id].value) 7723 continue; 7724 uc_se[clamp_id].value = eff[clamp_id]; 7725 uc_se[clamp_id].bucket_id = uclamp_bucket_id(eff[clamp_id]); 7726 clamps |= (0x1 << clamp_id); 7727 } 7728 if (!clamps) { 7729 css = css_rightmost_descendant(css); 7730 continue; 7731 } 7732 7733 /* Immediately update descendants RUNNABLE tasks */ 7734 uclamp_update_active_tasks(css, clamps); 7735 } 7736 } 7737 7738 /* 7739 * Integer 10^N with a given N exponent by casting to integer the literal "1eN" 7740 * C expression. Since there is no way to convert a macro argument (N) into a 7741 * character constant, use two levels of macros. 7742 */ 7743 #define _POW10(exp) ((unsigned int)1e##exp) 7744 #define POW10(exp) _POW10(exp) 7745 7746 struct uclamp_request { 7747 #define UCLAMP_PERCENT_SHIFT 2 7748 #define UCLAMP_PERCENT_SCALE (100 * POW10(UCLAMP_PERCENT_SHIFT)) 7749 s64 percent; 7750 u64 util; 7751 int ret; 7752 }; 7753 7754 static inline struct uclamp_request 7755 capacity_from_percent(char *buf) 7756 { 7757 struct uclamp_request req = { 7758 .percent = UCLAMP_PERCENT_SCALE, 7759 .util = SCHED_CAPACITY_SCALE, 7760 .ret = 0, 7761 }; 7762 7763 buf = strim(buf); 7764 if (strcmp(buf, "max")) { 7765 req.ret = cgroup_parse_float(buf, UCLAMP_PERCENT_SHIFT, 7766 &req.percent); 7767 if (req.ret) 7768 return req; 7769 if ((u64)req.percent > UCLAMP_PERCENT_SCALE) { 7770 req.ret = -ERANGE; 7771 return req; 7772 } 7773 7774 req.util = req.percent << SCHED_CAPACITY_SHIFT; 7775 req.util = DIV_ROUND_CLOSEST_ULL(req.util, UCLAMP_PERCENT_SCALE); 7776 } 7777 7778 return req; 7779 } 7780 7781 static ssize_t cpu_uclamp_write(struct kernfs_open_file *of, char *buf, 7782 size_t nbytes, loff_t off, 7783 enum uclamp_id clamp_id) 7784 { 7785 struct uclamp_request req; 7786 struct task_group *tg; 7787 7788 req = capacity_from_percent(buf); 7789 if (req.ret) 7790 return req.ret; 7791 7792 static_branch_enable(&sched_uclamp_used); 7793 7794 mutex_lock(&uclamp_mutex); 7795 rcu_read_lock(); 7796 7797 tg = css_tg(of_css(of)); 7798 if (tg->uclamp_req[clamp_id].value != req.util) 7799 uclamp_se_set(&tg->uclamp_req[clamp_id], req.util, false); 7800 7801 /* 7802 * Because of not recoverable conversion rounding we keep track of the 7803 * exact requested value 7804 */ 7805 tg->uclamp_pct[clamp_id] = req.percent; 7806 7807 /* Update effective clamps to track the most restrictive value */ 7808 cpu_util_update_eff(of_css(of)); 7809 7810 rcu_read_unlock(); 7811 mutex_unlock(&uclamp_mutex); 7812 7813 return nbytes; 7814 } 7815 7816 static ssize_t cpu_uclamp_min_write(struct kernfs_open_file *of, 7817 char *buf, size_t nbytes, 7818 loff_t off) 7819 { 7820 return cpu_uclamp_write(of, buf, nbytes, off, UCLAMP_MIN); 7821 } 7822 7823 static ssize_t cpu_uclamp_max_write(struct kernfs_open_file *of, 7824 char *buf, size_t nbytes, 7825 loff_t off) 7826 { 7827 return cpu_uclamp_write(of, buf, nbytes, off, UCLAMP_MAX); 7828 } 7829 7830 static inline void cpu_uclamp_print(struct seq_file *sf, 7831 enum uclamp_id clamp_id) 7832 { 7833 struct task_group *tg; 7834 u64 util_clamp; 7835 u64 percent; 7836 u32 rem; 7837 7838 rcu_read_lock(); 7839 tg = css_tg(seq_css(sf)); 7840 util_clamp = tg->uclamp_req[clamp_id].value; 7841 rcu_read_unlock(); 7842 7843 if (util_clamp == SCHED_CAPACITY_SCALE) { 7844 seq_puts(sf, "max\n"); 7845 return; 7846 } 7847 7848 percent = tg->uclamp_pct[clamp_id]; 7849 percent = div_u64_rem(percent, POW10(UCLAMP_PERCENT_SHIFT), &rem); 7850 seq_printf(sf, "%llu.%0*u\n", percent, UCLAMP_PERCENT_SHIFT, rem); 7851 } 7852 7853 static int cpu_uclamp_min_show(struct seq_file *sf, void *v) 7854 { 7855 cpu_uclamp_print(sf, UCLAMP_MIN); 7856 return 0; 7857 } 7858 7859 static int cpu_uclamp_max_show(struct seq_file *sf, void *v) 7860 { 7861 cpu_uclamp_print(sf, UCLAMP_MAX); 7862 return 0; 7863 } 7864 #endif /* CONFIG_UCLAMP_TASK_GROUP */ 7865 7866 #ifdef CONFIG_FAIR_GROUP_SCHED 7867 static int cpu_shares_write_u64(struct cgroup_subsys_state *css, 7868 struct cftype *cftype, u64 shareval) 7869 { 7870 if (shareval > scale_load_down(ULONG_MAX)) 7871 shareval = MAX_SHARES; 7872 return sched_group_set_shares(css_tg(css), scale_load(shareval)); 7873 } 7874 7875 static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css, 7876 struct cftype *cft) 7877 { 7878 struct task_group *tg = css_tg(css); 7879 7880 return (u64) scale_load_down(tg->shares); 7881 } 7882 7883 #ifdef CONFIG_CFS_BANDWIDTH 7884 static DEFINE_MUTEX(cfs_constraints_mutex); 7885 7886 const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */ 7887 static const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */ 7888 /* More than 203 days if BW_SHIFT equals 20. */ 7889 static const u64 max_cfs_runtime = MAX_BW * NSEC_PER_USEC; 7890 7891 static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime); 7892 7893 static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota) 7894 { 7895 int i, ret = 0, runtime_enabled, runtime_was_enabled; 7896 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 7897 7898 if (tg == &root_task_group) 7899 return -EINVAL; 7900 7901 /* 7902 * Ensure we have at some amount of bandwidth every period. This is 7903 * to prevent reaching a state of large arrears when throttled via 7904 * entity_tick() resulting in prolonged exit starvation. 7905 */ 7906 if (quota < min_cfs_quota_period || period < min_cfs_quota_period) 7907 return -EINVAL; 7908 7909 /* 7910 * Likewise, bound things on the otherside by preventing insane quota 7911 * periods. This also allows us to normalize in computing quota 7912 * feasibility. 7913 */ 7914 if (period > max_cfs_quota_period) 7915 return -EINVAL; 7916 7917 /* 7918 * Bound quota to defend quota against overflow during bandwidth shift. 7919 */ 7920 if (quota != RUNTIME_INF && quota > max_cfs_runtime) 7921 return -EINVAL; 7922 7923 /* 7924 * Prevent race between setting of cfs_rq->runtime_enabled and 7925 * unthrottle_offline_cfs_rqs(). 7926 */ 7927 get_online_cpus(); 7928 mutex_lock(&cfs_constraints_mutex); 7929 ret = __cfs_schedulable(tg, period, quota); 7930 if (ret) 7931 goto out_unlock; 7932 7933 runtime_enabled = quota != RUNTIME_INF; 7934 runtime_was_enabled = cfs_b->quota != RUNTIME_INF; 7935 /* 7936 * If we need to toggle cfs_bandwidth_used, off->on must occur 7937 * before making related changes, and on->off must occur afterwards 7938 */ 7939 if (runtime_enabled && !runtime_was_enabled) 7940 cfs_bandwidth_usage_inc(); 7941 raw_spin_lock_irq(&cfs_b->lock); 7942 cfs_b->period = ns_to_ktime(period); 7943 cfs_b->quota = quota; 7944 7945 __refill_cfs_bandwidth_runtime(cfs_b); 7946 7947 /* Restart the period timer (if active) to handle new period expiry: */ 7948 if (runtime_enabled) 7949 start_cfs_bandwidth(cfs_b); 7950 7951 raw_spin_unlock_irq(&cfs_b->lock); 7952 7953 for_each_online_cpu(i) { 7954 struct cfs_rq *cfs_rq = tg->cfs_rq[i]; 7955 struct rq *rq = cfs_rq->rq; 7956 struct rq_flags rf; 7957 7958 rq_lock_irq(rq, &rf); 7959 cfs_rq->runtime_enabled = runtime_enabled; 7960 cfs_rq->runtime_remaining = 0; 7961 7962 if (cfs_rq->throttled) 7963 unthrottle_cfs_rq(cfs_rq); 7964 rq_unlock_irq(rq, &rf); 7965 } 7966 if (runtime_was_enabled && !runtime_enabled) 7967 cfs_bandwidth_usage_dec(); 7968 out_unlock: 7969 mutex_unlock(&cfs_constraints_mutex); 7970 put_online_cpus(); 7971 7972 return ret; 7973 } 7974 7975 static int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us) 7976 { 7977 u64 quota, period; 7978 7979 period = ktime_to_ns(tg->cfs_bandwidth.period); 7980 if (cfs_quota_us < 0) 7981 quota = RUNTIME_INF; 7982 else if ((u64)cfs_quota_us <= U64_MAX / NSEC_PER_USEC) 7983 quota = (u64)cfs_quota_us * NSEC_PER_USEC; 7984 else 7985 return -EINVAL; 7986 7987 return tg_set_cfs_bandwidth(tg, period, quota); 7988 } 7989 7990 static long tg_get_cfs_quota(struct task_group *tg) 7991 { 7992 u64 quota_us; 7993 7994 if (tg->cfs_bandwidth.quota == RUNTIME_INF) 7995 return -1; 7996 7997 quota_us = tg->cfs_bandwidth.quota; 7998 do_div(quota_us, NSEC_PER_USEC); 7999 8000 return quota_us; 8001 } 8002 8003 static int tg_set_cfs_period(struct task_group *tg, long cfs_period_us) 8004 { 8005 u64 quota, period; 8006 8007 if ((u64)cfs_period_us > U64_MAX / NSEC_PER_USEC) 8008 return -EINVAL; 8009 8010 period = (u64)cfs_period_us * NSEC_PER_USEC; 8011 quota = tg->cfs_bandwidth.quota; 8012 8013 return tg_set_cfs_bandwidth(tg, period, quota); 8014 } 8015 8016 static long tg_get_cfs_period(struct task_group *tg) 8017 { 8018 u64 cfs_period_us; 8019 8020 cfs_period_us = ktime_to_ns(tg->cfs_bandwidth.period); 8021 do_div(cfs_period_us, NSEC_PER_USEC); 8022 8023 return cfs_period_us; 8024 } 8025 8026 static s64 cpu_cfs_quota_read_s64(struct cgroup_subsys_state *css, 8027 struct cftype *cft) 8028 { 8029 return tg_get_cfs_quota(css_tg(css)); 8030 } 8031 8032 static int cpu_cfs_quota_write_s64(struct cgroup_subsys_state *css, 8033 struct cftype *cftype, s64 cfs_quota_us) 8034 { 8035 return tg_set_cfs_quota(css_tg(css), cfs_quota_us); 8036 } 8037 8038 static u64 cpu_cfs_period_read_u64(struct cgroup_subsys_state *css, 8039 struct cftype *cft) 8040 { 8041 return tg_get_cfs_period(css_tg(css)); 8042 } 8043 8044 static int cpu_cfs_period_write_u64(struct cgroup_subsys_state *css, 8045 struct cftype *cftype, u64 cfs_period_us) 8046 { 8047 return tg_set_cfs_period(css_tg(css), cfs_period_us); 8048 } 8049 8050 struct cfs_schedulable_data { 8051 struct task_group *tg; 8052 u64 period, quota; 8053 }; 8054 8055 /* 8056 * normalize group quota/period to be quota/max_period 8057 * note: units are usecs 8058 */ 8059 static u64 normalize_cfs_quota(struct task_group *tg, 8060 struct cfs_schedulable_data *d) 8061 { 8062 u64 quota, period; 8063 8064 if (tg == d->tg) { 8065 period = d->period; 8066 quota = d->quota; 8067 } else { 8068 period = tg_get_cfs_period(tg); 8069 quota = tg_get_cfs_quota(tg); 8070 } 8071 8072 /* note: these should typically be equivalent */ 8073 if (quota == RUNTIME_INF || quota == -1) 8074 return RUNTIME_INF; 8075 8076 return to_ratio(period, quota); 8077 } 8078 8079 static int tg_cfs_schedulable_down(struct task_group *tg, void *data) 8080 { 8081 struct cfs_schedulable_data *d = data; 8082 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 8083 s64 quota = 0, parent_quota = -1; 8084 8085 if (!tg->parent) { 8086 quota = RUNTIME_INF; 8087 } else { 8088 struct cfs_bandwidth *parent_b = &tg->parent->cfs_bandwidth; 8089 8090 quota = normalize_cfs_quota(tg, d); 8091 parent_quota = parent_b->hierarchical_quota; 8092 8093 /* 8094 * Ensure max(child_quota) <= parent_quota. On cgroup2, 8095 * always take the min. On cgroup1, only inherit when no 8096 * limit is set: 8097 */ 8098 if (cgroup_subsys_on_dfl(cpu_cgrp_subsys)) { 8099 quota = min(quota, parent_quota); 8100 } else { 8101 if (quota == RUNTIME_INF) 8102 quota = parent_quota; 8103 else if (parent_quota != RUNTIME_INF && quota > parent_quota) 8104 return -EINVAL; 8105 } 8106 } 8107 cfs_b->hierarchical_quota = quota; 8108 8109 return 0; 8110 } 8111 8112 static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota) 8113 { 8114 int ret; 8115 struct cfs_schedulable_data data = { 8116 .tg = tg, 8117 .period = period, 8118 .quota = quota, 8119 }; 8120 8121 if (quota != RUNTIME_INF) { 8122 do_div(data.period, NSEC_PER_USEC); 8123 do_div(data.quota, NSEC_PER_USEC); 8124 } 8125 8126 rcu_read_lock(); 8127 ret = walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data); 8128 rcu_read_unlock(); 8129 8130 return ret; 8131 } 8132 8133 static int cpu_cfs_stat_show(struct seq_file *sf, void *v) 8134 { 8135 struct task_group *tg = css_tg(seq_css(sf)); 8136 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 8137 8138 seq_printf(sf, "nr_periods %d\n", cfs_b->nr_periods); 8139 seq_printf(sf, "nr_throttled %d\n", cfs_b->nr_throttled); 8140 seq_printf(sf, "throttled_time %llu\n", cfs_b->throttled_time); 8141 8142 if (schedstat_enabled() && tg != &root_task_group) { 8143 u64 ws = 0; 8144 int i; 8145 8146 for_each_possible_cpu(i) 8147 ws += schedstat_val(tg->se[i]->statistics.wait_sum); 8148 8149 seq_printf(sf, "wait_sum %llu\n", ws); 8150 } 8151 8152 return 0; 8153 } 8154 #endif /* CONFIG_CFS_BANDWIDTH */ 8155 #endif /* CONFIG_FAIR_GROUP_SCHED */ 8156 8157 #ifdef CONFIG_RT_GROUP_SCHED 8158 static int cpu_rt_runtime_write(struct cgroup_subsys_state *css, 8159 struct cftype *cft, s64 val) 8160 { 8161 return sched_group_set_rt_runtime(css_tg(css), val); 8162 } 8163 8164 static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css, 8165 struct cftype *cft) 8166 { 8167 return sched_group_rt_runtime(css_tg(css)); 8168 } 8169 8170 static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css, 8171 struct cftype *cftype, u64 rt_period_us) 8172 { 8173 return sched_group_set_rt_period(css_tg(css), rt_period_us); 8174 } 8175 8176 static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css, 8177 struct cftype *cft) 8178 { 8179 return sched_group_rt_period(css_tg(css)); 8180 } 8181 #endif /* CONFIG_RT_GROUP_SCHED */ 8182 8183 static struct cftype cpu_legacy_files[] = { 8184 #ifdef CONFIG_FAIR_GROUP_SCHED 8185 { 8186 .name = "shares", 8187 .read_u64 = cpu_shares_read_u64, 8188 .write_u64 = cpu_shares_write_u64, 8189 }, 8190 #endif 8191 #ifdef CONFIG_CFS_BANDWIDTH 8192 { 8193 .name = "cfs_quota_us", 8194 .read_s64 = cpu_cfs_quota_read_s64, 8195 .write_s64 = cpu_cfs_quota_write_s64, 8196 }, 8197 { 8198 .name = "cfs_period_us", 8199 .read_u64 = cpu_cfs_period_read_u64, 8200 .write_u64 = cpu_cfs_period_write_u64, 8201 }, 8202 { 8203 .name = "stat", 8204 .seq_show = cpu_cfs_stat_show, 8205 }, 8206 #endif 8207 #ifdef CONFIG_RT_GROUP_SCHED 8208 { 8209 .name = "rt_runtime_us", 8210 .read_s64 = cpu_rt_runtime_read, 8211 .write_s64 = cpu_rt_runtime_write, 8212 }, 8213 { 8214 .name = "rt_period_us", 8215 .read_u64 = cpu_rt_period_read_uint, 8216 .write_u64 = cpu_rt_period_write_uint, 8217 }, 8218 #endif 8219 #ifdef CONFIG_UCLAMP_TASK_GROUP 8220 { 8221 .name = "uclamp.min", 8222 .flags = CFTYPE_NOT_ON_ROOT, 8223 .seq_show = cpu_uclamp_min_show, 8224 .write = cpu_uclamp_min_write, 8225 }, 8226 { 8227 .name = "uclamp.max", 8228 .flags = CFTYPE_NOT_ON_ROOT, 8229 .seq_show = cpu_uclamp_max_show, 8230 .write = cpu_uclamp_max_write, 8231 }, 8232 #endif 8233 { } /* Terminate */ 8234 }; 8235 8236 static int cpu_extra_stat_show(struct seq_file *sf, 8237 struct cgroup_subsys_state *css) 8238 { 8239 #ifdef CONFIG_CFS_BANDWIDTH 8240 { 8241 struct task_group *tg = css_tg(css); 8242 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 8243 u64 throttled_usec; 8244 8245 throttled_usec = cfs_b->throttled_time; 8246 do_div(throttled_usec, NSEC_PER_USEC); 8247 8248 seq_printf(sf, "nr_periods %d\n" 8249 "nr_throttled %d\n" 8250 "throttled_usec %llu\n", 8251 cfs_b->nr_periods, cfs_b->nr_throttled, 8252 throttled_usec); 8253 } 8254 #endif 8255 return 0; 8256 } 8257 8258 #ifdef CONFIG_FAIR_GROUP_SCHED 8259 static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css, 8260 struct cftype *cft) 8261 { 8262 struct task_group *tg = css_tg(css); 8263 u64 weight = scale_load_down(tg->shares); 8264 8265 return DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024); 8266 } 8267 8268 static int cpu_weight_write_u64(struct cgroup_subsys_state *css, 8269 struct cftype *cft, u64 weight) 8270 { 8271 /* 8272 * cgroup weight knobs should use the common MIN, DFL and MAX 8273 * values which are 1, 100 and 10000 respectively. While it loses 8274 * a bit of range on both ends, it maps pretty well onto the shares 8275 * value used by scheduler and the round-trip conversions preserve 8276 * the original value over the entire range. 8277 */ 8278 if (weight < CGROUP_WEIGHT_MIN || weight > CGROUP_WEIGHT_MAX) 8279 return -ERANGE; 8280 8281 weight = DIV_ROUND_CLOSEST_ULL(weight * 1024, CGROUP_WEIGHT_DFL); 8282 8283 return sched_group_set_shares(css_tg(css), scale_load(weight)); 8284 } 8285 8286 static s64 cpu_weight_nice_read_s64(struct cgroup_subsys_state *css, 8287 struct cftype *cft) 8288 { 8289 unsigned long weight = scale_load_down(css_tg(css)->shares); 8290 int last_delta = INT_MAX; 8291 int prio, delta; 8292 8293 /* find the closest nice value to the current weight */ 8294 for (prio = 0; prio < ARRAY_SIZE(sched_prio_to_weight); prio++) { 8295 delta = abs(sched_prio_to_weight[prio] - weight); 8296 if (delta >= last_delta) 8297 break; 8298 last_delta = delta; 8299 } 8300 8301 return PRIO_TO_NICE(prio - 1 + MAX_RT_PRIO); 8302 } 8303 8304 static int cpu_weight_nice_write_s64(struct cgroup_subsys_state *css, 8305 struct cftype *cft, s64 nice) 8306 { 8307 unsigned long weight; 8308 int idx; 8309 8310 if (nice < MIN_NICE || nice > MAX_NICE) 8311 return -ERANGE; 8312 8313 idx = NICE_TO_PRIO(nice) - MAX_RT_PRIO; 8314 idx = array_index_nospec(idx, 40); 8315 weight = sched_prio_to_weight[idx]; 8316 8317 return sched_group_set_shares(css_tg(css), scale_load(weight)); 8318 } 8319 #endif 8320 8321 static void __maybe_unused cpu_period_quota_print(struct seq_file *sf, 8322 long period, long quota) 8323 { 8324 if (quota < 0) 8325 seq_puts(sf, "max"); 8326 else 8327 seq_printf(sf, "%ld", quota); 8328 8329 seq_printf(sf, " %ld\n", period); 8330 } 8331 8332 /* caller should put the current value in *@periodp before calling */ 8333 static int __maybe_unused cpu_period_quota_parse(char *buf, 8334 u64 *periodp, u64 *quotap) 8335 { 8336 char tok[21]; /* U64_MAX */ 8337 8338 if (sscanf(buf, "%20s %llu", tok, periodp) < 1) 8339 return -EINVAL; 8340 8341 *periodp *= NSEC_PER_USEC; 8342 8343 if (sscanf(tok, "%llu", quotap)) 8344 *quotap *= NSEC_PER_USEC; 8345 else if (!strcmp(tok, "max")) 8346 *quotap = RUNTIME_INF; 8347 else 8348 return -EINVAL; 8349 8350 return 0; 8351 } 8352 8353 #ifdef CONFIG_CFS_BANDWIDTH 8354 static int cpu_max_show(struct seq_file *sf, void *v) 8355 { 8356 struct task_group *tg = css_tg(seq_css(sf)); 8357 8358 cpu_period_quota_print(sf, tg_get_cfs_period(tg), tg_get_cfs_quota(tg)); 8359 return 0; 8360 } 8361 8362 static ssize_t cpu_max_write(struct kernfs_open_file *of, 8363 char *buf, size_t nbytes, loff_t off) 8364 { 8365 struct task_group *tg = css_tg(of_css(of)); 8366 u64 period = tg_get_cfs_period(tg); 8367 u64 quota; 8368 int ret; 8369 8370 ret = cpu_period_quota_parse(buf, &period, "a); 8371 if (!ret) 8372 ret = tg_set_cfs_bandwidth(tg, period, quota); 8373 return ret ?: nbytes; 8374 } 8375 #endif 8376 8377 static struct cftype cpu_files[] = { 8378 #ifdef CONFIG_FAIR_GROUP_SCHED 8379 { 8380 .name = "weight", 8381 .flags = CFTYPE_NOT_ON_ROOT, 8382 .read_u64 = cpu_weight_read_u64, 8383 .write_u64 = cpu_weight_write_u64, 8384 }, 8385 { 8386 .name = "weight.nice", 8387 .flags = CFTYPE_NOT_ON_ROOT, 8388 .read_s64 = cpu_weight_nice_read_s64, 8389 .write_s64 = cpu_weight_nice_write_s64, 8390 }, 8391 #endif 8392 #ifdef CONFIG_CFS_BANDWIDTH 8393 { 8394 .name = "max", 8395 .flags = CFTYPE_NOT_ON_ROOT, 8396 .seq_show = cpu_max_show, 8397 .write = cpu_max_write, 8398 }, 8399 #endif 8400 #ifdef CONFIG_UCLAMP_TASK_GROUP 8401 { 8402 .name = "uclamp.min", 8403 .flags = CFTYPE_NOT_ON_ROOT, 8404 .seq_show = cpu_uclamp_min_show, 8405 .write = cpu_uclamp_min_write, 8406 }, 8407 { 8408 .name = "uclamp.max", 8409 .flags = CFTYPE_NOT_ON_ROOT, 8410 .seq_show = cpu_uclamp_max_show, 8411 .write = cpu_uclamp_max_write, 8412 }, 8413 #endif 8414 { } /* terminate */ 8415 }; 8416 8417 struct cgroup_subsys cpu_cgrp_subsys = { 8418 .css_alloc = cpu_cgroup_css_alloc, 8419 .css_online = cpu_cgroup_css_online, 8420 .css_released = cpu_cgroup_css_released, 8421 .css_free = cpu_cgroup_css_free, 8422 .css_extra_stat_show = cpu_extra_stat_show, 8423 .fork = cpu_cgroup_fork, 8424 .can_attach = cpu_cgroup_can_attach, 8425 .attach = cpu_cgroup_attach, 8426 .legacy_cftypes = cpu_legacy_files, 8427 .dfl_cftypes = cpu_files, 8428 .early_init = true, 8429 .threaded = true, 8430 }; 8431 8432 #endif /* CONFIG_CGROUP_SCHED */ 8433 8434 void dump_cpu_task(int cpu) 8435 { 8436 pr_info("Task dump for CPU %d:\n", cpu); 8437 sched_show_task(cpu_curr(cpu)); 8438 } 8439 8440 /* 8441 * Nice levels are multiplicative, with a gentle 10% change for every 8442 * nice level changed. I.e. when a CPU-bound task goes from nice 0 to 8443 * nice 1, it will get ~10% less CPU time than another CPU-bound task 8444 * that remained on nice 0. 8445 * 8446 * The "10% effect" is relative and cumulative: from _any_ nice level, 8447 * if you go up 1 level, it's -10% CPU usage, if you go down 1 level 8448 * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25. 8449 * If a task goes up by ~10% and another task goes down by ~10% then 8450 * the relative distance between them is ~25%.) 8451 */ 8452 const int sched_prio_to_weight[40] = { 8453 /* -20 */ 88761, 71755, 56483, 46273, 36291, 8454 /* -15 */ 29154, 23254, 18705, 14949, 11916, 8455 /* -10 */ 9548, 7620, 6100, 4904, 3906, 8456 /* -5 */ 3121, 2501, 1991, 1586, 1277, 8457 /* 0 */ 1024, 820, 655, 526, 423, 8458 /* 5 */ 335, 272, 215, 172, 137, 8459 /* 10 */ 110, 87, 70, 56, 45, 8460 /* 15 */ 36, 29, 23, 18, 15, 8461 }; 8462 8463 /* 8464 * Inverse (2^32/x) values of the sched_prio_to_weight[] array, precalculated. 8465 * 8466 * In cases where the weight does not change often, we can use the 8467 * precalculated inverse to speed up arithmetics by turning divisions 8468 * into multiplications: 8469 */ 8470 const u32 sched_prio_to_wmult[40] = { 8471 /* -20 */ 48388, 59856, 76040, 92818, 118348, 8472 /* -15 */ 147320, 184698, 229616, 287308, 360437, 8473 /* -10 */ 449829, 563644, 704093, 875809, 1099582, 8474 /* -5 */ 1376151, 1717300, 2157191, 2708050, 3363326, 8475 /* 0 */ 4194304, 5237765, 6557202, 8165337, 10153587, 8476 /* 5 */ 12820798, 15790321, 19976592, 24970740, 31350126, 8477 /* 10 */ 39045157, 49367440, 61356676, 76695844, 95443717, 8478 /* 15 */ 119304647, 148102320, 186737708, 238609294, 286331153, 8479 }; 8480 8481 void call_trace_sched_update_nr_running(struct rq *rq, int count) 8482 { 8483 trace_sched_update_nr_running_tp(rq, count); 8484 } 8485