1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * kernel/sched/core.c 4 * 5 * Core kernel scheduler code and related syscalls 6 * 7 * Copyright (C) 1991-2002 Linus Torvalds 8 */ 9 #include "sched.h" 10 11 #include <linux/nospec.h> 12 13 #include <linux/kcov.h> 14 15 #include <asm/switch_to.h> 16 #include <asm/tlb.h> 17 18 #include "../workqueue_internal.h" 19 #include "../smpboot.h" 20 21 #include "pelt.h" 22 23 #define CREATE_TRACE_POINTS 24 #include <trace/events/sched.h> 25 26 /* 27 * Export tracepoints that act as a bare tracehook (ie: have no trace event 28 * associated with them) to allow external modules to probe them. 29 */ 30 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_cfs_tp); 31 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_rt_tp); 32 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_dl_tp); 33 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_irq_tp); 34 EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_se_tp); 35 EXPORT_TRACEPOINT_SYMBOL_GPL(sched_overutilized_tp); 36 37 DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); 38 39 #if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_JUMP_LABEL) 40 /* 41 * Debugging: various feature bits 42 * 43 * If SCHED_DEBUG is disabled, each compilation unit has its own copy of 44 * sysctl_sched_features, defined in sched.h, to allow constants propagation 45 * at compile time and compiler optimization based on features default. 46 */ 47 #define SCHED_FEAT(name, enabled) \ 48 (1UL << __SCHED_FEAT_##name) * enabled | 49 const_debug unsigned int sysctl_sched_features = 50 #include "features.h" 51 0; 52 #undef SCHED_FEAT 53 #endif 54 55 /* 56 * Number of tasks to iterate in a single balance run. 57 * Limited because this is done with IRQs disabled. 58 */ 59 const_debug unsigned int sysctl_sched_nr_migrate = 32; 60 61 /* 62 * period over which we measure -rt task CPU usage in us. 63 * default: 1s 64 */ 65 unsigned int sysctl_sched_rt_period = 1000000; 66 67 __read_mostly int scheduler_running; 68 69 /* 70 * part of the period that we allow rt tasks to run in us. 71 * default: 0.95s 72 */ 73 int sysctl_sched_rt_runtime = 950000; 74 75 /* 76 * __task_rq_lock - lock the rq @p resides on. 77 */ 78 struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf) 79 __acquires(rq->lock) 80 { 81 struct rq *rq; 82 83 lockdep_assert_held(&p->pi_lock); 84 85 for (;;) { 86 rq = task_rq(p); 87 raw_spin_lock(&rq->lock); 88 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) { 89 rq_pin_lock(rq, rf); 90 return rq; 91 } 92 raw_spin_unlock(&rq->lock); 93 94 while (unlikely(task_on_rq_migrating(p))) 95 cpu_relax(); 96 } 97 } 98 99 /* 100 * task_rq_lock - lock p->pi_lock and lock the rq @p resides on. 101 */ 102 struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf) 103 __acquires(p->pi_lock) 104 __acquires(rq->lock) 105 { 106 struct rq *rq; 107 108 for (;;) { 109 raw_spin_lock_irqsave(&p->pi_lock, rf->flags); 110 rq = task_rq(p); 111 raw_spin_lock(&rq->lock); 112 /* 113 * move_queued_task() task_rq_lock() 114 * 115 * ACQUIRE (rq->lock) 116 * [S] ->on_rq = MIGRATING [L] rq = task_rq() 117 * WMB (__set_task_cpu()) ACQUIRE (rq->lock); 118 * [S] ->cpu = new_cpu [L] task_rq() 119 * [L] ->on_rq 120 * RELEASE (rq->lock) 121 * 122 * If we observe the old CPU in task_rq_lock(), the acquire of 123 * the old rq->lock will fully serialize against the stores. 124 * 125 * If we observe the new CPU in task_rq_lock(), the address 126 * dependency headed by '[L] rq = task_rq()' and the acquire 127 * will pair with the WMB to ensure we then also see migrating. 128 */ 129 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) { 130 rq_pin_lock(rq, rf); 131 return rq; 132 } 133 raw_spin_unlock(&rq->lock); 134 raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags); 135 136 while (unlikely(task_on_rq_migrating(p))) 137 cpu_relax(); 138 } 139 } 140 141 /* 142 * RQ-clock updating methods: 143 */ 144 145 static void update_rq_clock_task(struct rq *rq, s64 delta) 146 { 147 /* 148 * In theory, the compile should just see 0 here, and optimize out the call 149 * to sched_rt_avg_update. But I don't trust it... 150 */ 151 s64 __maybe_unused steal = 0, irq_delta = 0; 152 153 #ifdef CONFIG_IRQ_TIME_ACCOUNTING 154 irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time; 155 156 /* 157 * Since irq_time is only updated on {soft,}irq_exit, we might run into 158 * this case when a previous update_rq_clock() happened inside a 159 * {soft,}irq region. 160 * 161 * When this happens, we stop ->clock_task and only update the 162 * prev_irq_time stamp to account for the part that fit, so that a next 163 * update will consume the rest. This ensures ->clock_task is 164 * monotonic. 165 * 166 * It does however cause some slight miss-attribution of {soft,}irq 167 * time, a more accurate solution would be to update the irq_time using 168 * the current rq->clock timestamp, except that would require using 169 * atomic ops. 170 */ 171 if (irq_delta > delta) 172 irq_delta = delta; 173 174 rq->prev_irq_time += irq_delta; 175 delta -= irq_delta; 176 #endif 177 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING 178 if (static_key_false((¶virt_steal_rq_enabled))) { 179 steal = paravirt_steal_clock(cpu_of(rq)); 180 steal -= rq->prev_steal_time_rq; 181 182 if (unlikely(steal > delta)) 183 steal = delta; 184 185 rq->prev_steal_time_rq += steal; 186 delta -= steal; 187 } 188 #endif 189 190 rq->clock_task += delta; 191 192 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ 193 if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY)) 194 update_irq_load_avg(rq, irq_delta + steal); 195 #endif 196 update_rq_clock_pelt(rq, delta); 197 } 198 199 void update_rq_clock(struct rq *rq) 200 { 201 s64 delta; 202 203 lockdep_assert_held(&rq->lock); 204 205 if (rq->clock_update_flags & RQCF_ACT_SKIP) 206 return; 207 208 #ifdef CONFIG_SCHED_DEBUG 209 if (sched_feat(WARN_DOUBLE_CLOCK)) 210 SCHED_WARN_ON(rq->clock_update_flags & RQCF_UPDATED); 211 rq->clock_update_flags |= RQCF_UPDATED; 212 #endif 213 214 delta = sched_clock_cpu(cpu_of(rq)) - rq->clock; 215 if (delta < 0) 216 return; 217 rq->clock += delta; 218 update_rq_clock_task(rq, delta); 219 } 220 221 222 #ifdef CONFIG_SCHED_HRTICK 223 /* 224 * Use HR-timers to deliver accurate preemption points. 225 */ 226 227 static void hrtick_clear(struct rq *rq) 228 { 229 if (hrtimer_active(&rq->hrtick_timer)) 230 hrtimer_cancel(&rq->hrtick_timer); 231 } 232 233 /* 234 * High-resolution timer tick. 235 * Runs from hardirq context with interrupts disabled. 236 */ 237 static enum hrtimer_restart hrtick(struct hrtimer *timer) 238 { 239 struct rq *rq = container_of(timer, struct rq, hrtick_timer); 240 struct rq_flags rf; 241 242 WARN_ON_ONCE(cpu_of(rq) != smp_processor_id()); 243 244 rq_lock(rq, &rf); 245 update_rq_clock(rq); 246 rq->curr->sched_class->task_tick(rq, rq->curr, 1); 247 rq_unlock(rq, &rf); 248 249 return HRTIMER_NORESTART; 250 } 251 252 #ifdef CONFIG_SMP 253 254 static void __hrtick_restart(struct rq *rq) 255 { 256 struct hrtimer *timer = &rq->hrtick_timer; 257 258 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED_HARD); 259 } 260 261 /* 262 * called from hardirq (IPI) context 263 */ 264 static void __hrtick_start(void *arg) 265 { 266 struct rq *rq = arg; 267 struct rq_flags rf; 268 269 rq_lock(rq, &rf); 270 __hrtick_restart(rq); 271 rq->hrtick_csd_pending = 0; 272 rq_unlock(rq, &rf); 273 } 274 275 /* 276 * Called to set the hrtick timer state. 277 * 278 * called with rq->lock held and irqs disabled 279 */ 280 void hrtick_start(struct rq *rq, u64 delay) 281 { 282 struct hrtimer *timer = &rq->hrtick_timer; 283 ktime_t time; 284 s64 delta; 285 286 /* 287 * Don't schedule slices shorter than 10000ns, that just 288 * doesn't make sense and can cause timer DoS. 289 */ 290 delta = max_t(s64, delay, 10000LL); 291 time = ktime_add_ns(timer->base->get_time(), delta); 292 293 hrtimer_set_expires(timer, time); 294 295 if (rq == this_rq()) { 296 __hrtick_restart(rq); 297 } else if (!rq->hrtick_csd_pending) { 298 smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd); 299 rq->hrtick_csd_pending = 1; 300 } 301 } 302 303 #else 304 /* 305 * Called to set the hrtick timer state. 306 * 307 * called with rq->lock held and irqs disabled 308 */ 309 void hrtick_start(struct rq *rq, u64 delay) 310 { 311 /* 312 * Don't schedule slices shorter than 10000ns, that just 313 * doesn't make sense. Rely on vruntime for fairness. 314 */ 315 delay = max_t(u64, delay, 10000LL); 316 hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay), 317 HRTIMER_MODE_REL_PINNED_HARD); 318 } 319 #endif /* CONFIG_SMP */ 320 321 static void hrtick_rq_init(struct rq *rq) 322 { 323 #ifdef CONFIG_SMP 324 rq->hrtick_csd_pending = 0; 325 326 rq->hrtick_csd.flags = 0; 327 rq->hrtick_csd.func = __hrtick_start; 328 rq->hrtick_csd.info = rq; 329 #endif 330 331 hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); 332 rq->hrtick_timer.function = hrtick; 333 } 334 #else /* CONFIG_SCHED_HRTICK */ 335 static inline void hrtick_clear(struct rq *rq) 336 { 337 } 338 339 static inline void hrtick_rq_init(struct rq *rq) 340 { 341 } 342 #endif /* CONFIG_SCHED_HRTICK */ 343 344 /* 345 * cmpxchg based fetch_or, macro so it works for different integer types 346 */ 347 #define fetch_or(ptr, mask) \ 348 ({ \ 349 typeof(ptr) _ptr = (ptr); \ 350 typeof(mask) _mask = (mask); \ 351 typeof(*_ptr) _old, _val = *_ptr; \ 352 \ 353 for (;;) { \ 354 _old = cmpxchg(_ptr, _val, _val | _mask); \ 355 if (_old == _val) \ 356 break; \ 357 _val = _old; \ 358 } \ 359 _old; \ 360 }) 361 362 #if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG) 363 /* 364 * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG, 365 * this avoids any races wrt polling state changes and thereby avoids 366 * spurious IPIs. 367 */ 368 static bool set_nr_and_not_polling(struct task_struct *p) 369 { 370 struct thread_info *ti = task_thread_info(p); 371 return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG); 372 } 373 374 /* 375 * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set. 376 * 377 * If this returns true, then the idle task promises to call 378 * sched_ttwu_pending() and reschedule soon. 379 */ 380 static bool set_nr_if_polling(struct task_struct *p) 381 { 382 struct thread_info *ti = task_thread_info(p); 383 typeof(ti->flags) old, val = READ_ONCE(ti->flags); 384 385 for (;;) { 386 if (!(val & _TIF_POLLING_NRFLAG)) 387 return false; 388 if (val & _TIF_NEED_RESCHED) 389 return true; 390 old = cmpxchg(&ti->flags, val, val | _TIF_NEED_RESCHED); 391 if (old == val) 392 break; 393 val = old; 394 } 395 return true; 396 } 397 398 #else 399 static bool set_nr_and_not_polling(struct task_struct *p) 400 { 401 set_tsk_need_resched(p); 402 return true; 403 } 404 405 #ifdef CONFIG_SMP 406 static bool set_nr_if_polling(struct task_struct *p) 407 { 408 return false; 409 } 410 #endif 411 #endif 412 413 static bool __wake_q_add(struct wake_q_head *head, struct task_struct *task) 414 { 415 struct wake_q_node *node = &task->wake_q; 416 417 /* 418 * Atomically grab the task, if ->wake_q is !nil already it means 419 * its already queued (either by us or someone else) and will get the 420 * wakeup due to that. 421 * 422 * In order to ensure that a pending wakeup will observe our pending 423 * state, even in the failed case, an explicit smp_mb() must be used. 424 */ 425 smp_mb__before_atomic(); 426 if (unlikely(cmpxchg_relaxed(&node->next, NULL, WAKE_Q_TAIL))) 427 return false; 428 429 /* 430 * The head is context local, there can be no concurrency. 431 */ 432 *head->lastp = node; 433 head->lastp = &node->next; 434 return true; 435 } 436 437 /** 438 * wake_q_add() - queue a wakeup for 'later' waking. 439 * @head: the wake_q_head to add @task to 440 * @task: the task to queue for 'later' wakeup 441 * 442 * Queue a task for later wakeup, most likely by the wake_up_q() call in the 443 * same context, _HOWEVER_ this is not guaranteed, the wakeup can come 444 * instantly. 445 * 446 * This function must be used as-if it were wake_up_process(); IOW the task 447 * must be ready to be woken at this location. 448 */ 449 void wake_q_add(struct wake_q_head *head, struct task_struct *task) 450 { 451 if (__wake_q_add(head, task)) 452 get_task_struct(task); 453 } 454 455 /** 456 * wake_q_add_safe() - safely queue a wakeup for 'later' waking. 457 * @head: the wake_q_head to add @task to 458 * @task: the task to queue for 'later' wakeup 459 * 460 * Queue a task for later wakeup, most likely by the wake_up_q() call in the 461 * same context, _HOWEVER_ this is not guaranteed, the wakeup can come 462 * instantly. 463 * 464 * This function must be used as-if it were wake_up_process(); IOW the task 465 * must be ready to be woken at this location. 466 * 467 * This function is essentially a task-safe equivalent to wake_q_add(). Callers 468 * that already hold reference to @task can call the 'safe' version and trust 469 * wake_q to do the right thing depending whether or not the @task is already 470 * queued for wakeup. 471 */ 472 void wake_q_add_safe(struct wake_q_head *head, struct task_struct *task) 473 { 474 if (!__wake_q_add(head, task)) 475 put_task_struct(task); 476 } 477 478 void wake_up_q(struct wake_q_head *head) 479 { 480 struct wake_q_node *node = head->first; 481 482 while (node != WAKE_Q_TAIL) { 483 struct task_struct *task; 484 485 task = container_of(node, struct task_struct, wake_q); 486 BUG_ON(!task); 487 /* Task can safely be re-inserted now: */ 488 node = node->next; 489 task->wake_q.next = NULL; 490 491 /* 492 * wake_up_process() executes a full barrier, which pairs with 493 * the queueing in wake_q_add() so as not to miss wakeups. 494 */ 495 wake_up_process(task); 496 put_task_struct(task); 497 } 498 } 499 500 /* 501 * resched_curr - mark rq's current task 'to be rescheduled now'. 502 * 503 * On UP this means the setting of the need_resched flag, on SMP it 504 * might also involve a cross-CPU call to trigger the scheduler on 505 * the target CPU. 506 */ 507 void resched_curr(struct rq *rq) 508 { 509 struct task_struct *curr = rq->curr; 510 int cpu; 511 512 lockdep_assert_held(&rq->lock); 513 514 if (test_tsk_need_resched(curr)) 515 return; 516 517 cpu = cpu_of(rq); 518 519 if (cpu == smp_processor_id()) { 520 set_tsk_need_resched(curr); 521 set_preempt_need_resched(); 522 return; 523 } 524 525 if (set_nr_and_not_polling(curr)) 526 smp_send_reschedule(cpu); 527 else 528 trace_sched_wake_idle_without_ipi(cpu); 529 } 530 531 void resched_cpu(int cpu) 532 { 533 struct rq *rq = cpu_rq(cpu); 534 unsigned long flags; 535 536 raw_spin_lock_irqsave(&rq->lock, flags); 537 if (cpu_online(cpu) || cpu == smp_processor_id()) 538 resched_curr(rq); 539 raw_spin_unlock_irqrestore(&rq->lock, flags); 540 } 541 542 #ifdef CONFIG_SMP 543 #ifdef CONFIG_NO_HZ_COMMON 544 /* 545 * In the semi idle case, use the nearest busy CPU for migrating timers 546 * from an idle CPU. This is good for power-savings. 547 * 548 * We don't do similar optimization for completely idle system, as 549 * selecting an idle CPU will add more delays to the timers than intended 550 * (as that CPU's timer base may not be uptodate wrt jiffies etc). 551 */ 552 int get_nohz_timer_target(void) 553 { 554 int i, cpu = smp_processor_id(); 555 struct sched_domain *sd; 556 557 if (!idle_cpu(cpu) && housekeeping_cpu(cpu, HK_FLAG_TIMER)) 558 return cpu; 559 560 rcu_read_lock(); 561 for_each_domain(cpu, sd) { 562 for_each_cpu(i, sched_domain_span(sd)) { 563 if (cpu == i) 564 continue; 565 566 if (!idle_cpu(i) && housekeeping_cpu(i, HK_FLAG_TIMER)) { 567 cpu = i; 568 goto unlock; 569 } 570 } 571 } 572 573 if (!housekeeping_cpu(cpu, HK_FLAG_TIMER)) 574 cpu = housekeeping_any_cpu(HK_FLAG_TIMER); 575 unlock: 576 rcu_read_unlock(); 577 return cpu; 578 } 579 580 /* 581 * When add_timer_on() enqueues a timer into the timer wheel of an 582 * idle CPU then this timer might expire before the next timer event 583 * which is scheduled to wake up that CPU. In case of a completely 584 * idle system the next event might even be infinite time into the 585 * future. wake_up_idle_cpu() ensures that the CPU is woken up and 586 * leaves the inner idle loop so the newly added timer is taken into 587 * account when the CPU goes back to idle and evaluates the timer 588 * wheel for the next timer event. 589 */ 590 static void wake_up_idle_cpu(int cpu) 591 { 592 struct rq *rq = cpu_rq(cpu); 593 594 if (cpu == smp_processor_id()) 595 return; 596 597 if (set_nr_and_not_polling(rq->idle)) 598 smp_send_reschedule(cpu); 599 else 600 trace_sched_wake_idle_without_ipi(cpu); 601 } 602 603 static bool wake_up_full_nohz_cpu(int cpu) 604 { 605 /* 606 * We just need the target to call irq_exit() and re-evaluate 607 * the next tick. The nohz full kick at least implies that. 608 * If needed we can still optimize that later with an 609 * empty IRQ. 610 */ 611 if (cpu_is_offline(cpu)) 612 return true; /* Don't try to wake offline CPUs. */ 613 if (tick_nohz_full_cpu(cpu)) { 614 if (cpu != smp_processor_id() || 615 tick_nohz_tick_stopped()) 616 tick_nohz_full_kick_cpu(cpu); 617 return true; 618 } 619 620 return false; 621 } 622 623 /* 624 * Wake up the specified CPU. If the CPU is going offline, it is the 625 * caller's responsibility to deal with the lost wakeup, for example, 626 * by hooking into the CPU_DEAD notifier like timers and hrtimers do. 627 */ 628 void wake_up_nohz_cpu(int cpu) 629 { 630 if (!wake_up_full_nohz_cpu(cpu)) 631 wake_up_idle_cpu(cpu); 632 } 633 634 static inline bool got_nohz_idle_kick(void) 635 { 636 int cpu = smp_processor_id(); 637 638 if (!(atomic_read(nohz_flags(cpu)) & NOHZ_KICK_MASK)) 639 return false; 640 641 if (idle_cpu(cpu) && !need_resched()) 642 return true; 643 644 /* 645 * We can't run Idle Load Balance on this CPU for this time so we 646 * cancel it and clear NOHZ_BALANCE_KICK 647 */ 648 atomic_andnot(NOHZ_KICK_MASK, nohz_flags(cpu)); 649 return false; 650 } 651 652 #else /* CONFIG_NO_HZ_COMMON */ 653 654 static inline bool got_nohz_idle_kick(void) 655 { 656 return false; 657 } 658 659 #endif /* CONFIG_NO_HZ_COMMON */ 660 661 #ifdef CONFIG_NO_HZ_FULL 662 bool sched_can_stop_tick(struct rq *rq) 663 { 664 int fifo_nr_running; 665 666 /* Deadline tasks, even if single, need the tick */ 667 if (rq->dl.dl_nr_running) 668 return false; 669 670 /* 671 * If there are more than one RR tasks, we need the tick to effect the 672 * actual RR behaviour. 673 */ 674 if (rq->rt.rr_nr_running) { 675 if (rq->rt.rr_nr_running == 1) 676 return true; 677 else 678 return false; 679 } 680 681 /* 682 * If there's no RR tasks, but FIFO tasks, we can skip the tick, no 683 * forced preemption between FIFO tasks. 684 */ 685 fifo_nr_running = rq->rt.rt_nr_running - rq->rt.rr_nr_running; 686 if (fifo_nr_running) 687 return true; 688 689 /* 690 * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left; 691 * if there's more than one we need the tick for involuntary 692 * preemption. 693 */ 694 if (rq->nr_running > 1) 695 return false; 696 697 return true; 698 } 699 #endif /* CONFIG_NO_HZ_FULL */ 700 #endif /* CONFIG_SMP */ 701 702 #if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \ 703 (defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH))) 704 /* 705 * Iterate task_group tree rooted at *from, calling @down when first entering a 706 * node and @up when leaving it for the final time. 707 * 708 * Caller must hold rcu_lock or sufficient equivalent. 709 */ 710 int walk_tg_tree_from(struct task_group *from, 711 tg_visitor down, tg_visitor up, void *data) 712 { 713 struct task_group *parent, *child; 714 int ret; 715 716 parent = from; 717 718 down: 719 ret = (*down)(parent, data); 720 if (ret) 721 goto out; 722 list_for_each_entry_rcu(child, &parent->children, siblings) { 723 parent = child; 724 goto down; 725 726 up: 727 continue; 728 } 729 ret = (*up)(parent, data); 730 if (ret || parent == from) 731 goto out; 732 733 child = parent; 734 parent = parent->parent; 735 if (parent) 736 goto up; 737 out: 738 return ret; 739 } 740 741 int tg_nop(struct task_group *tg, void *data) 742 { 743 return 0; 744 } 745 #endif 746 747 static void set_load_weight(struct task_struct *p, bool update_load) 748 { 749 int prio = p->static_prio - MAX_RT_PRIO; 750 struct load_weight *load = &p->se.load; 751 752 /* 753 * SCHED_IDLE tasks get minimal weight: 754 */ 755 if (task_has_idle_policy(p)) { 756 load->weight = scale_load(WEIGHT_IDLEPRIO); 757 load->inv_weight = WMULT_IDLEPRIO; 758 p->se.runnable_weight = load->weight; 759 return; 760 } 761 762 /* 763 * SCHED_OTHER tasks have to update their load when changing their 764 * weight 765 */ 766 if (update_load && p->sched_class == &fair_sched_class) { 767 reweight_task(p, prio); 768 } else { 769 load->weight = scale_load(sched_prio_to_weight[prio]); 770 load->inv_weight = sched_prio_to_wmult[prio]; 771 p->se.runnable_weight = load->weight; 772 } 773 } 774 775 #ifdef CONFIG_UCLAMP_TASK 776 /* 777 * Serializes updates of utilization clamp values 778 * 779 * The (slow-path) user-space triggers utilization clamp value updates which 780 * can require updates on (fast-path) scheduler's data structures used to 781 * support enqueue/dequeue operations. 782 * While the per-CPU rq lock protects fast-path update operations, user-space 783 * requests are serialized using a mutex to reduce the risk of conflicting 784 * updates or API abuses. 785 */ 786 static DEFINE_MUTEX(uclamp_mutex); 787 788 /* Max allowed minimum utilization */ 789 unsigned int sysctl_sched_uclamp_util_min = SCHED_CAPACITY_SCALE; 790 791 /* Max allowed maximum utilization */ 792 unsigned int sysctl_sched_uclamp_util_max = SCHED_CAPACITY_SCALE; 793 794 /* All clamps are required to be less or equal than these values */ 795 static struct uclamp_se uclamp_default[UCLAMP_CNT]; 796 797 /* Integer rounded range for each bucket */ 798 #define UCLAMP_BUCKET_DELTA DIV_ROUND_CLOSEST(SCHED_CAPACITY_SCALE, UCLAMP_BUCKETS) 799 800 #define for_each_clamp_id(clamp_id) \ 801 for ((clamp_id) = 0; (clamp_id) < UCLAMP_CNT; (clamp_id)++) 802 803 static inline unsigned int uclamp_bucket_id(unsigned int clamp_value) 804 { 805 return clamp_value / UCLAMP_BUCKET_DELTA; 806 } 807 808 static inline unsigned int uclamp_bucket_base_value(unsigned int clamp_value) 809 { 810 return UCLAMP_BUCKET_DELTA * uclamp_bucket_id(clamp_value); 811 } 812 813 static inline enum uclamp_id uclamp_none(enum uclamp_id clamp_id) 814 { 815 if (clamp_id == UCLAMP_MIN) 816 return 0; 817 return SCHED_CAPACITY_SCALE; 818 } 819 820 static inline void uclamp_se_set(struct uclamp_se *uc_se, 821 unsigned int value, bool user_defined) 822 { 823 uc_se->value = value; 824 uc_se->bucket_id = uclamp_bucket_id(value); 825 uc_se->user_defined = user_defined; 826 } 827 828 static inline unsigned int 829 uclamp_idle_value(struct rq *rq, enum uclamp_id clamp_id, 830 unsigned int clamp_value) 831 { 832 /* 833 * Avoid blocked utilization pushing up the frequency when we go 834 * idle (which drops the max-clamp) by retaining the last known 835 * max-clamp. 836 */ 837 if (clamp_id == UCLAMP_MAX) { 838 rq->uclamp_flags |= UCLAMP_FLAG_IDLE; 839 return clamp_value; 840 } 841 842 return uclamp_none(UCLAMP_MIN); 843 } 844 845 static inline void uclamp_idle_reset(struct rq *rq, enum uclamp_id clamp_id, 846 unsigned int clamp_value) 847 { 848 /* Reset max-clamp retention only on idle exit */ 849 if (!(rq->uclamp_flags & UCLAMP_FLAG_IDLE)) 850 return; 851 852 WRITE_ONCE(rq->uclamp[clamp_id].value, clamp_value); 853 } 854 855 static inline 856 enum uclamp_id uclamp_rq_max_value(struct rq *rq, enum uclamp_id clamp_id, 857 unsigned int clamp_value) 858 { 859 struct uclamp_bucket *bucket = rq->uclamp[clamp_id].bucket; 860 int bucket_id = UCLAMP_BUCKETS - 1; 861 862 /* 863 * Since both min and max clamps are max aggregated, find the 864 * top most bucket with tasks in. 865 */ 866 for ( ; bucket_id >= 0; bucket_id--) { 867 if (!bucket[bucket_id].tasks) 868 continue; 869 return bucket[bucket_id].value; 870 } 871 872 /* No tasks -- default clamp values */ 873 return uclamp_idle_value(rq, clamp_id, clamp_value); 874 } 875 876 static inline struct uclamp_se 877 uclamp_tg_restrict(struct task_struct *p, enum uclamp_id clamp_id) 878 { 879 struct uclamp_se uc_req = p->uclamp_req[clamp_id]; 880 #ifdef CONFIG_UCLAMP_TASK_GROUP 881 struct uclamp_se uc_max; 882 883 /* 884 * Tasks in autogroups or root task group will be 885 * restricted by system defaults. 886 */ 887 if (task_group_is_autogroup(task_group(p))) 888 return uc_req; 889 if (task_group(p) == &root_task_group) 890 return uc_req; 891 892 uc_max = task_group(p)->uclamp[clamp_id]; 893 if (uc_req.value > uc_max.value || !uc_req.user_defined) 894 return uc_max; 895 #endif 896 897 return uc_req; 898 } 899 900 /* 901 * The effective clamp bucket index of a task depends on, by increasing 902 * priority: 903 * - the task specific clamp value, when explicitly requested from userspace 904 * - the task group effective clamp value, for tasks not either in the root 905 * group or in an autogroup 906 * - the system default clamp value, defined by the sysadmin 907 */ 908 static inline struct uclamp_se 909 uclamp_eff_get(struct task_struct *p, enum uclamp_id clamp_id) 910 { 911 struct uclamp_se uc_req = uclamp_tg_restrict(p, clamp_id); 912 struct uclamp_se uc_max = uclamp_default[clamp_id]; 913 914 /* System default restrictions always apply */ 915 if (unlikely(uc_req.value > uc_max.value)) 916 return uc_max; 917 918 return uc_req; 919 } 920 921 enum uclamp_id uclamp_eff_value(struct task_struct *p, enum uclamp_id clamp_id) 922 { 923 struct uclamp_se uc_eff; 924 925 /* Task currently refcounted: use back-annotated (effective) value */ 926 if (p->uclamp[clamp_id].active) 927 return p->uclamp[clamp_id].value; 928 929 uc_eff = uclamp_eff_get(p, clamp_id); 930 931 return uc_eff.value; 932 } 933 934 /* 935 * When a task is enqueued on a rq, the clamp bucket currently defined by the 936 * task's uclamp::bucket_id is refcounted on that rq. This also immediately 937 * updates the rq's clamp value if required. 938 * 939 * Tasks can have a task-specific value requested from user-space, track 940 * within each bucket the maximum value for tasks refcounted in it. 941 * This "local max aggregation" allows to track the exact "requested" value 942 * for each bucket when all its RUNNABLE tasks require the same clamp. 943 */ 944 static inline void uclamp_rq_inc_id(struct rq *rq, struct task_struct *p, 945 enum uclamp_id clamp_id) 946 { 947 struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id]; 948 struct uclamp_se *uc_se = &p->uclamp[clamp_id]; 949 struct uclamp_bucket *bucket; 950 951 lockdep_assert_held(&rq->lock); 952 953 /* Update task effective clamp */ 954 p->uclamp[clamp_id] = uclamp_eff_get(p, clamp_id); 955 956 bucket = &uc_rq->bucket[uc_se->bucket_id]; 957 bucket->tasks++; 958 uc_se->active = true; 959 960 uclamp_idle_reset(rq, clamp_id, uc_se->value); 961 962 /* 963 * Local max aggregation: rq buckets always track the max 964 * "requested" clamp value of its RUNNABLE tasks. 965 */ 966 if (bucket->tasks == 1 || uc_se->value > bucket->value) 967 bucket->value = uc_se->value; 968 969 if (uc_se->value > READ_ONCE(uc_rq->value)) 970 WRITE_ONCE(uc_rq->value, uc_se->value); 971 } 972 973 /* 974 * When a task is dequeued from a rq, the clamp bucket refcounted by the task 975 * is released. If this is the last task reference counting the rq's max 976 * active clamp value, then the rq's clamp value is updated. 977 * 978 * Both refcounted tasks and rq's cached clamp values are expected to be 979 * always valid. If it's detected they are not, as defensive programming, 980 * enforce the expected state and warn. 981 */ 982 static inline void uclamp_rq_dec_id(struct rq *rq, struct task_struct *p, 983 enum uclamp_id clamp_id) 984 { 985 struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id]; 986 struct uclamp_se *uc_se = &p->uclamp[clamp_id]; 987 struct uclamp_bucket *bucket; 988 unsigned int bkt_clamp; 989 unsigned int rq_clamp; 990 991 lockdep_assert_held(&rq->lock); 992 993 bucket = &uc_rq->bucket[uc_se->bucket_id]; 994 SCHED_WARN_ON(!bucket->tasks); 995 if (likely(bucket->tasks)) 996 bucket->tasks--; 997 uc_se->active = false; 998 999 /* 1000 * Keep "local max aggregation" simple and accept to (possibly) 1001 * overboost some RUNNABLE tasks in the same bucket. 1002 * The rq clamp bucket value is reset to its base value whenever 1003 * there are no more RUNNABLE tasks refcounting it. 1004 */ 1005 if (likely(bucket->tasks)) 1006 return; 1007 1008 rq_clamp = READ_ONCE(uc_rq->value); 1009 /* 1010 * Defensive programming: this should never happen. If it happens, 1011 * e.g. due to future modification, warn and fixup the expected value. 1012 */ 1013 SCHED_WARN_ON(bucket->value > rq_clamp); 1014 if (bucket->value >= rq_clamp) { 1015 bkt_clamp = uclamp_rq_max_value(rq, clamp_id, uc_se->value); 1016 WRITE_ONCE(uc_rq->value, bkt_clamp); 1017 } 1018 } 1019 1020 static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p) 1021 { 1022 enum uclamp_id clamp_id; 1023 1024 if (unlikely(!p->sched_class->uclamp_enabled)) 1025 return; 1026 1027 for_each_clamp_id(clamp_id) 1028 uclamp_rq_inc_id(rq, p, clamp_id); 1029 1030 /* Reset clamp idle holding when there is one RUNNABLE task */ 1031 if (rq->uclamp_flags & UCLAMP_FLAG_IDLE) 1032 rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE; 1033 } 1034 1035 static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) 1036 { 1037 enum uclamp_id clamp_id; 1038 1039 if (unlikely(!p->sched_class->uclamp_enabled)) 1040 return; 1041 1042 for_each_clamp_id(clamp_id) 1043 uclamp_rq_dec_id(rq, p, clamp_id); 1044 } 1045 1046 static inline void 1047 uclamp_update_active(struct task_struct *p, enum uclamp_id clamp_id) 1048 { 1049 struct rq_flags rf; 1050 struct rq *rq; 1051 1052 /* 1053 * Lock the task and the rq where the task is (or was) queued. 1054 * 1055 * We might lock the (previous) rq of a !RUNNABLE task, but that's the 1056 * price to pay to safely serialize util_{min,max} updates with 1057 * enqueues, dequeues and migration operations. 1058 * This is the same locking schema used by __set_cpus_allowed_ptr(). 1059 */ 1060 rq = task_rq_lock(p, &rf); 1061 1062 /* 1063 * Setting the clamp bucket is serialized by task_rq_lock(). 1064 * If the task is not yet RUNNABLE and its task_struct is not 1065 * affecting a valid clamp bucket, the next time it's enqueued, 1066 * it will already see the updated clamp bucket value. 1067 */ 1068 if (!p->uclamp[clamp_id].active) { 1069 uclamp_rq_dec_id(rq, p, clamp_id); 1070 uclamp_rq_inc_id(rq, p, clamp_id); 1071 } 1072 1073 task_rq_unlock(rq, p, &rf); 1074 } 1075 1076 static inline void 1077 uclamp_update_active_tasks(struct cgroup_subsys_state *css, 1078 unsigned int clamps) 1079 { 1080 enum uclamp_id clamp_id; 1081 struct css_task_iter it; 1082 struct task_struct *p; 1083 1084 css_task_iter_start(css, 0, &it); 1085 while ((p = css_task_iter_next(&it))) { 1086 for_each_clamp_id(clamp_id) { 1087 if ((0x1 << clamp_id) & clamps) 1088 uclamp_update_active(p, clamp_id); 1089 } 1090 } 1091 css_task_iter_end(&it); 1092 } 1093 1094 #ifdef CONFIG_UCLAMP_TASK_GROUP 1095 static void cpu_util_update_eff(struct cgroup_subsys_state *css); 1096 static void uclamp_update_root_tg(void) 1097 { 1098 struct task_group *tg = &root_task_group; 1099 1100 uclamp_se_set(&tg->uclamp_req[UCLAMP_MIN], 1101 sysctl_sched_uclamp_util_min, false); 1102 uclamp_se_set(&tg->uclamp_req[UCLAMP_MAX], 1103 sysctl_sched_uclamp_util_max, false); 1104 1105 rcu_read_lock(); 1106 cpu_util_update_eff(&root_task_group.css); 1107 rcu_read_unlock(); 1108 } 1109 #else 1110 static void uclamp_update_root_tg(void) { } 1111 #endif 1112 1113 int sysctl_sched_uclamp_handler(struct ctl_table *table, int write, 1114 void __user *buffer, size_t *lenp, 1115 loff_t *ppos) 1116 { 1117 bool update_root_tg = false; 1118 int old_min, old_max; 1119 int result; 1120 1121 mutex_lock(&uclamp_mutex); 1122 old_min = sysctl_sched_uclamp_util_min; 1123 old_max = sysctl_sched_uclamp_util_max; 1124 1125 result = proc_dointvec(table, write, buffer, lenp, ppos); 1126 if (result) 1127 goto undo; 1128 if (!write) 1129 goto done; 1130 1131 if (sysctl_sched_uclamp_util_min > sysctl_sched_uclamp_util_max || 1132 sysctl_sched_uclamp_util_max > SCHED_CAPACITY_SCALE) { 1133 result = -EINVAL; 1134 goto undo; 1135 } 1136 1137 if (old_min != sysctl_sched_uclamp_util_min) { 1138 uclamp_se_set(&uclamp_default[UCLAMP_MIN], 1139 sysctl_sched_uclamp_util_min, false); 1140 update_root_tg = true; 1141 } 1142 if (old_max != sysctl_sched_uclamp_util_max) { 1143 uclamp_se_set(&uclamp_default[UCLAMP_MAX], 1144 sysctl_sched_uclamp_util_max, false); 1145 update_root_tg = true; 1146 } 1147 1148 if (update_root_tg) 1149 uclamp_update_root_tg(); 1150 1151 /* 1152 * We update all RUNNABLE tasks only when task groups are in use. 1153 * Otherwise, keep it simple and do just a lazy update at each next 1154 * task enqueue time. 1155 */ 1156 1157 goto done; 1158 1159 undo: 1160 sysctl_sched_uclamp_util_min = old_min; 1161 sysctl_sched_uclamp_util_max = old_max; 1162 done: 1163 mutex_unlock(&uclamp_mutex); 1164 1165 return result; 1166 } 1167 1168 static int uclamp_validate(struct task_struct *p, 1169 const struct sched_attr *attr) 1170 { 1171 unsigned int lower_bound = p->uclamp_req[UCLAMP_MIN].value; 1172 unsigned int upper_bound = p->uclamp_req[UCLAMP_MAX].value; 1173 1174 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MIN) 1175 lower_bound = attr->sched_util_min; 1176 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MAX) 1177 upper_bound = attr->sched_util_max; 1178 1179 if (lower_bound > upper_bound) 1180 return -EINVAL; 1181 if (upper_bound > SCHED_CAPACITY_SCALE) 1182 return -EINVAL; 1183 1184 return 0; 1185 } 1186 1187 static void __setscheduler_uclamp(struct task_struct *p, 1188 const struct sched_attr *attr) 1189 { 1190 enum uclamp_id clamp_id; 1191 1192 /* 1193 * On scheduling class change, reset to default clamps for tasks 1194 * without a task-specific value. 1195 */ 1196 for_each_clamp_id(clamp_id) { 1197 struct uclamp_se *uc_se = &p->uclamp_req[clamp_id]; 1198 unsigned int clamp_value = uclamp_none(clamp_id); 1199 1200 /* Keep using defined clamps across class changes */ 1201 if (uc_se->user_defined) 1202 continue; 1203 1204 /* By default, RT tasks always get 100% boost */ 1205 if (unlikely(rt_task(p) && clamp_id == UCLAMP_MIN)) 1206 clamp_value = uclamp_none(UCLAMP_MAX); 1207 1208 uclamp_se_set(uc_se, clamp_value, false); 1209 } 1210 1211 if (likely(!(attr->sched_flags & SCHED_FLAG_UTIL_CLAMP))) 1212 return; 1213 1214 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MIN) { 1215 uclamp_se_set(&p->uclamp_req[UCLAMP_MIN], 1216 attr->sched_util_min, true); 1217 } 1218 1219 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MAX) { 1220 uclamp_se_set(&p->uclamp_req[UCLAMP_MAX], 1221 attr->sched_util_max, true); 1222 } 1223 } 1224 1225 static void uclamp_fork(struct task_struct *p) 1226 { 1227 enum uclamp_id clamp_id; 1228 1229 for_each_clamp_id(clamp_id) 1230 p->uclamp[clamp_id].active = false; 1231 1232 if (likely(!p->sched_reset_on_fork)) 1233 return; 1234 1235 for_each_clamp_id(clamp_id) { 1236 unsigned int clamp_value = uclamp_none(clamp_id); 1237 1238 /* By default, RT tasks always get 100% boost */ 1239 if (unlikely(rt_task(p) && clamp_id == UCLAMP_MIN)) 1240 clamp_value = uclamp_none(UCLAMP_MAX); 1241 1242 uclamp_se_set(&p->uclamp_req[clamp_id], clamp_value, false); 1243 } 1244 } 1245 1246 static void __init init_uclamp(void) 1247 { 1248 struct uclamp_se uc_max = {}; 1249 enum uclamp_id clamp_id; 1250 int cpu; 1251 1252 mutex_init(&uclamp_mutex); 1253 1254 for_each_possible_cpu(cpu) { 1255 memset(&cpu_rq(cpu)->uclamp, 0, sizeof(struct uclamp_rq)); 1256 cpu_rq(cpu)->uclamp_flags = 0; 1257 } 1258 1259 for_each_clamp_id(clamp_id) { 1260 uclamp_se_set(&init_task.uclamp_req[clamp_id], 1261 uclamp_none(clamp_id), false); 1262 } 1263 1264 /* System defaults allow max clamp values for both indexes */ 1265 uclamp_se_set(&uc_max, uclamp_none(UCLAMP_MAX), false); 1266 for_each_clamp_id(clamp_id) { 1267 uclamp_default[clamp_id] = uc_max; 1268 #ifdef CONFIG_UCLAMP_TASK_GROUP 1269 root_task_group.uclamp_req[clamp_id] = uc_max; 1270 root_task_group.uclamp[clamp_id] = uc_max; 1271 #endif 1272 } 1273 } 1274 1275 #else /* CONFIG_UCLAMP_TASK */ 1276 static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p) { } 1277 static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) { } 1278 static inline int uclamp_validate(struct task_struct *p, 1279 const struct sched_attr *attr) 1280 { 1281 return -EOPNOTSUPP; 1282 } 1283 static void __setscheduler_uclamp(struct task_struct *p, 1284 const struct sched_attr *attr) { } 1285 static inline void uclamp_fork(struct task_struct *p) { } 1286 static inline void init_uclamp(void) { } 1287 #endif /* CONFIG_UCLAMP_TASK */ 1288 1289 static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags) 1290 { 1291 if (!(flags & ENQUEUE_NOCLOCK)) 1292 update_rq_clock(rq); 1293 1294 if (!(flags & ENQUEUE_RESTORE)) { 1295 sched_info_queued(rq, p); 1296 psi_enqueue(p, flags & ENQUEUE_WAKEUP); 1297 } 1298 1299 uclamp_rq_inc(rq, p); 1300 p->sched_class->enqueue_task(rq, p, flags); 1301 } 1302 1303 static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags) 1304 { 1305 if (!(flags & DEQUEUE_NOCLOCK)) 1306 update_rq_clock(rq); 1307 1308 if (!(flags & DEQUEUE_SAVE)) { 1309 sched_info_dequeued(rq, p); 1310 psi_dequeue(p, flags & DEQUEUE_SLEEP); 1311 } 1312 1313 uclamp_rq_dec(rq, p); 1314 p->sched_class->dequeue_task(rq, p, flags); 1315 } 1316 1317 void activate_task(struct rq *rq, struct task_struct *p, int flags) 1318 { 1319 if (task_contributes_to_load(p)) 1320 rq->nr_uninterruptible--; 1321 1322 enqueue_task(rq, p, flags); 1323 1324 p->on_rq = TASK_ON_RQ_QUEUED; 1325 } 1326 1327 void deactivate_task(struct rq *rq, struct task_struct *p, int flags) 1328 { 1329 p->on_rq = (flags & DEQUEUE_SLEEP) ? 0 : TASK_ON_RQ_MIGRATING; 1330 1331 if (task_contributes_to_load(p)) 1332 rq->nr_uninterruptible++; 1333 1334 dequeue_task(rq, p, flags); 1335 } 1336 1337 /* 1338 * __normal_prio - return the priority that is based on the static prio 1339 */ 1340 static inline int __normal_prio(struct task_struct *p) 1341 { 1342 return p->static_prio; 1343 } 1344 1345 /* 1346 * Calculate the expected normal priority: i.e. priority 1347 * without taking RT-inheritance into account. Might be 1348 * boosted by interactivity modifiers. Changes upon fork, 1349 * setprio syscalls, and whenever the interactivity 1350 * estimator recalculates. 1351 */ 1352 static inline int normal_prio(struct task_struct *p) 1353 { 1354 int prio; 1355 1356 if (task_has_dl_policy(p)) 1357 prio = MAX_DL_PRIO-1; 1358 else if (task_has_rt_policy(p)) 1359 prio = MAX_RT_PRIO-1 - p->rt_priority; 1360 else 1361 prio = __normal_prio(p); 1362 return prio; 1363 } 1364 1365 /* 1366 * Calculate the current priority, i.e. the priority 1367 * taken into account by the scheduler. This value might 1368 * be boosted by RT tasks, or might be boosted by 1369 * interactivity modifiers. Will be RT if the task got 1370 * RT-boosted. If not then it returns p->normal_prio. 1371 */ 1372 static int effective_prio(struct task_struct *p) 1373 { 1374 p->normal_prio = normal_prio(p); 1375 /* 1376 * If we are RT tasks or we were boosted to RT priority, 1377 * keep the priority unchanged. Otherwise, update priority 1378 * to the normal priority: 1379 */ 1380 if (!rt_prio(p->prio)) 1381 return p->normal_prio; 1382 return p->prio; 1383 } 1384 1385 /** 1386 * task_curr - is this task currently executing on a CPU? 1387 * @p: the task in question. 1388 * 1389 * Return: 1 if the task is currently executing. 0 otherwise. 1390 */ 1391 inline int task_curr(const struct task_struct *p) 1392 { 1393 return cpu_curr(task_cpu(p)) == p; 1394 } 1395 1396 /* 1397 * switched_from, switched_to and prio_changed must _NOT_ drop rq->lock, 1398 * use the balance_callback list if you want balancing. 1399 * 1400 * this means any call to check_class_changed() must be followed by a call to 1401 * balance_callback(). 1402 */ 1403 static inline void check_class_changed(struct rq *rq, struct task_struct *p, 1404 const struct sched_class *prev_class, 1405 int oldprio) 1406 { 1407 if (prev_class != p->sched_class) { 1408 if (prev_class->switched_from) 1409 prev_class->switched_from(rq, p); 1410 1411 p->sched_class->switched_to(rq, p); 1412 } else if (oldprio != p->prio || dl_task(p)) 1413 p->sched_class->prio_changed(rq, p, oldprio); 1414 } 1415 1416 void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags) 1417 { 1418 const struct sched_class *class; 1419 1420 if (p->sched_class == rq->curr->sched_class) { 1421 rq->curr->sched_class->check_preempt_curr(rq, p, flags); 1422 } else { 1423 for_each_class(class) { 1424 if (class == rq->curr->sched_class) 1425 break; 1426 if (class == p->sched_class) { 1427 resched_curr(rq); 1428 break; 1429 } 1430 } 1431 } 1432 1433 /* 1434 * A queue event has occurred, and we're going to schedule. In 1435 * this case, we can save a useless back to back clock update. 1436 */ 1437 if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr)) 1438 rq_clock_skip_update(rq); 1439 } 1440 1441 #ifdef CONFIG_SMP 1442 1443 static inline bool is_per_cpu_kthread(struct task_struct *p) 1444 { 1445 if (!(p->flags & PF_KTHREAD)) 1446 return false; 1447 1448 if (p->nr_cpus_allowed != 1) 1449 return false; 1450 1451 return true; 1452 } 1453 1454 /* 1455 * Per-CPU kthreads are allowed to run on !active && online CPUs, see 1456 * __set_cpus_allowed_ptr() and select_fallback_rq(). 1457 */ 1458 static inline bool is_cpu_allowed(struct task_struct *p, int cpu) 1459 { 1460 if (!cpumask_test_cpu(cpu, p->cpus_ptr)) 1461 return false; 1462 1463 if (is_per_cpu_kthread(p)) 1464 return cpu_online(cpu); 1465 1466 return cpu_active(cpu); 1467 } 1468 1469 /* 1470 * This is how migration works: 1471 * 1472 * 1) we invoke migration_cpu_stop() on the target CPU using 1473 * stop_one_cpu(). 1474 * 2) stopper starts to run (implicitly forcing the migrated thread 1475 * off the CPU) 1476 * 3) it checks whether the migrated task is still in the wrong runqueue. 1477 * 4) if it's in the wrong runqueue then the migration thread removes 1478 * it and puts it into the right queue. 1479 * 5) stopper completes and stop_one_cpu() returns and the migration 1480 * is done. 1481 */ 1482 1483 /* 1484 * move_queued_task - move a queued task to new rq. 1485 * 1486 * Returns (locked) new rq. Old rq's lock is released. 1487 */ 1488 static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf, 1489 struct task_struct *p, int new_cpu) 1490 { 1491 lockdep_assert_held(&rq->lock); 1492 1493 WRITE_ONCE(p->on_rq, TASK_ON_RQ_MIGRATING); 1494 dequeue_task(rq, p, DEQUEUE_NOCLOCK); 1495 set_task_cpu(p, new_cpu); 1496 rq_unlock(rq, rf); 1497 1498 rq = cpu_rq(new_cpu); 1499 1500 rq_lock(rq, rf); 1501 BUG_ON(task_cpu(p) != new_cpu); 1502 enqueue_task(rq, p, 0); 1503 p->on_rq = TASK_ON_RQ_QUEUED; 1504 check_preempt_curr(rq, p, 0); 1505 1506 return rq; 1507 } 1508 1509 struct migration_arg { 1510 struct task_struct *task; 1511 int dest_cpu; 1512 }; 1513 1514 /* 1515 * Move (not current) task off this CPU, onto the destination CPU. We're doing 1516 * this because either it can't run here any more (set_cpus_allowed() 1517 * away from this CPU, or CPU going down), or because we're 1518 * attempting to rebalance this task on exec (sched_exec). 1519 * 1520 * So we race with normal scheduler movements, but that's OK, as long 1521 * as the task is no longer on this CPU. 1522 */ 1523 static struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf, 1524 struct task_struct *p, int dest_cpu) 1525 { 1526 /* Affinity changed (again). */ 1527 if (!is_cpu_allowed(p, dest_cpu)) 1528 return rq; 1529 1530 update_rq_clock(rq); 1531 rq = move_queued_task(rq, rf, p, dest_cpu); 1532 1533 return rq; 1534 } 1535 1536 /* 1537 * migration_cpu_stop - this will be executed by a highprio stopper thread 1538 * and performs thread migration by bumping thread off CPU then 1539 * 'pushing' onto another runqueue. 1540 */ 1541 static int migration_cpu_stop(void *data) 1542 { 1543 struct migration_arg *arg = data; 1544 struct task_struct *p = arg->task; 1545 struct rq *rq = this_rq(); 1546 struct rq_flags rf; 1547 1548 /* 1549 * The original target CPU might have gone down and we might 1550 * be on another CPU but it doesn't matter. 1551 */ 1552 local_irq_disable(); 1553 /* 1554 * We need to explicitly wake pending tasks before running 1555 * __migrate_task() such that we will not miss enforcing cpus_ptr 1556 * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test. 1557 */ 1558 sched_ttwu_pending(); 1559 1560 raw_spin_lock(&p->pi_lock); 1561 rq_lock(rq, &rf); 1562 /* 1563 * If task_rq(p) != rq, it cannot be migrated here, because we're 1564 * holding rq->lock, if p->on_rq == 0 it cannot get enqueued because 1565 * we're holding p->pi_lock. 1566 */ 1567 if (task_rq(p) == rq) { 1568 if (task_on_rq_queued(p)) 1569 rq = __migrate_task(rq, &rf, p, arg->dest_cpu); 1570 else 1571 p->wake_cpu = arg->dest_cpu; 1572 } 1573 rq_unlock(rq, &rf); 1574 raw_spin_unlock(&p->pi_lock); 1575 1576 local_irq_enable(); 1577 return 0; 1578 } 1579 1580 /* 1581 * sched_class::set_cpus_allowed must do the below, but is not required to 1582 * actually call this function. 1583 */ 1584 void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask) 1585 { 1586 cpumask_copy(&p->cpus_mask, new_mask); 1587 p->nr_cpus_allowed = cpumask_weight(new_mask); 1588 } 1589 1590 void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) 1591 { 1592 struct rq *rq = task_rq(p); 1593 bool queued, running; 1594 1595 lockdep_assert_held(&p->pi_lock); 1596 1597 queued = task_on_rq_queued(p); 1598 running = task_current(rq, p); 1599 1600 if (queued) { 1601 /* 1602 * Because __kthread_bind() calls this on blocked tasks without 1603 * holding rq->lock. 1604 */ 1605 lockdep_assert_held(&rq->lock); 1606 dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); 1607 } 1608 if (running) 1609 put_prev_task(rq, p); 1610 1611 p->sched_class->set_cpus_allowed(p, new_mask); 1612 1613 if (queued) 1614 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); 1615 if (running) 1616 set_next_task(rq, p); 1617 } 1618 1619 /* 1620 * Change a given task's CPU affinity. Migrate the thread to a 1621 * proper CPU and schedule it away if the CPU it's executing on 1622 * is removed from the allowed bitmask. 1623 * 1624 * NOTE: the caller must have a valid reference to the task, the 1625 * task must not exit() & deallocate itself prematurely. The 1626 * call is not atomic; no spinlocks may be held. 1627 */ 1628 static int __set_cpus_allowed_ptr(struct task_struct *p, 1629 const struct cpumask *new_mask, bool check) 1630 { 1631 const struct cpumask *cpu_valid_mask = cpu_active_mask; 1632 unsigned int dest_cpu; 1633 struct rq_flags rf; 1634 struct rq *rq; 1635 int ret = 0; 1636 1637 rq = task_rq_lock(p, &rf); 1638 update_rq_clock(rq); 1639 1640 if (p->flags & PF_KTHREAD) { 1641 /* 1642 * Kernel threads are allowed on online && !active CPUs 1643 */ 1644 cpu_valid_mask = cpu_online_mask; 1645 } 1646 1647 /* 1648 * Must re-check here, to close a race against __kthread_bind(), 1649 * sched_setaffinity() is not guaranteed to observe the flag. 1650 */ 1651 if (check && (p->flags & PF_NO_SETAFFINITY)) { 1652 ret = -EINVAL; 1653 goto out; 1654 } 1655 1656 if (cpumask_equal(p->cpus_ptr, new_mask)) 1657 goto out; 1658 1659 dest_cpu = cpumask_any_and(cpu_valid_mask, new_mask); 1660 if (dest_cpu >= nr_cpu_ids) { 1661 ret = -EINVAL; 1662 goto out; 1663 } 1664 1665 do_set_cpus_allowed(p, new_mask); 1666 1667 if (p->flags & PF_KTHREAD) { 1668 /* 1669 * For kernel threads that do indeed end up on online && 1670 * !active we want to ensure they are strict per-CPU threads. 1671 */ 1672 WARN_ON(cpumask_intersects(new_mask, cpu_online_mask) && 1673 !cpumask_intersects(new_mask, cpu_active_mask) && 1674 p->nr_cpus_allowed != 1); 1675 } 1676 1677 /* Can the task run on the task's current CPU? If so, we're done */ 1678 if (cpumask_test_cpu(task_cpu(p), new_mask)) 1679 goto out; 1680 1681 if (task_running(rq, p) || p->state == TASK_WAKING) { 1682 struct migration_arg arg = { p, dest_cpu }; 1683 /* Need help from migration thread: drop lock and wait. */ 1684 task_rq_unlock(rq, p, &rf); 1685 stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg); 1686 return 0; 1687 } else if (task_on_rq_queued(p)) { 1688 /* 1689 * OK, since we're going to drop the lock immediately 1690 * afterwards anyway. 1691 */ 1692 rq = move_queued_task(rq, &rf, p, dest_cpu); 1693 } 1694 out: 1695 task_rq_unlock(rq, p, &rf); 1696 1697 return ret; 1698 } 1699 1700 int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) 1701 { 1702 return __set_cpus_allowed_ptr(p, new_mask, false); 1703 } 1704 EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr); 1705 1706 void set_task_cpu(struct task_struct *p, unsigned int new_cpu) 1707 { 1708 #ifdef CONFIG_SCHED_DEBUG 1709 /* 1710 * We should never call set_task_cpu() on a blocked task, 1711 * ttwu() will sort out the placement. 1712 */ 1713 WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING && 1714 !p->on_rq); 1715 1716 /* 1717 * Migrating fair class task must have p->on_rq = TASK_ON_RQ_MIGRATING, 1718 * because schedstat_wait_{start,end} rebase migrating task's wait_start 1719 * time relying on p->on_rq. 1720 */ 1721 WARN_ON_ONCE(p->state == TASK_RUNNING && 1722 p->sched_class == &fair_sched_class && 1723 (p->on_rq && !task_on_rq_migrating(p))); 1724 1725 #ifdef CONFIG_LOCKDEP 1726 /* 1727 * The caller should hold either p->pi_lock or rq->lock, when changing 1728 * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks. 1729 * 1730 * sched_move_task() holds both and thus holding either pins the cgroup, 1731 * see task_group(). 1732 * 1733 * Furthermore, all task_rq users should acquire both locks, see 1734 * task_rq_lock(). 1735 */ 1736 WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) || 1737 lockdep_is_held(&task_rq(p)->lock))); 1738 #endif 1739 /* 1740 * Clearly, migrating tasks to offline CPUs is a fairly daft thing. 1741 */ 1742 WARN_ON_ONCE(!cpu_online(new_cpu)); 1743 #endif 1744 1745 trace_sched_migrate_task(p, new_cpu); 1746 1747 if (task_cpu(p) != new_cpu) { 1748 if (p->sched_class->migrate_task_rq) 1749 p->sched_class->migrate_task_rq(p, new_cpu); 1750 p->se.nr_migrations++; 1751 rseq_migrate(p); 1752 perf_event_task_migrate(p); 1753 } 1754 1755 __set_task_cpu(p, new_cpu); 1756 } 1757 1758 #ifdef CONFIG_NUMA_BALANCING 1759 static void __migrate_swap_task(struct task_struct *p, int cpu) 1760 { 1761 if (task_on_rq_queued(p)) { 1762 struct rq *src_rq, *dst_rq; 1763 struct rq_flags srf, drf; 1764 1765 src_rq = task_rq(p); 1766 dst_rq = cpu_rq(cpu); 1767 1768 rq_pin_lock(src_rq, &srf); 1769 rq_pin_lock(dst_rq, &drf); 1770 1771 deactivate_task(src_rq, p, 0); 1772 set_task_cpu(p, cpu); 1773 activate_task(dst_rq, p, 0); 1774 check_preempt_curr(dst_rq, p, 0); 1775 1776 rq_unpin_lock(dst_rq, &drf); 1777 rq_unpin_lock(src_rq, &srf); 1778 1779 } else { 1780 /* 1781 * Task isn't running anymore; make it appear like we migrated 1782 * it before it went to sleep. This means on wakeup we make the 1783 * previous CPU our target instead of where it really is. 1784 */ 1785 p->wake_cpu = cpu; 1786 } 1787 } 1788 1789 struct migration_swap_arg { 1790 struct task_struct *src_task, *dst_task; 1791 int src_cpu, dst_cpu; 1792 }; 1793 1794 static int migrate_swap_stop(void *data) 1795 { 1796 struct migration_swap_arg *arg = data; 1797 struct rq *src_rq, *dst_rq; 1798 int ret = -EAGAIN; 1799 1800 if (!cpu_active(arg->src_cpu) || !cpu_active(arg->dst_cpu)) 1801 return -EAGAIN; 1802 1803 src_rq = cpu_rq(arg->src_cpu); 1804 dst_rq = cpu_rq(arg->dst_cpu); 1805 1806 double_raw_lock(&arg->src_task->pi_lock, 1807 &arg->dst_task->pi_lock); 1808 double_rq_lock(src_rq, dst_rq); 1809 1810 if (task_cpu(arg->dst_task) != arg->dst_cpu) 1811 goto unlock; 1812 1813 if (task_cpu(arg->src_task) != arg->src_cpu) 1814 goto unlock; 1815 1816 if (!cpumask_test_cpu(arg->dst_cpu, arg->src_task->cpus_ptr)) 1817 goto unlock; 1818 1819 if (!cpumask_test_cpu(arg->src_cpu, arg->dst_task->cpus_ptr)) 1820 goto unlock; 1821 1822 __migrate_swap_task(arg->src_task, arg->dst_cpu); 1823 __migrate_swap_task(arg->dst_task, arg->src_cpu); 1824 1825 ret = 0; 1826 1827 unlock: 1828 double_rq_unlock(src_rq, dst_rq); 1829 raw_spin_unlock(&arg->dst_task->pi_lock); 1830 raw_spin_unlock(&arg->src_task->pi_lock); 1831 1832 return ret; 1833 } 1834 1835 /* 1836 * Cross migrate two tasks 1837 */ 1838 int migrate_swap(struct task_struct *cur, struct task_struct *p, 1839 int target_cpu, int curr_cpu) 1840 { 1841 struct migration_swap_arg arg; 1842 int ret = -EINVAL; 1843 1844 arg = (struct migration_swap_arg){ 1845 .src_task = cur, 1846 .src_cpu = curr_cpu, 1847 .dst_task = p, 1848 .dst_cpu = target_cpu, 1849 }; 1850 1851 if (arg.src_cpu == arg.dst_cpu) 1852 goto out; 1853 1854 /* 1855 * These three tests are all lockless; this is OK since all of them 1856 * will be re-checked with proper locks held further down the line. 1857 */ 1858 if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu)) 1859 goto out; 1860 1861 if (!cpumask_test_cpu(arg.dst_cpu, arg.src_task->cpus_ptr)) 1862 goto out; 1863 1864 if (!cpumask_test_cpu(arg.src_cpu, arg.dst_task->cpus_ptr)) 1865 goto out; 1866 1867 trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu); 1868 ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg); 1869 1870 out: 1871 return ret; 1872 } 1873 #endif /* CONFIG_NUMA_BALANCING */ 1874 1875 /* 1876 * wait_task_inactive - wait for a thread to unschedule. 1877 * 1878 * If @match_state is nonzero, it's the @p->state value just checked and 1879 * not expected to change. If it changes, i.e. @p might have woken up, 1880 * then return zero. When we succeed in waiting for @p to be off its CPU, 1881 * we return a positive number (its total switch count). If a second call 1882 * a short while later returns the same number, the caller can be sure that 1883 * @p has remained unscheduled the whole time. 1884 * 1885 * The caller must ensure that the task *will* unschedule sometime soon, 1886 * else this function might spin for a *long* time. This function can't 1887 * be called with interrupts off, or it may introduce deadlock with 1888 * smp_call_function() if an IPI is sent by the same process we are 1889 * waiting to become inactive. 1890 */ 1891 unsigned long wait_task_inactive(struct task_struct *p, long match_state) 1892 { 1893 int running, queued; 1894 struct rq_flags rf; 1895 unsigned long ncsw; 1896 struct rq *rq; 1897 1898 for (;;) { 1899 /* 1900 * We do the initial early heuristics without holding 1901 * any task-queue locks at all. We'll only try to get 1902 * the runqueue lock when things look like they will 1903 * work out! 1904 */ 1905 rq = task_rq(p); 1906 1907 /* 1908 * If the task is actively running on another CPU 1909 * still, just relax and busy-wait without holding 1910 * any locks. 1911 * 1912 * NOTE! Since we don't hold any locks, it's not 1913 * even sure that "rq" stays as the right runqueue! 1914 * But we don't care, since "task_running()" will 1915 * return false if the runqueue has changed and p 1916 * is actually now running somewhere else! 1917 */ 1918 while (task_running(rq, p)) { 1919 if (match_state && unlikely(p->state != match_state)) 1920 return 0; 1921 cpu_relax(); 1922 } 1923 1924 /* 1925 * Ok, time to look more closely! We need the rq 1926 * lock now, to be *sure*. If we're wrong, we'll 1927 * just go back and repeat. 1928 */ 1929 rq = task_rq_lock(p, &rf); 1930 trace_sched_wait_task(p); 1931 running = task_running(rq, p); 1932 queued = task_on_rq_queued(p); 1933 ncsw = 0; 1934 if (!match_state || p->state == match_state) 1935 ncsw = p->nvcsw | LONG_MIN; /* sets MSB */ 1936 task_rq_unlock(rq, p, &rf); 1937 1938 /* 1939 * If it changed from the expected state, bail out now. 1940 */ 1941 if (unlikely(!ncsw)) 1942 break; 1943 1944 /* 1945 * Was it really running after all now that we 1946 * checked with the proper locks actually held? 1947 * 1948 * Oops. Go back and try again.. 1949 */ 1950 if (unlikely(running)) { 1951 cpu_relax(); 1952 continue; 1953 } 1954 1955 /* 1956 * It's not enough that it's not actively running, 1957 * it must be off the runqueue _entirely_, and not 1958 * preempted! 1959 * 1960 * So if it was still runnable (but just not actively 1961 * running right now), it's preempted, and we should 1962 * yield - it could be a while. 1963 */ 1964 if (unlikely(queued)) { 1965 ktime_t to = NSEC_PER_SEC / HZ; 1966 1967 set_current_state(TASK_UNINTERRUPTIBLE); 1968 schedule_hrtimeout(&to, HRTIMER_MODE_REL); 1969 continue; 1970 } 1971 1972 /* 1973 * Ahh, all good. It wasn't running, and it wasn't 1974 * runnable, which means that it will never become 1975 * running in the future either. We're all done! 1976 */ 1977 break; 1978 } 1979 1980 return ncsw; 1981 } 1982 1983 /*** 1984 * kick_process - kick a running thread to enter/exit the kernel 1985 * @p: the to-be-kicked thread 1986 * 1987 * Cause a process which is running on another CPU to enter 1988 * kernel-mode, without any delay. (to get signals handled.) 1989 * 1990 * NOTE: this function doesn't have to take the runqueue lock, 1991 * because all it wants to ensure is that the remote task enters 1992 * the kernel. If the IPI races and the task has been migrated 1993 * to another CPU then no harm is done and the purpose has been 1994 * achieved as well. 1995 */ 1996 void kick_process(struct task_struct *p) 1997 { 1998 int cpu; 1999 2000 preempt_disable(); 2001 cpu = task_cpu(p); 2002 if ((cpu != smp_processor_id()) && task_curr(p)) 2003 smp_send_reschedule(cpu); 2004 preempt_enable(); 2005 } 2006 EXPORT_SYMBOL_GPL(kick_process); 2007 2008 /* 2009 * ->cpus_ptr is protected by both rq->lock and p->pi_lock 2010 * 2011 * A few notes on cpu_active vs cpu_online: 2012 * 2013 * - cpu_active must be a subset of cpu_online 2014 * 2015 * - on CPU-up we allow per-CPU kthreads on the online && !active CPU, 2016 * see __set_cpus_allowed_ptr(). At this point the newly online 2017 * CPU isn't yet part of the sched domains, and balancing will not 2018 * see it. 2019 * 2020 * - on CPU-down we clear cpu_active() to mask the sched domains and 2021 * avoid the load balancer to place new tasks on the to be removed 2022 * CPU. Existing tasks will remain running there and will be taken 2023 * off. 2024 * 2025 * This means that fallback selection must not select !active CPUs. 2026 * And can assume that any active CPU must be online. Conversely 2027 * select_task_rq() below may allow selection of !active CPUs in order 2028 * to satisfy the above rules. 2029 */ 2030 static int select_fallback_rq(int cpu, struct task_struct *p) 2031 { 2032 int nid = cpu_to_node(cpu); 2033 const struct cpumask *nodemask = NULL; 2034 enum { cpuset, possible, fail } state = cpuset; 2035 int dest_cpu; 2036 2037 /* 2038 * If the node that the CPU is on has been offlined, cpu_to_node() 2039 * will return -1. There is no CPU on the node, and we should 2040 * select the CPU on the other node. 2041 */ 2042 if (nid != -1) { 2043 nodemask = cpumask_of_node(nid); 2044 2045 /* Look for allowed, online CPU in same node. */ 2046 for_each_cpu(dest_cpu, nodemask) { 2047 if (!cpu_active(dest_cpu)) 2048 continue; 2049 if (cpumask_test_cpu(dest_cpu, p->cpus_ptr)) 2050 return dest_cpu; 2051 } 2052 } 2053 2054 for (;;) { 2055 /* Any allowed, online CPU? */ 2056 for_each_cpu(dest_cpu, p->cpus_ptr) { 2057 if (!is_cpu_allowed(p, dest_cpu)) 2058 continue; 2059 2060 goto out; 2061 } 2062 2063 /* No more Mr. Nice Guy. */ 2064 switch (state) { 2065 case cpuset: 2066 if (IS_ENABLED(CONFIG_CPUSETS)) { 2067 cpuset_cpus_allowed_fallback(p); 2068 state = possible; 2069 break; 2070 } 2071 /* Fall-through */ 2072 case possible: 2073 do_set_cpus_allowed(p, cpu_possible_mask); 2074 state = fail; 2075 break; 2076 2077 case fail: 2078 BUG(); 2079 break; 2080 } 2081 } 2082 2083 out: 2084 if (state != cpuset) { 2085 /* 2086 * Don't tell them about moving exiting tasks or 2087 * kernel threads (both mm NULL), since they never 2088 * leave kernel. 2089 */ 2090 if (p->mm && printk_ratelimit()) { 2091 printk_deferred("process %d (%s) no longer affine to cpu%d\n", 2092 task_pid_nr(p), p->comm, cpu); 2093 } 2094 } 2095 2096 return dest_cpu; 2097 } 2098 2099 /* 2100 * The caller (fork, wakeup) owns p->pi_lock, ->cpus_ptr is stable. 2101 */ 2102 static inline 2103 int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags) 2104 { 2105 lockdep_assert_held(&p->pi_lock); 2106 2107 if (p->nr_cpus_allowed > 1) 2108 cpu = p->sched_class->select_task_rq(p, cpu, sd_flags, wake_flags); 2109 else 2110 cpu = cpumask_any(p->cpus_ptr); 2111 2112 /* 2113 * In order not to call set_task_cpu() on a blocking task we need 2114 * to rely on ttwu() to place the task on a valid ->cpus_ptr 2115 * CPU. 2116 * 2117 * Since this is common to all placement strategies, this lives here. 2118 * 2119 * [ this allows ->select_task() to simply return task_cpu(p) and 2120 * not worry about this generic constraint ] 2121 */ 2122 if (unlikely(!is_cpu_allowed(p, cpu))) 2123 cpu = select_fallback_rq(task_cpu(p), p); 2124 2125 return cpu; 2126 } 2127 2128 static void update_avg(u64 *avg, u64 sample) 2129 { 2130 s64 diff = sample - *avg; 2131 *avg += diff >> 3; 2132 } 2133 2134 void sched_set_stop_task(int cpu, struct task_struct *stop) 2135 { 2136 struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 }; 2137 struct task_struct *old_stop = cpu_rq(cpu)->stop; 2138 2139 if (stop) { 2140 /* 2141 * Make it appear like a SCHED_FIFO task, its something 2142 * userspace knows about and won't get confused about. 2143 * 2144 * Also, it will make PI more or less work without too 2145 * much confusion -- but then, stop work should not 2146 * rely on PI working anyway. 2147 */ 2148 sched_setscheduler_nocheck(stop, SCHED_FIFO, ¶m); 2149 2150 stop->sched_class = &stop_sched_class; 2151 } 2152 2153 cpu_rq(cpu)->stop = stop; 2154 2155 if (old_stop) { 2156 /* 2157 * Reset it back to a normal scheduling class so that 2158 * it can die in pieces. 2159 */ 2160 old_stop->sched_class = &rt_sched_class; 2161 } 2162 } 2163 2164 #else 2165 2166 static inline int __set_cpus_allowed_ptr(struct task_struct *p, 2167 const struct cpumask *new_mask, bool check) 2168 { 2169 return set_cpus_allowed_ptr(p, new_mask); 2170 } 2171 2172 #endif /* CONFIG_SMP */ 2173 2174 static void 2175 ttwu_stat(struct task_struct *p, int cpu, int wake_flags) 2176 { 2177 struct rq *rq; 2178 2179 if (!schedstat_enabled()) 2180 return; 2181 2182 rq = this_rq(); 2183 2184 #ifdef CONFIG_SMP 2185 if (cpu == rq->cpu) { 2186 __schedstat_inc(rq->ttwu_local); 2187 __schedstat_inc(p->se.statistics.nr_wakeups_local); 2188 } else { 2189 struct sched_domain *sd; 2190 2191 __schedstat_inc(p->se.statistics.nr_wakeups_remote); 2192 rcu_read_lock(); 2193 for_each_domain(rq->cpu, sd) { 2194 if (cpumask_test_cpu(cpu, sched_domain_span(sd))) { 2195 __schedstat_inc(sd->ttwu_wake_remote); 2196 break; 2197 } 2198 } 2199 rcu_read_unlock(); 2200 } 2201 2202 if (wake_flags & WF_MIGRATED) 2203 __schedstat_inc(p->se.statistics.nr_wakeups_migrate); 2204 #endif /* CONFIG_SMP */ 2205 2206 __schedstat_inc(rq->ttwu_count); 2207 __schedstat_inc(p->se.statistics.nr_wakeups); 2208 2209 if (wake_flags & WF_SYNC) 2210 __schedstat_inc(p->se.statistics.nr_wakeups_sync); 2211 } 2212 2213 /* 2214 * Mark the task runnable and perform wakeup-preemption. 2215 */ 2216 static void ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags, 2217 struct rq_flags *rf) 2218 { 2219 check_preempt_curr(rq, p, wake_flags); 2220 p->state = TASK_RUNNING; 2221 trace_sched_wakeup(p); 2222 2223 #ifdef CONFIG_SMP 2224 if (p->sched_class->task_woken) { 2225 /* 2226 * Our task @p is fully woken up and running; so its safe to 2227 * drop the rq->lock, hereafter rq is only used for statistics. 2228 */ 2229 rq_unpin_lock(rq, rf); 2230 p->sched_class->task_woken(rq, p); 2231 rq_repin_lock(rq, rf); 2232 } 2233 2234 if (rq->idle_stamp) { 2235 u64 delta = rq_clock(rq) - rq->idle_stamp; 2236 u64 max = 2*rq->max_idle_balance_cost; 2237 2238 update_avg(&rq->avg_idle, delta); 2239 2240 if (rq->avg_idle > max) 2241 rq->avg_idle = max; 2242 2243 rq->idle_stamp = 0; 2244 } 2245 #endif 2246 } 2247 2248 static void 2249 ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags, 2250 struct rq_flags *rf) 2251 { 2252 int en_flags = ENQUEUE_WAKEUP | ENQUEUE_NOCLOCK; 2253 2254 lockdep_assert_held(&rq->lock); 2255 2256 #ifdef CONFIG_SMP 2257 if (p->sched_contributes_to_load) 2258 rq->nr_uninterruptible--; 2259 2260 if (wake_flags & WF_MIGRATED) 2261 en_flags |= ENQUEUE_MIGRATED; 2262 #endif 2263 2264 activate_task(rq, p, en_flags); 2265 ttwu_do_wakeup(rq, p, wake_flags, rf); 2266 } 2267 2268 /* 2269 * Called in case the task @p isn't fully descheduled from its runqueue, 2270 * in this case we must do a remote wakeup. Its a 'light' wakeup though, 2271 * since all we need to do is flip p->state to TASK_RUNNING, since 2272 * the task is still ->on_rq. 2273 */ 2274 static int ttwu_remote(struct task_struct *p, int wake_flags) 2275 { 2276 struct rq_flags rf; 2277 struct rq *rq; 2278 int ret = 0; 2279 2280 rq = __task_rq_lock(p, &rf); 2281 if (task_on_rq_queued(p)) { 2282 /* check_preempt_curr() may use rq clock */ 2283 update_rq_clock(rq); 2284 ttwu_do_wakeup(rq, p, wake_flags, &rf); 2285 ret = 1; 2286 } 2287 __task_rq_unlock(rq, &rf); 2288 2289 return ret; 2290 } 2291 2292 #ifdef CONFIG_SMP 2293 void sched_ttwu_pending(void) 2294 { 2295 struct rq *rq = this_rq(); 2296 struct llist_node *llist = llist_del_all(&rq->wake_list); 2297 struct task_struct *p, *t; 2298 struct rq_flags rf; 2299 2300 if (!llist) 2301 return; 2302 2303 rq_lock_irqsave(rq, &rf); 2304 update_rq_clock(rq); 2305 2306 llist_for_each_entry_safe(p, t, llist, wake_entry) 2307 ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0, &rf); 2308 2309 rq_unlock_irqrestore(rq, &rf); 2310 } 2311 2312 void scheduler_ipi(void) 2313 { 2314 /* 2315 * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting 2316 * TIF_NEED_RESCHED remotely (for the first time) will also send 2317 * this IPI. 2318 */ 2319 preempt_fold_need_resched(); 2320 2321 if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick()) 2322 return; 2323 2324 /* 2325 * Not all reschedule IPI handlers call irq_enter/irq_exit, since 2326 * traditionally all their work was done from the interrupt return 2327 * path. Now that we actually do some work, we need to make sure 2328 * we do call them. 2329 * 2330 * Some archs already do call them, luckily irq_enter/exit nest 2331 * properly. 2332 * 2333 * Arguably we should visit all archs and update all handlers, 2334 * however a fair share of IPIs are still resched only so this would 2335 * somewhat pessimize the simple resched case. 2336 */ 2337 irq_enter(); 2338 sched_ttwu_pending(); 2339 2340 /* 2341 * Check if someone kicked us for doing the nohz idle load balance. 2342 */ 2343 if (unlikely(got_nohz_idle_kick())) { 2344 this_rq()->idle_balance = 1; 2345 raise_softirq_irqoff(SCHED_SOFTIRQ); 2346 } 2347 irq_exit(); 2348 } 2349 2350 static void ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags) 2351 { 2352 struct rq *rq = cpu_rq(cpu); 2353 2354 p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED); 2355 2356 if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) { 2357 if (!set_nr_if_polling(rq->idle)) 2358 smp_send_reschedule(cpu); 2359 else 2360 trace_sched_wake_idle_without_ipi(cpu); 2361 } 2362 } 2363 2364 void wake_up_if_idle(int cpu) 2365 { 2366 struct rq *rq = cpu_rq(cpu); 2367 struct rq_flags rf; 2368 2369 rcu_read_lock(); 2370 2371 if (!is_idle_task(rcu_dereference(rq->curr))) 2372 goto out; 2373 2374 if (set_nr_if_polling(rq->idle)) { 2375 trace_sched_wake_idle_without_ipi(cpu); 2376 } else { 2377 rq_lock_irqsave(rq, &rf); 2378 if (is_idle_task(rq->curr)) 2379 smp_send_reschedule(cpu); 2380 /* Else CPU is not idle, do nothing here: */ 2381 rq_unlock_irqrestore(rq, &rf); 2382 } 2383 2384 out: 2385 rcu_read_unlock(); 2386 } 2387 2388 bool cpus_share_cache(int this_cpu, int that_cpu) 2389 { 2390 return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu); 2391 } 2392 #endif /* CONFIG_SMP */ 2393 2394 static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags) 2395 { 2396 struct rq *rq = cpu_rq(cpu); 2397 struct rq_flags rf; 2398 2399 #if defined(CONFIG_SMP) 2400 if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) { 2401 sched_clock_cpu(cpu); /* Sync clocks across CPUs */ 2402 ttwu_queue_remote(p, cpu, wake_flags); 2403 return; 2404 } 2405 #endif 2406 2407 rq_lock(rq, &rf); 2408 update_rq_clock(rq); 2409 ttwu_do_activate(rq, p, wake_flags, &rf); 2410 rq_unlock(rq, &rf); 2411 } 2412 2413 /* 2414 * Notes on Program-Order guarantees on SMP systems. 2415 * 2416 * MIGRATION 2417 * 2418 * The basic program-order guarantee on SMP systems is that when a task [t] 2419 * migrates, all its activity on its old CPU [c0] happens-before any subsequent 2420 * execution on its new CPU [c1]. 2421 * 2422 * For migration (of runnable tasks) this is provided by the following means: 2423 * 2424 * A) UNLOCK of the rq(c0)->lock scheduling out task t 2425 * B) migration for t is required to synchronize *both* rq(c0)->lock and 2426 * rq(c1)->lock (if not at the same time, then in that order). 2427 * C) LOCK of the rq(c1)->lock scheduling in task 2428 * 2429 * Release/acquire chaining guarantees that B happens after A and C after B. 2430 * Note: the CPU doing B need not be c0 or c1 2431 * 2432 * Example: 2433 * 2434 * CPU0 CPU1 CPU2 2435 * 2436 * LOCK rq(0)->lock 2437 * sched-out X 2438 * sched-in Y 2439 * UNLOCK rq(0)->lock 2440 * 2441 * LOCK rq(0)->lock // orders against CPU0 2442 * dequeue X 2443 * UNLOCK rq(0)->lock 2444 * 2445 * LOCK rq(1)->lock 2446 * enqueue X 2447 * UNLOCK rq(1)->lock 2448 * 2449 * LOCK rq(1)->lock // orders against CPU2 2450 * sched-out Z 2451 * sched-in X 2452 * UNLOCK rq(1)->lock 2453 * 2454 * 2455 * BLOCKING -- aka. SLEEP + WAKEUP 2456 * 2457 * For blocking we (obviously) need to provide the same guarantee as for 2458 * migration. However the means are completely different as there is no lock 2459 * chain to provide order. Instead we do: 2460 * 2461 * 1) smp_store_release(X->on_cpu, 0) 2462 * 2) smp_cond_load_acquire(!X->on_cpu) 2463 * 2464 * Example: 2465 * 2466 * CPU0 (schedule) CPU1 (try_to_wake_up) CPU2 (schedule) 2467 * 2468 * LOCK rq(0)->lock LOCK X->pi_lock 2469 * dequeue X 2470 * sched-out X 2471 * smp_store_release(X->on_cpu, 0); 2472 * 2473 * smp_cond_load_acquire(&X->on_cpu, !VAL); 2474 * X->state = WAKING 2475 * set_task_cpu(X,2) 2476 * 2477 * LOCK rq(2)->lock 2478 * enqueue X 2479 * X->state = RUNNING 2480 * UNLOCK rq(2)->lock 2481 * 2482 * LOCK rq(2)->lock // orders against CPU1 2483 * sched-out Z 2484 * sched-in X 2485 * UNLOCK rq(2)->lock 2486 * 2487 * UNLOCK X->pi_lock 2488 * UNLOCK rq(0)->lock 2489 * 2490 * 2491 * However, for wakeups there is a second guarantee we must provide, namely we 2492 * must ensure that CONDITION=1 done by the caller can not be reordered with 2493 * accesses to the task state; see try_to_wake_up() and set_current_state(). 2494 */ 2495 2496 /** 2497 * try_to_wake_up - wake up a thread 2498 * @p: the thread to be awakened 2499 * @state: the mask of task states that can be woken 2500 * @wake_flags: wake modifier flags (WF_*) 2501 * 2502 * If (@state & @p->state) @p->state = TASK_RUNNING. 2503 * 2504 * If the task was not queued/runnable, also place it back on a runqueue. 2505 * 2506 * Atomic against schedule() which would dequeue a task, also see 2507 * set_current_state(). 2508 * 2509 * This function executes a full memory barrier before accessing the task 2510 * state; see set_current_state(). 2511 * 2512 * Return: %true if @p->state changes (an actual wakeup was done), 2513 * %false otherwise. 2514 */ 2515 static int 2516 try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) 2517 { 2518 unsigned long flags; 2519 int cpu, success = 0; 2520 2521 preempt_disable(); 2522 if (p == current) { 2523 /* 2524 * We're waking current, this means 'p->on_rq' and 'task_cpu(p) 2525 * == smp_processor_id()'. Together this means we can special 2526 * case the whole 'p->on_rq && ttwu_remote()' case below 2527 * without taking any locks. 2528 * 2529 * In particular: 2530 * - we rely on Program-Order guarantees for all the ordering, 2531 * - we're serialized against set_special_state() by virtue of 2532 * it disabling IRQs (this allows not taking ->pi_lock). 2533 */ 2534 if (!(p->state & state)) 2535 goto out; 2536 2537 success = 1; 2538 cpu = task_cpu(p); 2539 trace_sched_waking(p); 2540 p->state = TASK_RUNNING; 2541 trace_sched_wakeup(p); 2542 goto out; 2543 } 2544 2545 /* 2546 * If we are going to wake up a thread waiting for CONDITION we 2547 * need to ensure that CONDITION=1 done by the caller can not be 2548 * reordered with p->state check below. This pairs with mb() in 2549 * set_current_state() the waiting thread does. 2550 */ 2551 raw_spin_lock_irqsave(&p->pi_lock, flags); 2552 smp_mb__after_spinlock(); 2553 if (!(p->state & state)) 2554 goto unlock; 2555 2556 trace_sched_waking(p); 2557 2558 /* We're going to change ->state: */ 2559 success = 1; 2560 cpu = task_cpu(p); 2561 2562 /* 2563 * Ensure we load p->on_rq _after_ p->state, otherwise it would 2564 * be possible to, falsely, observe p->on_rq == 0 and get stuck 2565 * in smp_cond_load_acquire() below. 2566 * 2567 * sched_ttwu_pending() try_to_wake_up() 2568 * STORE p->on_rq = 1 LOAD p->state 2569 * UNLOCK rq->lock 2570 * 2571 * __schedule() (switch to task 'p') 2572 * LOCK rq->lock smp_rmb(); 2573 * smp_mb__after_spinlock(); 2574 * UNLOCK rq->lock 2575 * 2576 * [task p] 2577 * STORE p->state = UNINTERRUPTIBLE LOAD p->on_rq 2578 * 2579 * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in 2580 * __schedule(). See the comment for smp_mb__after_spinlock(). 2581 */ 2582 smp_rmb(); 2583 if (p->on_rq && ttwu_remote(p, wake_flags)) 2584 goto unlock; 2585 2586 #ifdef CONFIG_SMP 2587 /* 2588 * Ensure we load p->on_cpu _after_ p->on_rq, otherwise it would be 2589 * possible to, falsely, observe p->on_cpu == 0. 2590 * 2591 * One must be running (->on_cpu == 1) in order to remove oneself 2592 * from the runqueue. 2593 * 2594 * __schedule() (switch to task 'p') try_to_wake_up() 2595 * STORE p->on_cpu = 1 LOAD p->on_rq 2596 * UNLOCK rq->lock 2597 * 2598 * __schedule() (put 'p' to sleep) 2599 * LOCK rq->lock smp_rmb(); 2600 * smp_mb__after_spinlock(); 2601 * STORE p->on_rq = 0 LOAD p->on_cpu 2602 * 2603 * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in 2604 * __schedule(). See the comment for smp_mb__after_spinlock(). 2605 */ 2606 smp_rmb(); 2607 2608 /* 2609 * If the owning (remote) CPU is still in the middle of schedule() with 2610 * this task as prev, wait until its done referencing the task. 2611 * 2612 * Pairs with the smp_store_release() in finish_task(). 2613 * 2614 * This ensures that tasks getting woken will be fully ordered against 2615 * their previous state and preserve Program Order. 2616 */ 2617 smp_cond_load_acquire(&p->on_cpu, !VAL); 2618 2619 p->sched_contributes_to_load = !!task_contributes_to_load(p); 2620 p->state = TASK_WAKING; 2621 2622 if (p->in_iowait) { 2623 delayacct_blkio_end(p); 2624 atomic_dec(&task_rq(p)->nr_iowait); 2625 } 2626 2627 cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags); 2628 if (task_cpu(p) != cpu) { 2629 wake_flags |= WF_MIGRATED; 2630 psi_ttwu_dequeue(p); 2631 set_task_cpu(p, cpu); 2632 } 2633 2634 #else /* CONFIG_SMP */ 2635 2636 if (p->in_iowait) { 2637 delayacct_blkio_end(p); 2638 atomic_dec(&task_rq(p)->nr_iowait); 2639 } 2640 2641 #endif /* CONFIG_SMP */ 2642 2643 ttwu_queue(p, cpu, wake_flags); 2644 unlock: 2645 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 2646 out: 2647 if (success) 2648 ttwu_stat(p, cpu, wake_flags); 2649 preempt_enable(); 2650 2651 return success; 2652 } 2653 2654 /** 2655 * wake_up_process - Wake up a specific process 2656 * @p: The process to be woken up. 2657 * 2658 * Attempt to wake up the nominated process and move it to the set of runnable 2659 * processes. 2660 * 2661 * Return: 1 if the process was woken up, 0 if it was already running. 2662 * 2663 * This function executes a full memory barrier before accessing the task state. 2664 */ 2665 int wake_up_process(struct task_struct *p) 2666 { 2667 return try_to_wake_up(p, TASK_NORMAL, 0); 2668 } 2669 EXPORT_SYMBOL(wake_up_process); 2670 2671 int wake_up_state(struct task_struct *p, unsigned int state) 2672 { 2673 return try_to_wake_up(p, state, 0); 2674 } 2675 2676 /* 2677 * Perform scheduler related setup for a newly forked process p. 2678 * p is forked by current. 2679 * 2680 * __sched_fork() is basic setup used by init_idle() too: 2681 */ 2682 static void __sched_fork(unsigned long clone_flags, struct task_struct *p) 2683 { 2684 p->on_rq = 0; 2685 2686 p->se.on_rq = 0; 2687 p->se.exec_start = 0; 2688 p->se.sum_exec_runtime = 0; 2689 p->se.prev_sum_exec_runtime = 0; 2690 p->se.nr_migrations = 0; 2691 p->se.vruntime = 0; 2692 INIT_LIST_HEAD(&p->se.group_node); 2693 2694 #ifdef CONFIG_FAIR_GROUP_SCHED 2695 p->se.cfs_rq = NULL; 2696 #endif 2697 2698 #ifdef CONFIG_SCHEDSTATS 2699 /* Even if schedstat is disabled, there should not be garbage */ 2700 memset(&p->se.statistics, 0, sizeof(p->se.statistics)); 2701 #endif 2702 2703 RB_CLEAR_NODE(&p->dl.rb_node); 2704 init_dl_task_timer(&p->dl); 2705 init_dl_inactive_task_timer(&p->dl); 2706 __dl_clear_params(p); 2707 2708 INIT_LIST_HEAD(&p->rt.run_list); 2709 p->rt.timeout = 0; 2710 p->rt.time_slice = sched_rr_timeslice; 2711 p->rt.on_rq = 0; 2712 p->rt.on_list = 0; 2713 2714 #ifdef CONFIG_PREEMPT_NOTIFIERS 2715 INIT_HLIST_HEAD(&p->preempt_notifiers); 2716 #endif 2717 2718 #ifdef CONFIG_COMPACTION 2719 p->capture_control = NULL; 2720 #endif 2721 init_numa_balancing(clone_flags, p); 2722 } 2723 2724 DEFINE_STATIC_KEY_FALSE(sched_numa_balancing); 2725 2726 #ifdef CONFIG_NUMA_BALANCING 2727 2728 void set_numabalancing_state(bool enabled) 2729 { 2730 if (enabled) 2731 static_branch_enable(&sched_numa_balancing); 2732 else 2733 static_branch_disable(&sched_numa_balancing); 2734 } 2735 2736 #ifdef CONFIG_PROC_SYSCTL 2737 int sysctl_numa_balancing(struct ctl_table *table, int write, 2738 void __user *buffer, size_t *lenp, loff_t *ppos) 2739 { 2740 struct ctl_table t; 2741 int err; 2742 int state = static_branch_likely(&sched_numa_balancing); 2743 2744 if (write && !capable(CAP_SYS_ADMIN)) 2745 return -EPERM; 2746 2747 t = *table; 2748 t.data = &state; 2749 err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); 2750 if (err < 0) 2751 return err; 2752 if (write) 2753 set_numabalancing_state(state); 2754 return err; 2755 } 2756 #endif 2757 #endif 2758 2759 #ifdef CONFIG_SCHEDSTATS 2760 2761 DEFINE_STATIC_KEY_FALSE(sched_schedstats); 2762 static bool __initdata __sched_schedstats = false; 2763 2764 static void set_schedstats(bool enabled) 2765 { 2766 if (enabled) 2767 static_branch_enable(&sched_schedstats); 2768 else 2769 static_branch_disable(&sched_schedstats); 2770 } 2771 2772 void force_schedstat_enabled(void) 2773 { 2774 if (!schedstat_enabled()) { 2775 pr_info("kernel profiling enabled schedstats, disable via kernel.sched_schedstats.\n"); 2776 static_branch_enable(&sched_schedstats); 2777 } 2778 } 2779 2780 static int __init setup_schedstats(char *str) 2781 { 2782 int ret = 0; 2783 if (!str) 2784 goto out; 2785 2786 /* 2787 * This code is called before jump labels have been set up, so we can't 2788 * change the static branch directly just yet. Instead set a temporary 2789 * variable so init_schedstats() can do it later. 2790 */ 2791 if (!strcmp(str, "enable")) { 2792 __sched_schedstats = true; 2793 ret = 1; 2794 } else if (!strcmp(str, "disable")) { 2795 __sched_schedstats = false; 2796 ret = 1; 2797 } 2798 out: 2799 if (!ret) 2800 pr_warn("Unable to parse schedstats=\n"); 2801 2802 return ret; 2803 } 2804 __setup("schedstats=", setup_schedstats); 2805 2806 static void __init init_schedstats(void) 2807 { 2808 set_schedstats(__sched_schedstats); 2809 } 2810 2811 #ifdef CONFIG_PROC_SYSCTL 2812 int sysctl_schedstats(struct ctl_table *table, int write, 2813 void __user *buffer, size_t *lenp, loff_t *ppos) 2814 { 2815 struct ctl_table t; 2816 int err; 2817 int state = static_branch_likely(&sched_schedstats); 2818 2819 if (write && !capable(CAP_SYS_ADMIN)) 2820 return -EPERM; 2821 2822 t = *table; 2823 t.data = &state; 2824 err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); 2825 if (err < 0) 2826 return err; 2827 if (write) 2828 set_schedstats(state); 2829 return err; 2830 } 2831 #endif /* CONFIG_PROC_SYSCTL */ 2832 #else /* !CONFIG_SCHEDSTATS */ 2833 static inline void init_schedstats(void) {} 2834 #endif /* CONFIG_SCHEDSTATS */ 2835 2836 /* 2837 * fork()/clone()-time setup: 2838 */ 2839 int sched_fork(unsigned long clone_flags, struct task_struct *p) 2840 { 2841 unsigned long flags; 2842 2843 __sched_fork(clone_flags, p); 2844 /* 2845 * We mark the process as NEW here. This guarantees that 2846 * nobody will actually run it, and a signal or other external 2847 * event cannot wake it up and insert it on the runqueue either. 2848 */ 2849 p->state = TASK_NEW; 2850 2851 /* 2852 * Make sure we do not leak PI boosting priority to the child. 2853 */ 2854 p->prio = current->normal_prio; 2855 2856 uclamp_fork(p); 2857 2858 /* 2859 * Revert to default priority/policy on fork if requested. 2860 */ 2861 if (unlikely(p->sched_reset_on_fork)) { 2862 if (task_has_dl_policy(p) || task_has_rt_policy(p)) { 2863 p->policy = SCHED_NORMAL; 2864 p->static_prio = NICE_TO_PRIO(0); 2865 p->rt_priority = 0; 2866 } else if (PRIO_TO_NICE(p->static_prio) < 0) 2867 p->static_prio = NICE_TO_PRIO(0); 2868 2869 p->prio = p->normal_prio = __normal_prio(p); 2870 set_load_weight(p, false); 2871 2872 /* 2873 * We don't need the reset flag anymore after the fork. It has 2874 * fulfilled its duty: 2875 */ 2876 p->sched_reset_on_fork = 0; 2877 } 2878 2879 if (dl_prio(p->prio)) 2880 return -EAGAIN; 2881 else if (rt_prio(p->prio)) 2882 p->sched_class = &rt_sched_class; 2883 else 2884 p->sched_class = &fair_sched_class; 2885 2886 init_entity_runnable_average(&p->se); 2887 2888 /* 2889 * The child is not yet in the pid-hash so no cgroup attach races, 2890 * and the cgroup is pinned to this child due to cgroup_fork() 2891 * is ran before sched_fork(). 2892 * 2893 * Silence PROVE_RCU. 2894 */ 2895 raw_spin_lock_irqsave(&p->pi_lock, flags); 2896 /* 2897 * We're setting the CPU for the first time, we don't migrate, 2898 * so use __set_task_cpu(). 2899 */ 2900 __set_task_cpu(p, smp_processor_id()); 2901 if (p->sched_class->task_fork) 2902 p->sched_class->task_fork(p); 2903 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 2904 2905 #ifdef CONFIG_SCHED_INFO 2906 if (likely(sched_info_on())) 2907 memset(&p->sched_info, 0, sizeof(p->sched_info)); 2908 #endif 2909 #if defined(CONFIG_SMP) 2910 p->on_cpu = 0; 2911 #endif 2912 init_task_preempt_count(p); 2913 #ifdef CONFIG_SMP 2914 plist_node_init(&p->pushable_tasks, MAX_PRIO); 2915 RB_CLEAR_NODE(&p->pushable_dl_tasks); 2916 #endif 2917 return 0; 2918 } 2919 2920 unsigned long to_ratio(u64 period, u64 runtime) 2921 { 2922 if (runtime == RUNTIME_INF) 2923 return BW_UNIT; 2924 2925 /* 2926 * Doing this here saves a lot of checks in all 2927 * the calling paths, and returning zero seems 2928 * safe for them anyway. 2929 */ 2930 if (period == 0) 2931 return 0; 2932 2933 return div64_u64(runtime << BW_SHIFT, period); 2934 } 2935 2936 /* 2937 * wake_up_new_task - wake up a newly created task for the first time. 2938 * 2939 * This function will do some initial scheduler statistics housekeeping 2940 * that must be done for every newly created context, then puts the task 2941 * on the runqueue and wakes it. 2942 */ 2943 void wake_up_new_task(struct task_struct *p) 2944 { 2945 struct rq_flags rf; 2946 struct rq *rq; 2947 2948 raw_spin_lock_irqsave(&p->pi_lock, rf.flags); 2949 p->state = TASK_RUNNING; 2950 #ifdef CONFIG_SMP 2951 /* 2952 * Fork balancing, do it here and not earlier because: 2953 * - cpus_ptr can change in the fork path 2954 * - any previously selected CPU might disappear through hotplug 2955 * 2956 * Use __set_task_cpu() to avoid calling sched_class::migrate_task_rq, 2957 * as we're not fully set-up yet. 2958 */ 2959 p->recent_used_cpu = task_cpu(p); 2960 __set_task_cpu(p, select_task_rq(p, task_cpu(p), SD_BALANCE_FORK, 0)); 2961 #endif 2962 rq = __task_rq_lock(p, &rf); 2963 update_rq_clock(rq); 2964 post_init_entity_util_avg(p); 2965 2966 activate_task(rq, p, ENQUEUE_NOCLOCK); 2967 trace_sched_wakeup_new(p); 2968 check_preempt_curr(rq, p, WF_FORK); 2969 #ifdef CONFIG_SMP 2970 if (p->sched_class->task_woken) { 2971 /* 2972 * Nothing relies on rq->lock after this, so its fine to 2973 * drop it. 2974 */ 2975 rq_unpin_lock(rq, &rf); 2976 p->sched_class->task_woken(rq, p); 2977 rq_repin_lock(rq, &rf); 2978 } 2979 #endif 2980 task_rq_unlock(rq, p, &rf); 2981 } 2982 2983 #ifdef CONFIG_PREEMPT_NOTIFIERS 2984 2985 static DEFINE_STATIC_KEY_FALSE(preempt_notifier_key); 2986 2987 void preempt_notifier_inc(void) 2988 { 2989 static_branch_inc(&preempt_notifier_key); 2990 } 2991 EXPORT_SYMBOL_GPL(preempt_notifier_inc); 2992 2993 void preempt_notifier_dec(void) 2994 { 2995 static_branch_dec(&preempt_notifier_key); 2996 } 2997 EXPORT_SYMBOL_GPL(preempt_notifier_dec); 2998 2999 /** 3000 * preempt_notifier_register - tell me when current is being preempted & rescheduled 3001 * @notifier: notifier struct to register 3002 */ 3003 void preempt_notifier_register(struct preempt_notifier *notifier) 3004 { 3005 if (!static_branch_unlikely(&preempt_notifier_key)) 3006 WARN(1, "registering preempt_notifier while notifiers disabled\n"); 3007 3008 hlist_add_head(¬ifier->link, ¤t->preempt_notifiers); 3009 } 3010 EXPORT_SYMBOL_GPL(preempt_notifier_register); 3011 3012 /** 3013 * preempt_notifier_unregister - no longer interested in preemption notifications 3014 * @notifier: notifier struct to unregister 3015 * 3016 * This is *not* safe to call from within a preemption notifier. 3017 */ 3018 void preempt_notifier_unregister(struct preempt_notifier *notifier) 3019 { 3020 hlist_del(¬ifier->link); 3021 } 3022 EXPORT_SYMBOL_GPL(preempt_notifier_unregister); 3023 3024 static void __fire_sched_in_preempt_notifiers(struct task_struct *curr) 3025 { 3026 struct preempt_notifier *notifier; 3027 3028 hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) 3029 notifier->ops->sched_in(notifier, raw_smp_processor_id()); 3030 } 3031 3032 static __always_inline void fire_sched_in_preempt_notifiers(struct task_struct *curr) 3033 { 3034 if (static_branch_unlikely(&preempt_notifier_key)) 3035 __fire_sched_in_preempt_notifiers(curr); 3036 } 3037 3038 static void 3039 __fire_sched_out_preempt_notifiers(struct task_struct *curr, 3040 struct task_struct *next) 3041 { 3042 struct preempt_notifier *notifier; 3043 3044 hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) 3045 notifier->ops->sched_out(notifier, next); 3046 } 3047 3048 static __always_inline void 3049 fire_sched_out_preempt_notifiers(struct task_struct *curr, 3050 struct task_struct *next) 3051 { 3052 if (static_branch_unlikely(&preempt_notifier_key)) 3053 __fire_sched_out_preempt_notifiers(curr, next); 3054 } 3055 3056 #else /* !CONFIG_PREEMPT_NOTIFIERS */ 3057 3058 static inline void fire_sched_in_preempt_notifiers(struct task_struct *curr) 3059 { 3060 } 3061 3062 static inline void 3063 fire_sched_out_preempt_notifiers(struct task_struct *curr, 3064 struct task_struct *next) 3065 { 3066 } 3067 3068 #endif /* CONFIG_PREEMPT_NOTIFIERS */ 3069 3070 static inline void prepare_task(struct task_struct *next) 3071 { 3072 #ifdef CONFIG_SMP 3073 /* 3074 * Claim the task as running, we do this before switching to it 3075 * such that any running task will have this set. 3076 */ 3077 next->on_cpu = 1; 3078 #endif 3079 } 3080 3081 static inline void finish_task(struct task_struct *prev) 3082 { 3083 #ifdef CONFIG_SMP 3084 /* 3085 * After ->on_cpu is cleared, the task can be moved to a different CPU. 3086 * We must ensure this doesn't happen until the switch is completely 3087 * finished. 3088 * 3089 * In particular, the load of prev->state in finish_task_switch() must 3090 * happen before this. 3091 * 3092 * Pairs with the smp_cond_load_acquire() in try_to_wake_up(). 3093 */ 3094 smp_store_release(&prev->on_cpu, 0); 3095 #endif 3096 } 3097 3098 static inline void 3099 prepare_lock_switch(struct rq *rq, struct task_struct *next, struct rq_flags *rf) 3100 { 3101 /* 3102 * Since the runqueue lock will be released by the next 3103 * task (which is an invalid locking op but in the case 3104 * of the scheduler it's an obvious special-case), so we 3105 * do an early lockdep release here: 3106 */ 3107 rq_unpin_lock(rq, rf); 3108 spin_release(&rq->lock.dep_map, 1, _THIS_IP_); 3109 #ifdef CONFIG_DEBUG_SPINLOCK 3110 /* this is a valid case when another task releases the spinlock */ 3111 rq->lock.owner = next; 3112 #endif 3113 } 3114 3115 static inline void finish_lock_switch(struct rq *rq) 3116 { 3117 /* 3118 * If we are tracking spinlock dependencies then we have to 3119 * fix up the runqueue lock - which gets 'carried over' from 3120 * prev into current: 3121 */ 3122 spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_); 3123 raw_spin_unlock_irq(&rq->lock); 3124 } 3125 3126 /* 3127 * NOP if the arch has not defined these: 3128 */ 3129 3130 #ifndef prepare_arch_switch 3131 # define prepare_arch_switch(next) do { } while (0) 3132 #endif 3133 3134 #ifndef finish_arch_post_lock_switch 3135 # define finish_arch_post_lock_switch() do { } while (0) 3136 #endif 3137 3138 /** 3139 * prepare_task_switch - prepare to switch tasks 3140 * @rq: the runqueue preparing to switch 3141 * @prev: the current task that is being switched out 3142 * @next: the task we are going to switch to. 3143 * 3144 * This is called with the rq lock held and interrupts off. It must 3145 * be paired with a subsequent finish_task_switch after the context 3146 * switch. 3147 * 3148 * prepare_task_switch sets up locking and calls architecture specific 3149 * hooks. 3150 */ 3151 static inline void 3152 prepare_task_switch(struct rq *rq, struct task_struct *prev, 3153 struct task_struct *next) 3154 { 3155 kcov_prepare_switch(prev); 3156 sched_info_switch(rq, prev, next); 3157 perf_event_task_sched_out(prev, next); 3158 rseq_preempt(prev); 3159 fire_sched_out_preempt_notifiers(prev, next); 3160 prepare_task(next); 3161 prepare_arch_switch(next); 3162 } 3163 3164 /** 3165 * finish_task_switch - clean up after a task-switch 3166 * @prev: the thread we just switched away from. 3167 * 3168 * finish_task_switch must be called after the context switch, paired 3169 * with a prepare_task_switch call before the context switch. 3170 * finish_task_switch will reconcile locking set up by prepare_task_switch, 3171 * and do any other architecture-specific cleanup actions. 3172 * 3173 * Note that we may have delayed dropping an mm in context_switch(). If 3174 * so, we finish that here outside of the runqueue lock. (Doing it 3175 * with the lock held can cause deadlocks; see schedule() for 3176 * details.) 3177 * 3178 * The context switch have flipped the stack from under us and restored the 3179 * local variables which were saved when this task called schedule() in the 3180 * past. prev == current is still correct but we need to recalculate this_rq 3181 * because prev may have moved to another CPU. 3182 */ 3183 static struct rq *finish_task_switch(struct task_struct *prev) 3184 __releases(rq->lock) 3185 { 3186 struct rq *rq = this_rq(); 3187 struct mm_struct *mm = rq->prev_mm; 3188 long prev_state; 3189 3190 /* 3191 * The previous task will have left us with a preempt_count of 2 3192 * because it left us after: 3193 * 3194 * schedule() 3195 * preempt_disable(); // 1 3196 * __schedule() 3197 * raw_spin_lock_irq(&rq->lock) // 2 3198 * 3199 * Also, see FORK_PREEMPT_COUNT. 3200 */ 3201 if (WARN_ONCE(preempt_count() != 2*PREEMPT_DISABLE_OFFSET, 3202 "corrupted preempt_count: %s/%d/0x%x\n", 3203 current->comm, current->pid, preempt_count())) 3204 preempt_count_set(FORK_PREEMPT_COUNT); 3205 3206 rq->prev_mm = NULL; 3207 3208 /* 3209 * A task struct has one reference for the use as "current". 3210 * If a task dies, then it sets TASK_DEAD in tsk->state and calls 3211 * schedule one last time. The schedule call will never return, and 3212 * the scheduled task must drop that reference. 3213 * 3214 * We must observe prev->state before clearing prev->on_cpu (in 3215 * finish_task), otherwise a concurrent wakeup can get prev 3216 * running on another CPU and we could rave with its RUNNING -> DEAD 3217 * transition, resulting in a double drop. 3218 */ 3219 prev_state = prev->state; 3220 vtime_task_switch(prev); 3221 perf_event_task_sched_in(prev, current); 3222 finish_task(prev); 3223 finish_lock_switch(rq); 3224 finish_arch_post_lock_switch(); 3225 kcov_finish_switch(current); 3226 3227 fire_sched_in_preempt_notifiers(current); 3228 /* 3229 * When switching through a kernel thread, the loop in 3230 * membarrier_{private,global}_expedited() may have observed that 3231 * kernel thread and not issued an IPI. It is therefore possible to 3232 * schedule between user->kernel->user threads without passing though 3233 * switch_mm(). Membarrier requires a barrier after storing to 3234 * rq->curr, before returning to userspace, so provide them here: 3235 * 3236 * - a full memory barrier for {PRIVATE,GLOBAL}_EXPEDITED, implicitly 3237 * provided by mmdrop(), 3238 * - a sync_core for SYNC_CORE. 3239 */ 3240 if (mm) { 3241 membarrier_mm_sync_core_before_usermode(mm); 3242 mmdrop(mm); 3243 } 3244 if (unlikely(prev_state == TASK_DEAD)) { 3245 if (prev->sched_class->task_dead) 3246 prev->sched_class->task_dead(prev); 3247 3248 /* 3249 * Remove function-return probe instances associated with this 3250 * task and put them back on the free list. 3251 */ 3252 kprobe_flush_task(prev); 3253 3254 /* Task is done with its stack. */ 3255 put_task_stack(prev); 3256 3257 put_task_struct_rcu_user(prev); 3258 } 3259 3260 tick_nohz_task_switch(); 3261 return rq; 3262 } 3263 3264 #ifdef CONFIG_SMP 3265 3266 /* rq->lock is NOT held, but preemption is disabled */ 3267 static void __balance_callback(struct rq *rq) 3268 { 3269 struct callback_head *head, *next; 3270 void (*func)(struct rq *rq); 3271 unsigned long flags; 3272 3273 raw_spin_lock_irqsave(&rq->lock, flags); 3274 head = rq->balance_callback; 3275 rq->balance_callback = NULL; 3276 while (head) { 3277 func = (void (*)(struct rq *))head->func; 3278 next = head->next; 3279 head->next = NULL; 3280 head = next; 3281 3282 func(rq); 3283 } 3284 raw_spin_unlock_irqrestore(&rq->lock, flags); 3285 } 3286 3287 static inline void balance_callback(struct rq *rq) 3288 { 3289 if (unlikely(rq->balance_callback)) 3290 __balance_callback(rq); 3291 } 3292 3293 #else 3294 3295 static inline void balance_callback(struct rq *rq) 3296 { 3297 } 3298 3299 #endif 3300 3301 /** 3302 * schedule_tail - first thing a freshly forked thread must call. 3303 * @prev: the thread we just switched away from. 3304 */ 3305 asmlinkage __visible void schedule_tail(struct task_struct *prev) 3306 __releases(rq->lock) 3307 { 3308 struct rq *rq; 3309 3310 /* 3311 * New tasks start with FORK_PREEMPT_COUNT, see there and 3312 * finish_task_switch() for details. 3313 * 3314 * finish_task_switch() will drop rq->lock() and lower preempt_count 3315 * and the preempt_enable() will end up enabling preemption (on 3316 * PREEMPT_COUNT kernels). 3317 */ 3318 3319 rq = finish_task_switch(prev); 3320 balance_callback(rq); 3321 preempt_enable(); 3322 3323 if (current->set_child_tid) 3324 put_user(task_pid_vnr(current), current->set_child_tid); 3325 3326 calculate_sigpending(); 3327 } 3328 3329 /* 3330 * context_switch - switch to the new MM and the new thread's register state. 3331 */ 3332 static __always_inline struct rq * 3333 context_switch(struct rq *rq, struct task_struct *prev, 3334 struct task_struct *next, struct rq_flags *rf) 3335 { 3336 prepare_task_switch(rq, prev, next); 3337 3338 /* 3339 * For paravirt, this is coupled with an exit in switch_to to 3340 * combine the page table reload and the switch backend into 3341 * one hypercall. 3342 */ 3343 arch_start_context_switch(prev); 3344 3345 /* 3346 * kernel -> kernel lazy + transfer active 3347 * user -> kernel lazy + mmgrab() active 3348 * 3349 * kernel -> user switch + mmdrop() active 3350 * user -> user switch 3351 */ 3352 if (!next->mm) { // to kernel 3353 enter_lazy_tlb(prev->active_mm, next); 3354 3355 next->active_mm = prev->active_mm; 3356 if (prev->mm) // from user 3357 mmgrab(prev->active_mm); 3358 else 3359 prev->active_mm = NULL; 3360 } else { // to user 3361 membarrier_switch_mm(rq, prev->active_mm, next->mm); 3362 /* 3363 * sys_membarrier() requires an smp_mb() between setting 3364 * rq->curr / membarrier_switch_mm() and returning to userspace. 3365 * 3366 * The below provides this either through switch_mm(), or in 3367 * case 'prev->active_mm == next->mm' through 3368 * finish_task_switch()'s mmdrop(). 3369 */ 3370 switch_mm_irqs_off(prev->active_mm, next->mm, next); 3371 3372 if (!prev->mm) { // from kernel 3373 /* will mmdrop() in finish_task_switch(). */ 3374 rq->prev_mm = prev->active_mm; 3375 prev->active_mm = NULL; 3376 } 3377 } 3378 3379 rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP); 3380 3381 prepare_lock_switch(rq, next, rf); 3382 3383 /* Here we just switch the register state and the stack. */ 3384 switch_to(prev, next, prev); 3385 barrier(); 3386 3387 return finish_task_switch(prev); 3388 } 3389 3390 /* 3391 * nr_running and nr_context_switches: 3392 * 3393 * externally visible scheduler statistics: current number of runnable 3394 * threads, total number of context switches performed since bootup. 3395 */ 3396 unsigned long nr_running(void) 3397 { 3398 unsigned long i, sum = 0; 3399 3400 for_each_online_cpu(i) 3401 sum += cpu_rq(i)->nr_running; 3402 3403 return sum; 3404 } 3405 3406 /* 3407 * Check if only the current task is running on the CPU. 3408 * 3409 * Caution: this function does not check that the caller has disabled 3410 * preemption, thus the result might have a time-of-check-to-time-of-use 3411 * race. The caller is responsible to use it correctly, for example: 3412 * 3413 * - from a non-preemptible section (of course) 3414 * 3415 * - from a thread that is bound to a single CPU 3416 * 3417 * - in a loop with very short iterations (e.g. a polling loop) 3418 */ 3419 bool single_task_running(void) 3420 { 3421 return raw_rq()->nr_running == 1; 3422 } 3423 EXPORT_SYMBOL(single_task_running); 3424 3425 unsigned long long nr_context_switches(void) 3426 { 3427 int i; 3428 unsigned long long sum = 0; 3429 3430 for_each_possible_cpu(i) 3431 sum += cpu_rq(i)->nr_switches; 3432 3433 return sum; 3434 } 3435 3436 /* 3437 * Consumers of these two interfaces, like for example the cpuidle menu 3438 * governor, are using nonsensical data. Preferring shallow idle state selection 3439 * for a CPU that has IO-wait which might not even end up running the task when 3440 * it does become runnable. 3441 */ 3442 3443 unsigned long nr_iowait_cpu(int cpu) 3444 { 3445 return atomic_read(&cpu_rq(cpu)->nr_iowait); 3446 } 3447 3448 /* 3449 * IO-wait accounting, and how its mostly bollocks (on SMP). 3450 * 3451 * The idea behind IO-wait account is to account the idle time that we could 3452 * have spend running if it were not for IO. That is, if we were to improve the 3453 * storage performance, we'd have a proportional reduction in IO-wait time. 3454 * 3455 * This all works nicely on UP, where, when a task blocks on IO, we account 3456 * idle time as IO-wait, because if the storage were faster, it could've been 3457 * running and we'd not be idle. 3458 * 3459 * This has been extended to SMP, by doing the same for each CPU. This however 3460 * is broken. 3461 * 3462 * Imagine for instance the case where two tasks block on one CPU, only the one 3463 * CPU will have IO-wait accounted, while the other has regular idle. Even 3464 * though, if the storage were faster, both could've ran at the same time, 3465 * utilising both CPUs. 3466 * 3467 * This means, that when looking globally, the current IO-wait accounting on 3468 * SMP is a lower bound, by reason of under accounting. 3469 * 3470 * Worse, since the numbers are provided per CPU, they are sometimes 3471 * interpreted per CPU, and that is nonsensical. A blocked task isn't strictly 3472 * associated with any one particular CPU, it can wake to another CPU than it 3473 * blocked on. This means the per CPU IO-wait number is meaningless. 3474 * 3475 * Task CPU affinities can make all that even more 'interesting'. 3476 */ 3477 3478 unsigned long nr_iowait(void) 3479 { 3480 unsigned long i, sum = 0; 3481 3482 for_each_possible_cpu(i) 3483 sum += nr_iowait_cpu(i); 3484 3485 return sum; 3486 } 3487 3488 #ifdef CONFIG_SMP 3489 3490 /* 3491 * sched_exec - execve() is a valuable balancing opportunity, because at 3492 * this point the task has the smallest effective memory and cache footprint. 3493 */ 3494 void sched_exec(void) 3495 { 3496 struct task_struct *p = current; 3497 unsigned long flags; 3498 int dest_cpu; 3499 3500 raw_spin_lock_irqsave(&p->pi_lock, flags); 3501 dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), SD_BALANCE_EXEC, 0); 3502 if (dest_cpu == smp_processor_id()) 3503 goto unlock; 3504 3505 if (likely(cpu_active(dest_cpu))) { 3506 struct migration_arg arg = { p, dest_cpu }; 3507 3508 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 3509 stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg); 3510 return; 3511 } 3512 unlock: 3513 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 3514 } 3515 3516 #endif 3517 3518 DEFINE_PER_CPU(struct kernel_stat, kstat); 3519 DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat); 3520 3521 EXPORT_PER_CPU_SYMBOL(kstat); 3522 EXPORT_PER_CPU_SYMBOL(kernel_cpustat); 3523 3524 /* 3525 * The function fair_sched_class.update_curr accesses the struct curr 3526 * and its field curr->exec_start; when called from task_sched_runtime(), 3527 * we observe a high rate of cache misses in practice. 3528 * Prefetching this data results in improved performance. 3529 */ 3530 static inline void prefetch_curr_exec_start(struct task_struct *p) 3531 { 3532 #ifdef CONFIG_FAIR_GROUP_SCHED 3533 struct sched_entity *curr = (&p->se)->cfs_rq->curr; 3534 #else 3535 struct sched_entity *curr = (&task_rq(p)->cfs)->curr; 3536 #endif 3537 prefetch(curr); 3538 prefetch(&curr->exec_start); 3539 } 3540 3541 /* 3542 * Return accounted runtime for the task. 3543 * In case the task is currently running, return the runtime plus current's 3544 * pending runtime that have not been accounted yet. 3545 */ 3546 unsigned long long task_sched_runtime(struct task_struct *p) 3547 { 3548 struct rq_flags rf; 3549 struct rq *rq; 3550 u64 ns; 3551 3552 #if defined(CONFIG_64BIT) && defined(CONFIG_SMP) 3553 /* 3554 * 64-bit doesn't need locks to atomically read a 64-bit value. 3555 * So we have a optimization chance when the task's delta_exec is 0. 3556 * Reading ->on_cpu is racy, but this is ok. 3557 * 3558 * If we race with it leaving CPU, we'll take a lock. So we're correct. 3559 * If we race with it entering CPU, unaccounted time is 0. This is 3560 * indistinguishable from the read occurring a few cycles earlier. 3561 * If we see ->on_cpu without ->on_rq, the task is leaving, and has 3562 * been accounted, so we're correct here as well. 3563 */ 3564 if (!p->on_cpu || !task_on_rq_queued(p)) 3565 return p->se.sum_exec_runtime; 3566 #endif 3567 3568 rq = task_rq_lock(p, &rf); 3569 /* 3570 * Must be ->curr _and_ ->on_rq. If dequeued, we would 3571 * project cycles that may never be accounted to this 3572 * thread, breaking clock_gettime(). 3573 */ 3574 if (task_current(rq, p) && task_on_rq_queued(p)) { 3575 prefetch_curr_exec_start(p); 3576 update_rq_clock(rq); 3577 p->sched_class->update_curr(rq); 3578 } 3579 ns = p->se.sum_exec_runtime; 3580 task_rq_unlock(rq, p, &rf); 3581 3582 return ns; 3583 } 3584 3585 /* 3586 * This function gets called by the timer code, with HZ frequency. 3587 * We call it with interrupts disabled. 3588 */ 3589 void scheduler_tick(void) 3590 { 3591 int cpu = smp_processor_id(); 3592 struct rq *rq = cpu_rq(cpu); 3593 struct task_struct *curr = rq->curr; 3594 struct rq_flags rf; 3595 3596 sched_clock_tick(); 3597 3598 rq_lock(rq, &rf); 3599 3600 update_rq_clock(rq); 3601 curr->sched_class->task_tick(rq, curr, 0); 3602 calc_global_load_tick(rq); 3603 psi_task_tick(rq); 3604 3605 rq_unlock(rq, &rf); 3606 3607 perf_event_task_tick(); 3608 3609 #ifdef CONFIG_SMP 3610 rq->idle_balance = idle_cpu(cpu); 3611 trigger_load_balance(rq); 3612 #endif 3613 } 3614 3615 #ifdef CONFIG_NO_HZ_FULL 3616 3617 struct tick_work { 3618 int cpu; 3619 atomic_t state; 3620 struct delayed_work work; 3621 }; 3622 /* Values for ->state, see diagram below. */ 3623 #define TICK_SCHED_REMOTE_OFFLINE 0 3624 #define TICK_SCHED_REMOTE_OFFLINING 1 3625 #define TICK_SCHED_REMOTE_RUNNING 2 3626 3627 /* 3628 * State diagram for ->state: 3629 * 3630 * 3631 * TICK_SCHED_REMOTE_OFFLINE 3632 * | ^ 3633 * | | 3634 * | | sched_tick_remote() 3635 * | | 3636 * | | 3637 * +--TICK_SCHED_REMOTE_OFFLINING 3638 * | ^ 3639 * | | 3640 * sched_tick_start() | | sched_tick_stop() 3641 * | | 3642 * V | 3643 * TICK_SCHED_REMOTE_RUNNING 3644 * 3645 * 3646 * Other transitions get WARN_ON_ONCE(), except that sched_tick_remote() 3647 * and sched_tick_start() are happy to leave the state in RUNNING. 3648 */ 3649 3650 static struct tick_work __percpu *tick_work_cpu; 3651 3652 static void sched_tick_remote(struct work_struct *work) 3653 { 3654 struct delayed_work *dwork = to_delayed_work(work); 3655 struct tick_work *twork = container_of(dwork, struct tick_work, work); 3656 int cpu = twork->cpu; 3657 struct rq *rq = cpu_rq(cpu); 3658 struct task_struct *curr; 3659 struct rq_flags rf; 3660 u64 delta; 3661 int os; 3662 3663 /* 3664 * Handle the tick only if it appears the remote CPU is running in full 3665 * dynticks mode. The check is racy by nature, but missing a tick or 3666 * having one too much is no big deal because the scheduler tick updates 3667 * statistics and checks timeslices in a time-independent way, regardless 3668 * of when exactly it is running. 3669 */ 3670 if (idle_cpu(cpu) || !tick_nohz_tick_stopped_cpu(cpu)) 3671 goto out_requeue; 3672 3673 rq_lock_irq(rq, &rf); 3674 curr = rq->curr; 3675 if (is_idle_task(curr) || cpu_is_offline(cpu)) 3676 goto out_unlock; 3677 3678 update_rq_clock(rq); 3679 delta = rq_clock_task(rq) - curr->se.exec_start; 3680 3681 /* 3682 * Make sure the next tick runs within a reasonable 3683 * amount of time. 3684 */ 3685 WARN_ON_ONCE(delta > (u64)NSEC_PER_SEC * 3); 3686 curr->sched_class->task_tick(rq, curr, 0); 3687 3688 out_unlock: 3689 rq_unlock_irq(rq, &rf); 3690 3691 out_requeue: 3692 /* 3693 * Run the remote tick once per second (1Hz). This arbitrary 3694 * frequency is large enough to avoid overload but short enough 3695 * to keep scheduler internal stats reasonably up to date. But 3696 * first update state to reflect hotplug activity if required. 3697 */ 3698 os = atomic_fetch_add_unless(&twork->state, -1, TICK_SCHED_REMOTE_RUNNING); 3699 WARN_ON_ONCE(os == TICK_SCHED_REMOTE_OFFLINE); 3700 if (os == TICK_SCHED_REMOTE_RUNNING) 3701 queue_delayed_work(system_unbound_wq, dwork, HZ); 3702 } 3703 3704 static void sched_tick_start(int cpu) 3705 { 3706 int os; 3707 struct tick_work *twork; 3708 3709 if (housekeeping_cpu(cpu, HK_FLAG_TICK)) 3710 return; 3711 3712 WARN_ON_ONCE(!tick_work_cpu); 3713 3714 twork = per_cpu_ptr(tick_work_cpu, cpu); 3715 os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_RUNNING); 3716 WARN_ON_ONCE(os == TICK_SCHED_REMOTE_RUNNING); 3717 if (os == TICK_SCHED_REMOTE_OFFLINE) { 3718 twork->cpu = cpu; 3719 INIT_DELAYED_WORK(&twork->work, sched_tick_remote); 3720 queue_delayed_work(system_unbound_wq, &twork->work, HZ); 3721 } 3722 } 3723 3724 #ifdef CONFIG_HOTPLUG_CPU 3725 static void sched_tick_stop(int cpu) 3726 { 3727 struct tick_work *twork; 3728 int os; 3729 3730 if (housekeeping_cpu(cpu, HK_FLAG_TICK)) 3731 return; 3732 3733 WARN_ON_ONCE(!tick_work_cpu); 3734 3735 twork = per_cpu_ptr(tick_work_cpu, cpu); 3736 /* There cannot be competing actions, but don't rely on stop-machine. */ 3737 os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_OFFLINING); 3738 WARN_ON_ONCE(os != TICK_SCHED_REMOTE_RUNNING); 3739 /* Don't cancel, as this would mess up the state machine. */ 3740 } 3741 #endif /* CONFIG_HOTPLUG_CPU */ 3742 3743 int __init sched_tick_offload_init(void) 3744 { 3745 tick_work_cpu = alloc_percpu(struct tick_work); 3746 BUG_ON(!tick_work_cpu); 3747 return 0; 3748 } 3749 3750 #else /* !CONFIG_NO_HZ_FULL */ 3751 static inline void sched_tick_start(int cpu) { } 3752 static inline void sched_tick_stop(int cpu) { } 3753 #endif 3754 3755 #if defined(CONFIG_PREEMPTION) && (defined(CONFIG_DEBUG_PREEMPT) || \ 3756 defined(CONFIG_TRACE_PREEMPT_TOGGLE)) 3757 /* 3758 * If the value passed in is equal to the current preempt count 3759 * then we just disabled preemption. Start timing the latency. 3760 */ 3761 static inline void preempt_latency_start(int val) 3762 { 3763 if (preempt_count() == val) { 3764 unsigned long ip = get_lock_parent_ip(); 3765 #ifdef CONFIG_DEBUG_PREEMPT 3766 current->preempt_disable_ip = ip; 3767 #endif 3768 trace_preempt_off(CALLER_ADDR0, ip); 3769 } 3770 } 3771 3772 void preempt_count_add(int val) 3773 { 3774 #ifdef CONFIG_DEBUG_PREEMPT 3775 /* 3776 * Underflow? 3777 */ 3778 if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) 3779 return; 3780 #endif 3781 __preempt_count_add(val); 3782 #ifdef CONFIG_DEBUG_PREEMPT 3783 /* 3784 * Spinlock count overflowing soon? 3785 */ 3786 DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >= 3787 PREEMPT_MASK - 10); 3788 #endif 3789 preempt_latency_start(val); 3790 } 3791 EXPORT_SYMBOL(preempt_count_add); 3792 NOKPROBE_SYMBOL(preempt_count_add); 3793 3794 /* 3795 * If the value passed in equals to the current preempt count 3796 * then we just enabled preemption. Stop timing the latency. 3797 */ 3798 static inline void preempt_latency_stop(int val) 3799 { 3800 if (preempt_count() == val) 3801 trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip()); 3802 } 3803 3804 void preempt_count_sub(int val) 3805 { 3806 #ifdef CONFIG_DEBUG_PREEMPT 3807 /* 3808 * Underflow? 3809 */ 3810 if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) 3811 return; 3812 /* 3813 * Is the spinlock portion underflowing? 3814 */ 3815 if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) && 3816 !(preempt_count() & PREEMPT_MASK))) 3817 return; 3818 #endif 3819 3820 preempt_latency_stop(val); 3821 __preempt_count_sub(val); 3822 } 3823 EXPORT_SYMBOL(preempt_count_sub); 3824 NOKPROBE_SYMBOL(preempt_count_sub); 3825 3826 #else 3827 static inline void preempt_latency_start(int val) { } 3828 static inline void preempt_latency_stop(int val) { } 3829 #endif 3830 3831 static inline unsigned long get_preempt_disable_ip(struct task_struct *p) 3832 { 3833 #ifdef CONFIG_DEBUG_PREEMPT 3834 return p->preempt_disable_ip; 3835 #else 3836 return 0; 3837 #endif 3838 } 3839 3840 /* 3841 * Print scheduling while atomic bug: 3842 */ 3843 static noinline void __schedule_bug(struct task_struct *prev) 3844 { 3845 /* Save this before calling printk(), since that will clobber it */ 3846 unsigned long preempt_disable_ip = get_preempt_disable_ip(current); 3847 3848 if (oops_in_progress) 3849 return; 3850 3851 printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n", 3852 prev->comm, prev->pid, preempt_count()); 3853 3854 debug_show_held_locks(prev); 3855 print_modules(); 3856 if (irqs_disabled()) 3857 print_irqtrace_events(prev); 3858 if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) 3859 && in_atomic_preempt_off()) { 3860 pr_err("Preemption disabled at:"); 3861 print_ip_sym(preempt_disable_ip); 3862 pr_cont("\n"); 3863 } 3864 if (panic_on_warn) 3865 panic("scheduling while atomic\n"); 3866 3867 dump_stack(); 3868 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 3869 } 3870 3871 /* 3872 * Various schedule()-time debugging checks and statistics: 3873 */ 3874 static inline void schedule_debug(struct task_struct *prev, bool preempt) 3875 { 3876 #ifdef CONFIG_SCHED_STACK_END_CHECK 3877 if (task_stack_end_corrupted(prev)) 3878 panic("corrupted stack end detected inside scheduler\n"); 3879 #endif 3880 3881 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP 3882 if (!preempt && prev->state && prev->non_block_count) { 3883 printk(KERN_ERR "BUG: scheduling in a non-blocking section: %s/%d/%i\n", 3884 prev->comm, prev->pid, prev->non_block_count); 3885 dump_stack(); 3886 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 3887 } 3888 #endif 3889 3890 if (unlikely(in_atomic_preempt_off())) { 3891 __schedule_bug(prev); 3892 preempt_count_set(PREEMPT_DISABLED); 3893 } 3894 rcu_sleep_check(); 3895 3896 profile_hit(SCHED_PROFILING, __builtin_return_address(0)); 3897 3898 schedstat_inc(this_rq()->sched_count); 3899 } 3900 3901 /* 3902 * Pick up the highest-prio task: 3903 */ 3904 static inline struct task_struct * 3905 pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 3906 { 3907 const struct sched_class *class; 3908 struct task_struct *p; 3909 3910 /* 3911 * Optimization: we know that if all tasks are in the fair class we can 3912 * call that function directly, but only if the @prev task wasn't of a 3913 * higher scheduling class, because otherwise those loose the 3914 * opportunity to pull in more work from other CPUs. 3915 */ 3916 if (likely((prev->sched_class == &idle_sched_class || 3917 prev->sched_class == &fair_sched_class) && 3918 rq->nr_running == rq->cfs.h_nr_running)) { 3919 3920 p = fair_sched_class.pick_next_task(rq, prev, rf); 3921 if (unlikely(p == RETRY_TASK)) 3922 goto restart; 3923 3924 /* Assumes fair_sched_class->next == idle_sched_class */ 3925 if (unlikely(!p)) 3926 p = idle_sched_class.pick_next_task(rq, prev, rf); 3927 3928 return p; 3929 } 3930 3931 restart: 3932 /* 3933 * Ensure that we put DL/RT tasks before the pick loop, such that they 3934 * can PULL higher prio tasks when we lower the RQ 'priority'. 3935 */ 3936 prev->sched_class->put_prev_task(rq, prev, rf); 3937 if (!rq->nr_running) 3938 newidle_balance(rq, rf); 3939 3940 for_each_class(class) { 3941 p = class->pick_next_task(rq, NULL, NULL); 3942 if (p) 3943 return p; 3944 } 3945 3946 /* The idle class should always have a runnable task: */ 3947 BUG(); 3948 } 3949 3950 /* 3951 * __schedule() is the main scheduler function. 3952 * 3953 * The main means of driving the scheduler and thus entering this function are: 3954 * 3955 * 1. Explicit blocking: mutex, semaphore, waitqueue, etc. 3956 * 3957 * 2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return 3958 * paths. For example, see arch/x86/entry_64.S. 3959 * 3960 * To drive preemption between tasks, the scheduler sets the flag in timer 3961 * interrupt handler scheduler_tick(). 3962 * 3963 * 3. Wakeups don't really cause entry into schedule(). They add a 3964 * task to the run-queue and that's it. 3965 * 3966 * Now, if the new task added to the run-queue preempts the current 3967 * task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets 3968 * called on the nearest possible occasion: 3969 * 3970 * - If the kernel is preemptible (CONFIG_PREEMPTION=y): 3971 * 3972 * - in syscall or exception context, at the next outmost 3973 * preempt_enable(). (this might be as soon as the wake_up()'s 3974 * spin_unlock()!) 3975 * 3976 * - in IRQ context, return from interrupt-handler to 3977 * preemptible context 3978 * 3979 * - If the kernel is not preemptible (CONFIG_PREEMPTION is not set) 3980 * then at the next: 3981 * 3982 * - cond_resched() call 3983 * - explicit schedule() call 3984 * - return from syscall or exception to user-space 3985 * - return from interrupt-handler to user-space 3986 * 3987 * WARNING: must be called with preemption disabled! 3988 */ 3989 static void __sched notrace __schedule(bool preempt) 3990 { 3991 struct task_struct *prev, *next; 3992 unsigned long *switch_count; 3993 struct rq_flags rf; 3994 struct rq *rq; 3995 int cpu; 3996 3997 cpu = smp_processor_id(); 3998 rq = cpu_rq(cpu); 3999 prev = rq->curr; 4000 4001 schedule_debug(prev, preempt); 4002 4003 if (sched_feat(HRTICK)) 4004 hrtick_clear(rq); 4005 4006 local_irq_disable(); 4007 rcu_note_context_switch(preempt); 4008 4009 /* 4010 * Make sure that signal_pending_state()->signal_pending() below 4011 * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE) 4012 * done by the caller to avoid the race with signal_wake_up(). 4013 * 4014 * The membarrier system call requires a full memory barrier 4015 * after coming from user-space, before storing to rq->curr. 4016 */ 4017 rq_lock(rq, &rf); 4018 smp_mb__after_spinlock(); 4019 4020 /* Promote REQ to ACT */ 4021 rq->clock_update_flags <<= 1; 4022 update_rq_clock(rq); 4023 4024 switch_count = &prev->nivcsw; 4025 if (!preempt && prev->state) { 4026 if (signal_pending_state(prev->state, prev)) { 4027 prev->state = TASK_RUNNING; 4028 } else { 4029 deactivate_task(rq, prev, DEQUEUE_SLEEP | DEQUEUE_NOCLOCK); 4030 4031 if (prev->in_iowait) { 4032 atomic_inc(&rq->nr_iowait); 4033 delayacct_blkio_start(); 4034 } 4035 } 4036 switch_count = &prev->nvcsw; 4037 } 4038 4039 next = pick_next_task(rq, prev, &rf); 4040 clear_tsk_need_resched(prev); 4041 clear_preempt_need_resched(); 4042 4043 if (likely(prev != next)) { 4044 rq->nr_switches++; 4045 /* 4046 * RCU users of rcu_dereference(rq->curr) may not see 4047 * changes to task_struct made by pick_next_task(). 4048 */ 4049 RCU_INIT_POINTER(rq->curr, next); 4050 /* 4051 * The membarrier system call requires each architecture 4052 * to have a full memory barrier after updating 4053 * rq->curr, before returning to user-space. 4054 * 4055 * Here are the schemes providing that barrier on the 4056 * various architectures: 4057 * - mm ? switch_mm() : mmdrop() for x86, s390, sparc, PowerPC. 4058 * switch_mm() rely on membarrier_arch_switch_mm() on PowerPC. 4059 * - finish_lock_switch() for weakly-ordered 4060 * architectures where spin_unlock is a full barrier, 4061 * - switch_to() for arm64 (weakly-ordered, spin_unlock 4062 * is a RELEASE barrier), 4063 */ 4064 ++*switch_count; 4065 4066 trace_sched_switch(preempt, prev, next); 4067 4068 /* Also unlocks the rq: */ 4069 rq = context_switch(rq, prev, next, &rf); 4070 } else { 4071 rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP); 4072 rq_unlock_irq(rq, &rf); 4073 } 4074 4075 balance_callback(rq); 4076 } 4077 4078 void __noreturn do_task_dead(void) 4079 { 4080 /* Causes final put_task_struct in finish_task_switch(): */ 4081 set_special_state(TASK_DEAD); 4082 4083 /* Tell freezer to ignore us: */ 4084 current->flags |= PF_NOFREEZE; 4085 4086 __schedule(false); 4087 BUG(); 4088 4089 /* Avoid "noreturn function does return" - but don't continue if BUG() is a NOP: */ 4090 for (;;) 4091 cpu_relax(); 4092 } 4093 4094 static inline void sched_submit_work(struct task_struct *tsk) 4095 { 4096 if (!tsk->state) 4097 return; 4098 4099 /* 4100 * If a worker went to sleep, notify and ask workqueue whether 4101 * it wants to wake up a task to maintain concurrency. 4102 * As this function is called inside the schedule() context, 4103 * we disable preemption to avoid it calling schedule() again 4104 * in the possible wakeup of a kworker. 4105 */ 4106 if (tsk->flags & PF_WQ_WORKER) { 4107 preempt_disable(); 4108 wq_worker_sleeping(tsk); 4109 preempt_enable_no_resched(); 4110 } 4111 4112 if (tsk_is_pi_blocked(tsk)) 4113 return; 4114 4115 /* 4116 * If we are going to sleep and we have plugged IO queued, 4117 * make sure to submit it to avoid deadlocks. 4118 */ 4119 if (blk_needs_flush_plug(tsk)) 4120 blk_schedule_flush_plug(tsk); 4121 } 4122 4123 static void sched_update_worker(struct task_struct *tsk) 4124 { 4125 if (tsk->flags & PF_WQ_WORKER) 4126 wq_worker_running(tsk); 4127 } 4128 4129 asmlinkage __visible void __sched schedule(void) 4130 { 4131 struct task_struct *tsk = current; 4132 4133 sched_submit_work(tsk); 4134 do { 4135 preempt_disable(); 4136 __schedule(false); 4137 sched_preempt_enable_no_resched(); 4138 } while (need_resched()); 4139 sched_update_worker(tsk); 4140 } 4141 EXPORT_SYMBOL(schedule); 4142 4143 /* 4144 * synchronize_rcu_tasks() makes sure that no task is stuck in preempted 4145 * state (have scheduled out non-voluntarily) by making sure that all 4146 * tasks have either left the run queue or have gone into user space. 4147 * As idle tasks do not do either, they must not ever be preempted 4148 * (schedule out non-voluntarily). 4149 * 4150 * schedule_idle() is similar to schedule_preempt_disable() except that it 4151 * never enables preemption because it does not call sched_submit_work(). 4152 */ 4153 void __sched schedule_idle(void) 4154 { 4155 /* 4156 * As this skips calling sched_submit_work(), which the idle task does 4157 * regardless because that function is a nop when the task is in a 4158 * TASK_RUNNING state, make sure this isn't used someplace that the 4159 * current task can be in any other state. Note, idle is always in the 4160 * TASK_RUNNING state. 4161 */ 4162 WARN_ON_ONCE(current->state); 4163 do { 4164 __schedule(false); 4165 } while (need_resched()); 4166 } 4167 4168 #ifdef CONFIG_CONTEXT_TRACKING 4169 asmlinkage __visible void __sched schedule_user(void) 4170 { 4171 /* 4172 * If we come here after a random call to set_need_resched(), 4173 * or we have been woken up remotely but the IPI has not yet arrived, 4174 * we haven't yet exited the RCU idle mode. Do it here manually until 4175 * we find a better solution. 4176 * 4177 * NB: There are buggy callers of this function. Ideally we 4178 * should warn if prev_state != CONTEXT_USER, but that will trigger 4179 * too frequently to make sense yet. 4180 */ 4181 enum ctx_state prev_state = exception_enter(); 4182 schedule(); 4183 exception_exit(prev_state); 4184 } 4185 #endif 4186 4187 /** 4188 * schedule_preempt_disabled - called with preemption disabled 4189 * 4190 * Returns with preemption disabled. Note: preempt_count must be 1 4191 */ 4192 void __sched schedule_preempt_disabled(void) 4193 { 4194 sched_preempt_enable_no_resched(); 4195 schedule(); 4196 preempt_disable(); 4197 } 4198 4199 static void __sched notrace preempt_schedule_common(void) 4200 { 4201 do { 4202 /* 4203 * Because the function tracer can trace preempt_count_sub() 4204 * and it also uses preempt_enable/disable_notrace(), if 4205 * NEED_RESCHED is set, the preempt_enable_notrace() called 4206 * by the function tracer will call this function again and 4207 * cause infinite recursion. 4208 * 4209 * Preemption must be disabled here before the function 4210 * tracer can trace. Break up preempt_disable() into two 4211 * calls. One to disable preemption without fear of being 4212 * traced. The other to still record the preemption latency, 4213 * which can also be traced by the function tracer. 4214 */ 4215 preempt_disable_notrace(); 4216 preempt_latency_start(1); 4217 __schedule(true); 4218 preempt_latency_stop(1); 4219 preempt_enable_no_resched_notrace(); 4220 4221 /* 4222 * Check again in case we missed a preemption opportunity 4223 * between schedule and now. 4224 */ 4225 } while (need_resched()); 4226 } 4227 4228 #ifdef CONFIG_PREEMPTION 4229 /* 4230 * This is the entry point to schedule() from in-kernel preemption 4231 * off of preempt_enable. 4232 */ 4233 asmlinkage __visible void __sched notrace preempt_schedule(void) 4234 { 4235 /* 4236 * If there is a non-zero preempt_count or interrupts are disabled, 4237 * we do not want to preempt the current task. Just return.. 4238 */ 4239 if (likely(!preemptible())) 4240 return; 4241 4242 preempt_schedule_common(); 4243 } 4244 NOKPROBE_SYMBOL(preempt_schedule); 4245 EXPORT_SYMBOL(preempt_schedule); 4246 4247 /** 4248 * preempt_schedule_notrace - preempt_schedule called by tracing 4249 * 4250 * The tracing infrastructure uses preempt_enable_notrace to prevent 4251 * recursion and tracing preempt enabling caused by the tracing 4252 * infrastructure itself. But as tracing can happen in areas coming 4253 * from userspace or just about to enter userspace, a preempt enable 4254 * can occur before user_exit() is called. This will cause the scheduler 4255 * to be called when the system is still in usermode. 4256 * 4257 * To prevent this, the preempt_enable_notrace will use this function 4258 * instead of preempt_schedule() to exit user context if needed before 4259 * calling the scheduler. 4260 */ 4261 asmlinkage __visible void __sched notrace preempt_schedule_notrace(void) 4262 { 4263 enum ctx_state prev_ctx; 4264 4265 if (likely(!preemptible())) 4266 return; 4267 4268 do { 4269 /* 4270 * Because the function tracer can trace preempt_count_sub() 4271 * and it also uses preempt_enable/disable_notrace(), if 4272 * NEED_RESCHED is set, the preempt_enable_notrace() called 4273 * by the function tracer will call this function again and 4274 * cause infinite recursion. 4275 * 4276 * Preemption must be disabled here before the function 4277 * tracer can trace. Break up preempt_disable() into two 4278 * calls. One to disable preemption without fear of being 4279 * traced. The other to still record the preemption latency, 4280 * which can also be traced by the function tracer. 4281 */ 4282 preempt_disable_notrace(); 4283 preempt_latency_start(1); 4284 /* 4285 * Needs preempt disabled in case user_exit() is traced 4286 * and the tracer calls preempt_enable_notrace() causing 4287 * an infinite recursion. 4288 */ 4289 prev_ctx = exception_enter(); 4290 __schedule(true); 4291 exception_exit(prev_ctx); 4292 4293 preempt_latency_stop(1); 4294 preempt_enable_no_resched_notrace(); 4295 } while (need_resched()); 4296 } 4297 EXPORT_SYMBOL_GPL(preempt_schedule_notrace); 4298 4299 #endif /* CONFIG_PREEMPTION */ 4300 4301 /* 4302 * This is the entry point to schedule() from kernel preemption 4303 * off of irq context. 4304 * Note, that this is called and return with irqs disabled. This will 4305 * protect us against recursive calling from irq. 4306 */ 4307 asmlinkage __visible void __sched preempt_schedule_irq(void) 4308 { 4309 enum ctx_state prev_state; 4310 4311 /* Catch callers which need to be fixed */ 4312 BUG_ON(preempt_count() || !irqs_disabled()); 4313 4314 prev_state = exception_enter(); 4315 4316 do { 4317 preempt_disable(); 4318 local_irq_enable(); 4319 __schedule(true); 4320 local_irq_disable(); 4321 sched_preempt_enable_no_resched(); 4322 } while (need_resched()); 4323 4324 exception_exit(prev_state); 4325 } 4326 4327 int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flags, 4328 void *key) 4329 { 4330 return try_to_wake_up(curr->private, mode, wake_flags); 4331 } 4332 EXPORT_SYMBOL(default_wake_function); 4333 4334 #ifdef CONFIG_RT_MUTEXES 4335 4336 static inline int __rt_effective_prio(struct task_struct *pi_task, int prio) 4337 { 4338 if (pi_task) 4339 prio = min(prio, pi_task->prio); 4340 4341 return prio; 4342 } 4343 4344 static inline int rt_effective_prio(struct task_struct *p, int prio) 4345 { 4346 struct task_struct *pi_task = rt_mutex_get_top_task(p); 4347 4348 return __rt_effective_prio(pi_task, prio); 4349 } 4350 4351 /* 4352 * rt_mutex_setprio - set the current priority of a task 4353 * @p: task to boost 4354 * @pi_task: donor task 4355 * 4356 * This function changes the 'effective' priority of a task. It does 4357 * not touch ->normal_prio like __setscheduler(). 4358 * 4359 * Used by the rt_mutex code to implement priority inheritance 4360 * logic. Call site only calls if the priority of the task changed. 4361 */ 4362 void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) 4363 { 4364 int prio, oldprio, queued, running, queue_flag = 4365 DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; 4366 const struct sched_class *prev_class; 4367 struct rq_flags rf; 4368 struct rq *rq; 4369 4370 /* XXX used to be waiter->prio, not waiter->task->prio */ 4371 prio = __rt_effective_prio(pi_task, p->normal_prio); 4372 4373 /* 4374 * If nothing changed; bail early. 4375 */ 4376 if (p->pi_top_task == pi_task && prio == p->prio && !dl_prio(prio)) 4377 return; 4378 4379 rq = __task_rq_lock(p, &rf); 4380 update_rq_clock(rq); 4381 /* 4382 * Set under pi_lock && rq->lock, such that the value can be used under 4383 * either lock. 4384 * 4385 * Note that there is loads of tricky to make this pointer cache work 4386 * right. rt_mutex_slowunlock()+rt_mutex_postunlock() work together to 4387 * ensure a task is de-boosted (pi_task is set to NULL) before the 4388 * task is allowed to run again (and can exit). This ensures the pointer 4389 * points to a blocked task -- which guaratees the task is present. 4390 */ 4391 p->pi_top_task = pi_task; 4392 4393 /* 4394 * For FIFO/RR we only need to set prio, if that matches we're done. 4395 */ 4396 if (prio == p->prio && !dl_prio(prio)) 4397 goto out_unlock; 4398 4399 /* 4400 * Idle task boosting is a nono in general. There is one 4401 * exception, when PREEMPT_RT and NOHZ is active: 4402 * 4403 * The idle task calls get_next_timer_interrupt() and holds 4404 * the timer wheel base->lock on the CPU and another CPU wants 4405 * to access the timer (probably to cancel it). We can safely 4406 * ignore the boosting request, as the idle CPU runs this code 4407 * with interrupts disabled and will complete the lock 4408 * protected section without being interrupted. So there is no 4409 * real need to boost. 4410 */ 4411 if (unlikely(p == rq->idle)) { 4412 WARN_ON(p != rq->curr); 4413 WARN_ON(p->pi_blocked_on); 4414 goto out_unlock; 4415 } 4416 4417 trace_sched_pi_setprio(p, pi_task); 4418 oldprio = p->prio; 4419 4420 if (oldprio == prio) 4421 queue_flag &= ~DEQUEUE_MOVE; 4422 4423 prev_class = p->sched_class; 4424 queued = task_on_rq_queued(p); 4425 running = task_current(rq, p); 4426 if (queued) 4427 dequeue_task(rq, p, queue_flag); 4428 if (running) 4429 put_prev_task(rq, p); 4430 4431 /* 4432 * Boosting condition are: 4433 * 1. -rt task is running and holds mutex A 4434 * --> -dl task blocks on mutex A 4435 * 4436 * 2. -dl task is running and holds mutex A 4437 * --> -dl task blocks on mutex A and could preempt the 4438 * running task 4439 */ 4440 if (dl_prio(prio)) { 4441 if (!dl_prio(p->normal_prio) || 4442 (pi_task && dl_entity_preempt(&pi_task->dl, &p->dl))) { 4443 p->dl.dl_boosted = 1; 4444 queue_flag |= ENQUEUE_REPLENISH; 4445 } else 4446 p->dl.dl_boosted = 0; 4447 p->sched_class = &dl_sched_class; 4448 } else if (rt_prio(prio)) { 4449 if (dl_prio(oldprio)) 4450 p->dl.dl_boosted = 0; 4451 if (oldprio < prio) 4452 queue_flag |= ENQUEUE_HEAD; 4453 p->sched_class = &rt_sched_class; 4454 } else { 4455 if (dl_prio(oldprio)) 4456 p->dl.dl_boosted = 0; 4457 if (rt_prio(oldprio)) 4458 p->rt.timeout = 0; 4459 p->sched_class = &fair_sched_class; 4460 } 4461 4462 p->prio = prio; 4463 4464 if (queued) 4465 enqueue_task(rq, p, queue_flag); 4466 if (running) 4467 set_next_task(rq, p); 4468 4469 check_class_changed(rq, p, prev_class, oldprio); 4470 out_unlock: 4471 /* Avoid rq from going away on us: */ 4472 preempt_disable(); 4473 __task_rq_unlock(rq, &rf); 4474 4475 balance_callback(rq); 4476 preempt_enable(); 4477 } 4478 #else 4479 static inline int rt_effective_prio(struct task_struct *p, int prio) 4480 { 4481 return prio; 4482 } 4483 #endif 4484 4485 void set_user_nice(struct task_struct *p, long nice) 4486 { 4487 bool queued, running; 4488 int old_prio, delta; 4489 struct rq_flags rf; 4490 struct rq *rq; 4491 4492 if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE) 4493 return; 4494 /* 4495 * We have to be careful, if called from sys_setpriority(), 4496 * the task might be in the middle of scheduling on another CPU. 4497 */ 4498 rq = task_rq_lock(p, &rf); 4499 update_rq_clock(rq); 4500 4501 /* 4502 * The RT priorities are set via sched_setscheduler(), but we still 4503 * allow the 'normal' nice value to be set - but as expected 4504 * it wont have any effect on scheduling until the task is 4505 * SCHED_DEADLINE, SCHED_FIFO or SCHED_RR: 4506 */ 4507 if (task_has_dl_policy(p) || task_has_rt_policy(p)) { 4508 p->static_prio = NICE_TO_PRIO(nice); 4509 goto out_unlock; 4510 } 4511 queued = task_on_rq_queued(p); 4512 running = task_current(rq, p); 4513 if (queued) 4514 dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); 4515 if (running) 4516 put_prev_task(rq, p); 4517 4518 p->static_prio = NICE_TO_PRIO(nice); 4519 set_load_weight(p, true); 4520 old_prio = p->prio; 4521 p->prio = effective_prio(p); 4522 delta = p->prio - old_prio; 4523 4524 if (queued) { 4525 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); 4526 /* 4527 * If the task increased its priority or is running and 4528 * lowered its priority, then reschedule its CPU: 4529 */ 4530 if (delta < 0 || (delta > 0 && task_running(rq, p))) 4531 resched_curr(rq); 4532 } 4533 if (running) 4534 set_next_task(rq, p); 4535 out_unlock: 4536 task_rq_unlock(rq, p, &rf); 4537 } 4538 EXPORT_SYMBOL(set_user_nice); 4539 4540 /* 4541 * can_nice - check if a task can reduce its nice value 4542 * @p: task 4543 * @nice: nice value 4544 */ 4545 int can_nice(const struct task_struct *p, const int nice) 4546 { 4547 /* Convert nice value [19,-20] to rlimit style value [1,40]: */ 4548 int nice_rlim = nice_to_rlimit(nice); 4549 4550 return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) || 4551 capable(CAP_SYS_NICE)); 4552 } 4553 4554 #ifdef __ARCH_WANT_SYS_NICE 4555 4556 /* 4557 * sys_nice - change the priority of the current process. 4558 * @increment: priority increment 4559 * 4560 * sys_setpriority is a more generic, but much slower function that 4561 * does similar things. 4562 */ 4563 SYSCALL_DEFINE1(nice, int, increment) 4564 { 4565 long nice, retval; 4566 4567 /* 4568 * Setpriority might change our priority at the same moment. 4569 * We don't have to worry. Conceptually one call occurs first 4570 * and we have a single winner. 4571 */ 4572 increment = clamp(increment, -NICE_WIDTH, NICE_WIDTH); 4573 nice = task_nice(current) + increment; 4574 4575 nice = clamp_val(nice, MIN_NICE, MAX_NICE); 4576 if (increment < 0 && !can_nice(current, nice)) 4577 return -EPERM; 4578 4579 retval = security_task_setnice(current, nice); 4580 if (retval) 4581 return retval; 4582 4583 set_user_nice(current, nice); 4584 return 0; 4585 } 4586 4587 #endif 4588 4589 /** 4590 * task_prio - return the priority value of a given task. 4591 * @p: the task in question. 4592 * 4593 * Return: The priority value as seen by users in /proc. 4594 * RT tasks are offset by -200. Normal tasks are centered 4595 * around 0, value goes from -16 to +15. 4596 */ 4597 int task_prio(const struct task_struct *p) 4598 { 4599 return p->prio - MAX_RT_PRIO; 4600 } 4601 4602 /** 4603 * idle_cpu - is a given CPU idle currently? 4604 * @cpu: the processor in question. 4605 * 4606 * Return: 1 if the CPU is currently idle. 0 otherwise. 4607 */ 4608 int idle_cpu(int cpu) 4609 { 4610 struct rq *rq = cpu_rq(cpu); 4611 4612 if (rq->curr != rq->idle) 4613 return 0; 4614 4615 if (rq->nr_running) 4616 return 0; 4617 4618 #ifdef CONFIG_SMP 4619 if (!llist_empty(&rq->wake_list)) 4620 return 0; 4621 #endif 4622 4623 return 1; 4624 } 4625 4626 /** 4627 * available_idle_cpu - is a given CPU idle for enqueuing work. 4628 * @cpu: the CPU in question. 4629 * 4630 * Return: 1 if the CPU is currently idle. 0 otherwise. 4631 */ 4632 int available_idle_cpu(int cpu) 4633 { 4634 if (!idle_cpu(cpu)) 4635 return 0; 4636 4637 if (vcpu_is_preempted(cpu)) 4638 return 0; 4639 4640 return 1; 4641 } 4642 4643 /** 4644 * idle_task - return the idle task for a given CPU. 4645 * @cpu: the processor in question. 4646 * 4647 * Return: The idle task for the CPU @cpu. 4648 */ 4649 struct task_struct *idle_task(int cpu) 4650 { 4651 return cpu_rq(cpu)->idle; 4652 } 4653 4654 /** 4655 * find_process_by_pid - find a process with a matching PID value. 4656 * @pid: the pid in question. 4657 * 4658 * The task of @pid, if found. %NULL otherwise. 4659 */ 4660 static struct task_struct *find_process_by_pid(pid_t pid) 4661 { 4662 return pid ? find_task_by_vpid(pid) : current; 4663 } 4664 4665 /* 4666 * sched_setparam() passes in -1 for its policy, to let the functions 4667 * it calls know not to change it. 4668 */ 4669 #define SETPARAM_POLICY -1 4670 4671 static void __setscheduler_params(struct task_struct *p, 4672 const struct sched_attr *attr) 4673 { 4674 int policy = attr->sched_policy; 4675 4676 if (policy == SETPARAM_POLICY) 4677 policy = p->policy; 4678 4679 p->policy = policy; 4680 4681 if (dl_policy(policy)) 4682 __setparam_dl(p, attr); 4683 else if (fair_policy(policy)) 4684 p->static_prio = NICE_TO_PRIO(attr->sched_nice); 4685 4686 /* 4687 * __sched_setscheduler() ensures attr->sched_priority == 0 when 4688 * !rt_policy. Always setting this ensures that things like 4689 * getparam()/getattr() don't report silly values for !rt tasks. 4690 */ 4691 p->rt_priority = attr->sched_priority; 4692 p->normal_prio = normal_prio(p); 4693 set_load_weight(p, true); 4694 } 4695 4696 /* Actually do priority change: must hold pi & rq lock. */ 4697 static void __setscheduler(struct rq *rq, struct task_struct *p, 4698 const struct sched_attr *attr, bool keep_boost) 4699 { 4700 /* 4701 * If params can't change scheduling class changes aren't allowed 4702 * either. 4703 */ 4704 if (attr->sched_flags & SCHED_FLAG_KEEP_PARAMS) 4705 return; 4706 4707 __setscheduler_params(p, attr); 4708 4709 /* 4710 * Keep a potential priority boosting if called from 4711 * sched_setscheduler(). 4712 */ 4713 p->prio = normal_prio(p); 4714 if (keep_boost) 4715 p->prio = rt_effective_prio(p, p->prio); 4716 4717 if (dl_prio(p->prio)) 4718 p->sched_class = &dl_sched_class; 4719 else if (rt_prio(p->prio)) 4720 p->sched_class = &rt_sched_class; 4721 else 4722 p->sched_class = &fair_sched_class; 4723 } 4724 4725 /* 4726 * Check the target process has a UID that matches the current process's: 4727 */ 4728 static bool check_same_owner(struct task_struct *p) 4729 { 4730 const struct cred *cred = current_cred(), *pcred; 4731 bool match; 4732 4733 rcu_read_lock(); 4734 pcred = __task_cred(p); 4735 match = (uid_eq(cred->euid, pcred->euid) || 4736 uid_eq(cred->euid, pcred->uid)); 4737 rcu_read_unlock(); 4738 return match; 4739 } 4740 4741 static int __sched_setscheduler(struct task_struct *p, 4742 const struct sched_attr *attr, 4743 bool user, bool pi) 4744 { 4745 int newprio = dl_policy(attr->sched_policy) ? MAX_DL_PRIO - 1 : 4746 MAX_RT_PRIO - 1 - attr->sched_priority; 4747 int retval, oldprio, oldpolicy = -1, queued, running; 4748 int new_effective_prio, policy = attr->sched_policy; 4749 const struct sched_class *prev_class; 4750 struct rq_flags rf; 4751 int reset_on_fork; 4752 int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; 4753 struct rq *rq; 4754 4755 /* The pi code expects interrupts enabled */ 4756 BUG_ON(pi && in_interrupt()); 4757 recheck: 4758 /* Double check policy once rq lock held: */ 4759 if (policy < 0) { 4760 reset_on_fork = p->sched_reset_on_fork; 4761 policy = oldpolicy = p->policy; 4762 } else { 4763 reset_on_fork = !!(attr->sched_flags & SCHED_FLAG_RESET_ON_FORK); 4764 4765 if (!valid_policy(policy)) 4766 return -EINVAL; 4767 } 4768 4769 if (attr->sched_flags & ~(SCHED_FLAG_ALL | SCHED_FLAG_SUGOV)) 4770 return -EINVAL; 4771 4772 /* 4773 * Valid priorities for SCHED_FIFO and SCHED_RR are 4774 * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL, 4775 * SCHED_BATCH and SCHED_IDLE is 0. 4776 */ 4777 if ((p->mm && attr->sched_priority > MAX_USER_RT_PRIO-1) || 4778 (!p->mm && attr->sched_priority > MAX_RT_PRIO-1)) 4779 return -EINVAL; 4780 if ((dl_policy(policy) && !__checkparam_dl(attr)) || 4781 (rt_policy(policy) != (attr->sched_priority != 0))) 4782 return -EINVAL; 4783 4784 /* 4785 * Allow unprivileged RT tasks to decrease priority: 4786 */ 4787 if (user && !capable(CAP_SYS_NICE)) { 4788 if (fair_policy(policy)) { 4789 if (attr->sched_nice < task_nice(p) && 4790 !can_nice(p, attr->sched_nice)) 4791 return -EPERM; 4792 } 4793 4794 if (rt_policy(policy)) { 4795 unsigned long rlim_rtprio = 4796 task_rlimit(p, RLIMIT_RTPRIO); 4797 4798 /* Can't set/change the rt policy: */ 4799 if (policy != p->policy && !rlim_rtprio) 4800 return -EPERM; 4801 4802 /* Can't increase priority: */ 4803 if (attr->sched_priority > p->rt_priority && 4804 attr->sched_priority > rlim_rtprio) 4805 return -EPERM; 4806 } 4807 4808 /* 4809 * Can't set/change SCHED_DEADLINE policy at all for now 4810 * (safest behavior); in the future we would like to allow 4811 * unprivileged DL tasks to increase their relative deadline 4812 * or reduce their runtime (both ways reducing utilization) 4813 */ 4814 if (dl_policy(policy)) 4815 return -EPERM; 4816 4817 /* 4818 * Treat SCHED_IDLE as nice 20. Only allow a switch to 4819 * SCHED_NORMAL if the RLIMIT_NICE would normally permit it. 4820 */ 4821 if (task_has_idle_policy(p) && !idle_policy(policy)) { 4822 if (!can_nice(p, task_nice(p))) 4823 return -EPERM; 4824 } 4825 4826 /* Can't change other user's priorities: */ 4827 if (!check_same_owner(p)) 4828 return -EPERM; 4829 4830 /* Normal users shall not reset the sched_reset_on_fork flag: */ 4831 if (p->sched_reset_on_fork && !reset_on_fork) 4832 return -EPERM; 4833 } 4834 4835 if (user) { 4836 if (attr->sched_flags & SCHED_FLAG_SUGOV) 4837 return -EINVAL; 4838 4839 retval = security_task_setscheduler(p); 4840 if (retval) 4841 return retval; 4842 } 4843 4844 /* Update task specific "requested" clamps */ 4845 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP) { 4846 retval = uclamp_validate(p, attr); 4847 if (retval) 4848 return retval; 4849 } 4850 4851 if (pi) 4852 cpuset_read_lock(); 4853 4854 /* 4855 * Make sure no PI-waiters arrive (or leave) while we are 4856 * changing the priority of the task: 4857 * 4858 * To be able to change p->policy safely, the appropriate 4859 * runqueue lock must be held. 4860 */ 4861 rq = task_rq_lock(p, &rf); 4862 update_rq_clock(rq); 4863 4864 /* 4865 * Changing the policy of the stop threads its a very bad idea: 4866 */ 4867 if (p == rq->stop) { 4868 retval = -EINVAL; 4869 goto unlock; 4870 } 4871 4872 /* 4873 * If not changing anything there's no need to proceed further, 4874 * but store a possible modification of reset_on_fork. 4875 */ 4876 if (unlikely(policy == p->policy)) { 4877 if (fair_policy(policy) && attr->sched_nice != task_nice(p)) 4878 goto change; 4879 if (rt_policy(policy) && attr->sched_priority != p->rt_priority) 4880 goto change; 4881 if (dl_policy(policy) && dl_param_changed(p, attr)) 4882 goto change; 4883 if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP) 4884 goto change; 4885 4886 p->sched_reset_on_fork = reset_on_fork; 4887 retval = 0; 4888 goto unlock; 4889 } 4890 change: 4891 4892 if (user) { 4893 #ifdef CONFIG_RT_GROUP_SCHED 4894 /* 4895 * Do not allow realtime tasks into groups that have no runtime 4896 * assigned. 4897 */ 4898 if (rt_bandwidth_enabled() && rt_policy(policy) && 4899 task_group(p)->rt_bandwidth.rt_runtime == 0 && 4900 !task_group_is_autogroup(task_group(p))) { 4901 retval = -EPERM; 4902 goto unlock; 4903 } 4904 #endif 4905 #ifdef CONFIG_SMP 4906 if (dl_bandwidth_enabled() && dl_policy(policy) && 4907 !(attr->sched_flags & SCHED_FLAG_SUGOV)) { 4908 cpumask_t *span = rq->rd->span; 4909 4910 /* 4911 * Don't allow tasks with an affinity mask smaller than 4912 * the entire root_domain to become SCHED_DEADLINE. We 4913 * will also fail if there's no bandwidth available. 4914 */ 4915 if (!cpumask_subset(span, p->cpus_ptr) || 4916 rq->rd->dl_bw.bw == 0) { 4917 retval = -EPERM; 4918 goto unlock; 4919 } 4920 } 4921 #endif 4922 } 4923 4924 /* Re-check policy now with rq lock held: */ 4925 if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { 4926 policy = oldpolicy = -1; 4927 task_rq_unlock(rq, p, &rf); 4928 if (pi) 4929 cpuset_read_unlock(); 4930 goto recheck; 4931 } 4932 4933 /* 4934 * If setscheduling to SCHED_DEADLINE (or changing the parameters 4935 * of a SCHED_DEADLINE task) we need to check if enough bandwidth 4936 * is available. 4937 */ 4938 if ((dl_policy(policy) || dl_task(p)) && sched_dl_overflow(p, policy, attr)) { 4939 retval = -EBUSY; 4940 goto unlock; 4941 } 4942 4943 p->sched_reset_on_fork = reset_on_fork; 4944 oldprio = p->prio; 4945 4946 if (pi) { 4947 /* 4948 * Take priority boosted tasks into account. If the new 4949 * effective priority is unchanged, we just store the new 4950 * normal parameters and do not touch the scheduler class and 4951 * the runqueue. This will be done when the task deboost 4952 * itself. 4953 */ 4954 new_effective_prio = rt_effective_prio(p, newprio); 4955 if (new_effective_prio == oldprio) 4956 queue_flags &= ~DEQUEUE_MOVE; 4957 } 4958 4959 queued = task_on_rq_queued(p); 4960 running = task_current(rq, p); 4961 if (queued) 4962 dequeue_task(rq, p, queue_flags); 4963 if (running) 4964 put_prev_task(rq, p); 4965 4966 prev_class = p->sched_class; 4967 4968 __setscheduler(rq, p, attr, pi); 4969 __setscheduler_uclamp(p, attr); 4970 4971 if (queued) { 4972 /* 4973 * We enqueue to tail when the priority of a task is 4974 * increased (user space view). 4975 */ 4976 if (oldprio < p->prio) 4977 queue_flags |= ENQUEUE_HEAD; 4978 4979 enqueue_task(rq, p, queue_flags); 4980 } 4981 if (running) 4982 set_next_task(rq, p); 4983 4984 check_class_changed(rq, p, prev_class, oldprio); 4985 4986 /* Avoid rq from going away on us: */ 4987 preempt_disable(); 4988 task_rq_unlock(rq, p, &rf); 4989 4990 if (pi) { 4991 cpuset_read_unlock(); 4992 rt_mutex_adjust_pi(p); 4993 } 4994 4995 /* Run balance callbacks after we've adjusted the PI chain: */ 4996 balance_callback(rq); 4997 preempt_enable(); 4998 4999 return 0; 5000 5001 unlock: 5002 task_rq_unlock(rq, p, &rf); 5003 if (pi) 5004 cpuset_read_unlock(); 5005 return retval; 5006 } 5007 5008 static int _sched_setscheduler(struct task_struct *p, int policy, 5009 const struct sched_param *param, bool check) 5010 { 5011 struct sched_attr attr = { 5012 .sched_policy = policy, 5013 .sched_priority = param->sched_priority, 5014 .sched_nice = PRIO_TO_NICE(p->static_prio), 5015 }; 5016 5017 /* Fixup the legacy SCHED_RESET_ON_FORK hack. */ 5018 if ((policy != SETPARAM_POLICY) && (policy & SCHED_RESET_ON_FORK)) { 5019 attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK; 5020 policy &= ~SCHED_RESET_ON_FORK; 5021 attr.sched_policy = policy; 5022 } 5023 5024 return __sched_setscheduler(p, &attr, check, true); 5025 } 5026 /** 5027 * sched_setscheduler - change the scheduling policy and/or RT priority of a thread. 5028 * @p: the task in question. 5029 * @policy: new policy. 5030 * @param: structure containing the new RT priority. 5031 * 5032 * Return: 0 on success. An error code otherwise. 5033 * 5034 * NOTE that the task may be already dead. 5035 */ 5036 int sched_setscheduler(struct task_struct *p, int policy, 5037 const struct sched_param *param) 5038 { 5039 return _sched_setscheduler(p, policy, param, true); 5040 } 5041 EXPORT_SYMBOL_GPL(sched_setscheduler); 5042 5043 int sched_setattr(struct task_struct *p, const struct sched_attr *attr) 5044 { 5045 return __sched_setscheduler(p, attr, true, true); 5046 } 5047 EXPORT_SYMBOL_GPL(sched_setattr); 5048 5049 int sched_setattr_nocheck(struct task_struct *p, const struct sched_attr *attr) 5050 { 5051 return __sched_setscheduler(p, attr, false, true); 5052 } 5053 5054 /** 5055 * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace. 5056 * @p: the task in question. 5057 * @policy: new policy. 5058 * @param: structure containing the new RT priority. 5059 * 5060 * Just like sched_setscheduler, only don't bother checking if the 5061 * current context has permission. For example, this is needed in 5062 * stop_machine(): we create temporary high priority worker threads, 5063 * but our caller might not have that capability. 5064 * 5065 * Return: 0 on success. An error code otherwise. 5066 */ 5067 int sched_setscheduler_nocheck(struct task_struct *p, int policy, 5068 const struct sched_param *param) 5069 { 5070 return _sched_setscheduler(p, policy, param, false); 5071 } 5072 EXPORT_SYMBOL_GPL(sched_setscheduler_nocheck); 5073 5074 static int 5075 do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param) 5076 { 5077 struct sched_param lparam; 5078 struct task_struct *p; 5079 int retval; 5080 5081 if (!param || pid < 0) 5082 return -EINVAL; 5083 if (copy_from_user(&lparam, param, sizeof(struct sched_param))) 5084 return -EFAULT; 5085 5086 rcu_read_lock(); 5087 retval = -ESRCH; 5088 p = find_process_by_pid(pid); 5089 if (likely(p)) 5090 get_task_struct(p); 5091 rcu_read_unlock(); 5092 5093 if (likely(p)) { 5094 retval = sched_setscheduler(p, policy, &lparam); 5095 put_task_struct(p); 5096 } 5097 5098 return retval; 5099 } 5100 5101 /* 5102 * Mimics kernel/events/core.c perf_copy_attr(). 5103 */ 5104 static int sched_copy_attr(struct sched_attr __user *uattr, struct sched_attr *attr) 5105 { 5106 u32 size; 5107 int ret; 5108 5109 /* Zero the full structure, so that a short copy will be nice: */ 5110 memset(attr, 0, sizeof(*attr)); 5111 5112 ret = get_user(size, &uattr->size); 5113 if (ret) 5114 return ret; 5115 5116 /* ABI compatibility quirk: */ 5117 if (!size) 5118 size = SCHED_ATTR_SIZE_VER0; 5119 if (size < SCHED_ATTR_SIZE_VER0 || size > PAGE_SIZE) 5120 goto err_size; 5121 5122 ret = copy_struct_from_user(attr, sizeof(*attr), uattr, size); 5123 if (ret) { 5124 if (ret == -E2BIG) 5125 goto err_size; 5126 return ret; 5127 } 5128 5129 if ((attr->sched_flags & SCHED_FLAG_UTIL_CLAMP) && 5130 size < SCHED_ATTR_SIZE_VER1) 5131 return -EINVAL; 5132 5133 /* 5134 * XXX: Do we want to be lenient like existing syscalls; or do we want 5135 * to be strict and return an error on out-of-bounds values? 5136 */ 5137 attr->sched_nice = clamp(attr->sched_nice, MIN_NICE, MAX_NICE); 5138 5139 return 0; 5140 5141 err_size: 5142 put_user(sizeof(*attr), &uattr->size); 5143 return -E2BIG; 5144 } 5145 5146 /** 5147 * sys_sched_setscheduler - set/change the scheduler policy and RT priority 5148 * @pid: the pid in question. 5149 * @policy: new policy. 5150 * @param: structure containing the new RT priority. 5151 * 5152 * Return: 0 on success. An error code otherwise. 5153 */ 5154 SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy, struct sched_param __user *, param) 5155 { 5156 if (policy < 0) 5157 return -EINVAL; 5158 5159 return do_sched_setscheduler(pid, policy, param); 5160 } 5161 5162 /** 5163 * sys_sched_setparam - set/change the RT priority of a thread 5164 * @pid: the pid in question. 5165 * @param: structure containing the new RT priority. 5166 * 5167 * Return: 0 on success. An error code otherwise. 5168 */ 5169 SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param) 5170 { 5171 return do_sched_setscheduler(pid, SETPARAM_POLICY, param); 5172 } 5173 5174 /** 5175 * sys_sched_setattr - same as above, but with extended sched_attr 5176 * @pid: the pid in question. 5177 * @uattr: structure containing the extended parameters. 5178 * @flags: for future extension. 5179 */ 5180 SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr, 5181 unsigned int, flags) 5182 { 5183 struct sched_attr attr; 5184 struct task_struct *p; 5185 int retval; 5186 5187 if (!uattr || pid < 0 || flags) 5188 return -EINVAL; 5189 5190 retval = sched_copy_attr(uattr, &attr); 5191 if (retval) 5192 return retval; 5193 5194 if ((int)attr.sched_policy < 0) 5195 return -EINVAL; 5196 if (attr.sched_flags & SCHED_FLAG_KEEP_POLICY) 5197 attr.sched_policy = SETPARAM_POLICY; 5198 5199 rcu_read_lock(); 5200 retval = -ESRCH; 5201 p = find_process_by_pid(pid); 5202 if (likely(p)) 5203 get_task_struct(p); 5204 rcu_read_unlock(); 5205 5206 if (likely(p)) { 5207 retval = sched_setattr(p, &attr); 5208 put_task_struct(p); 5209 } 5210 5211 return retval; 5212 } 5213 5214 /** 5215 * sys_sched_getscheduler - get the policy (scheduling class) of a thread 5216 * @pid: the pid in question. 5217 * 5218 * Return: On success, the policy of the thread. Otherwise, a negative error 5219 * code. 5220 */ 5221 SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid) 5222 { 5223 struct task_struct *p; 5224 int retval; 5225 5226 if (pid < 0) 5227 return -EINVAL; 5228 5229 retval = -ESRCH; 5230 rcu_read_lock(); 5231 p = find_process_by_pid(pid); 5232 if (p) { 5233 retval = security_task_getscheduler(p); 5234 if (!retval) 5235 retval = p->policy 5236 | (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0); 5237 } 5238 rcu_read_unlock(); 5239 return retval; 5240 } 5241 5242 /** 5243 * sys_sched_getparam - get the RT priority of a thread 5244 * @pid: the pid in question. 5245 * @param: structure containing the RT priority. 5246 * 5247 * Return: On success, 0 and the RT priority is in @param. Otherwise, an error 5248 * code. 5249 */ 5250 SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param) 5251 { 5252 struct sched_param lp = { .sched_priority = 0 }; 5253 struct task_struct *p; 5254 int retval; 5255 5256 if (!param || pid < 0) 5257 return -EINVAL; 5258 5259 rcu_read_lock(); 5260 p = find_process_by_pid(pid); 5261 retval = -ESRCH; 5262 if (!p) 5263 goto out_unlock; 5264 5265 retval = security_task_getscheduler(p); 5266 if (retval) 5267 goto out_unlock; 5268 5269 if (task_has_rt_policy(p)) 5270 lp.sched_priority = p->rt_priority; 5271 rcu_read_unlock(); 5272 5273 /* 5274 * This one might sleep, we cannot do it with a spinlock held ... 5275 */ 5276 retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0; 5277 5278 return retval; 5279 5280 out_unlock: 5281 rcu_read_unlock(); 5282 return retval; 5283 } 5284 5285 /* 5286 * Copy the kernel size attribute structure (which might be larger 5287 * than what user-space knows about) to user-space. 5288 * 5289 * Note that all cases are valid: user-space buffer can be larger or 5290 * smaller than the kernel-space buffer. The usual case is that both 5291 * have the same size. 5292 */ 5293 static int 5294 sched_attr_copy_to_user(struct sched_attr __user *uattr, 5295 struct sched_attr *kattr, 5296 unsigned int usize) 5297 { 5298 unsigned int ksize = sizeof(*kattr); 5299 5300 if (!access_ok(uattr, usize)) 5301 return -EFAULT; 5302 5303 /* 5304 * sched_getattr() ABI forwards and backwards compatibility: 5305 * 5306 * If usize == ksize then we just copy everything to user-space and all is good. 5307 * 5308 * If usize < ksize then we only copy as much as user-space has space for, 5309 * this keeps ABI compatibility as well. We skip the rest. 5310 * 5311 * If usize > ksize then user-space is using a newer version of the ABI, 5312 * which part the kernel doesn't know about. Just ignore it - tooling can 5313 * detect the kernel's knowledge of attributes from the attr->size value 5314 * which is set to ksize in this case. 5315 */ 5316 kattr->size = min(usize, ksize); 5317 5318 if (copy_to_user(uattr, kattr, kattr->size)) 5319 return -EFAULT; 5320 5321 return 0; 5322 } 5323 5324 /** 5325 * sys_sched_getattr - similar to sched_getparam, but with sched_attr 5326 * @pid: the pid in question. 5327 * @uattr: structure containing the extended parameters. 5328 * @usize: sizeof(attr) for fwd/bwd comp. 5329 * @flags: for future extension. 5330 */ 5331 SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr, 5332 unsigned int, usize, unsigned int, flags) 5333 { 5334 struct sched_attr kattr = { }; 5335 struct task_struct *p; 5336 int retval; 5337 5338 if (!uattr || pid < 0 || usize > PAGE_SIZE || 5339 usize < SCHED_ATTR_SIZE_VER0 || flags) 5340 return -EINVAL; 5341 5342 rcu_read_lock(); 5343 p = find_process_by_pid(pid); 5344 retval = -ESRCH; 5345 if (!p) 5346 goto out_unlock; 5347 5348 retval = security_task_getscheduler(p); 5349 if (retval) 5350 goto out_unlock; 5351 5352 kattr.sched_policy = p->policy; 5353 if (p->sched_reset_on_fork) 5354 kattr.sched_flags |= SCHED_FLAG_RESET_ON_FORK; 5355 if (task_has_dl_policy(p)) 5356 __getparam_dl(p, &kattr); 5357 else if (task_has_rt_policy(p)) 5358 kattr.sched_priority = p->rt_priority; 5359 else 5360 kattr.sched_nice = task_nice(p); 5361 5362 #ifdef CONFIG_UCLAMP_TASK 5363 kattr.sched_util_min = p->uclamp_req[UCLAMP_MIN].value; 5364 kattr.sched_util_max = p->uclamp_req[UCLAMP_MAX].value; 5365 #endif 5366 5367 rcu_read_unlock(); 5368 5369 return sched_attr_copy_to_user(uattr, &kattr, usize); 5370 5371 out_unlock: 5372 rcu_read_unlock(); 5373 return retval; 5374 } 5375 5376 long sched_setaffinity(pid_t pid, const struct cpumask *in_mask) 5377 { 5378 cpumask_var_t cpus_allowed, new_mask; 5379 struct task_struct *p; 5380 int retval; 5381 5382 rcu_read_lock(); 5383 5384 p = find_process_by_pid(pid); 5385 if (!p) { 5386 rcu_read_unlock(); 5387 return -ESRCH; 5388 } 5389 5390 /* Prevent p going away */ 5391 get_task_struct(p); 5392 rcu_read_unlock(); 5393 5394 if (p->flags & PF_NO_SETAFFINITY) { 5395 retval = -EINVAL; 5396 goto out_put_task; 5397 } 5398 if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) { 5399 retval = -ENOMEM; 5400 goto out_put_task; 5401 } 5402 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) { 5403 retval = -ENOMEM; 5404 goto out_free_cpus_allowed; 5405 } 5406 retval = -EPERM; 5407 if (!check_same_owner(p)) { 5408 rcu_read_lock(); 5409 if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) { 5410 rcu_read_unlock(); 5411 goto out_free_new_mask; 5412 } 5413 rcu_read_unlock(); 5414 } 5415 5416 retval = security_task_setscheduler(p); 5417 if (retval) 5418 goto out_free_new_mask; 5419 5420 5421 cpuset_cpus_allowed(p, cpus_allowed); 5422 cpumask_and(new_mask, in_mask, cpus_allowed); 5423 5424 /* 5425 * Since bandwidth control happens on root_domain basis, 5426 * if admission test is enabled, we only admit -deadline 5427 * tasks allowed to run on all the CPUs in the task's 5428 * root_domain. 5429 */ 5430 #ifdef CONFIG_SMP 5431 if (task_has_dl_policy(p) && dl_bandwidth_enabled()) { 5432 rcu_read_lock(); 5433 if (!cpumask_subset(task_rq(p)->rd->span, new_mask)) { 5434 retval = -EBUSY; 5435 rcu_read_unlock(); 5436 goto out_free_new_mask; 5437 } 5438 rcu_read_unlock(); 5439 } 5440 #endif 5441 again: 5442 retval = __set_cpus_allowed_ptr(p, new_mask, true); 5443 5444 if (!retval) { 5445 cpuset_cpus_allowed(p, cpus_allowed); 5446 if (!cpumask_subset(new_mask, cpus_allowed)) { 5447 /* 5448 * We must have raced with a concurrent cpuset 5449 * update. Just reset the cpus_allowed to the 5450 * cpuset's cpus_allowed 5451 */ 5452 cpumask_copy(new_mask, cpus_allowed); 5453 goto again; 5454 } 5455 } 5456 out_free_new_mask: 5457 free_cpumask_var(new_mask); 5458 out_free_cpus_allowed: 5459 free_cpumask_var(cpus_allowed); 5460 out_put_task: 5461 put_task_struct(p); 5462 return retval; 5463 } 5464 5465 static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len, 5466 struct cpumask *new_mask) 5467 { 5468 if (len < cpumask_size()) 5469 cpumask_clear(new_mask); 5470 else if (len > cpumask_size()) 5471 len = cpumask_size(); 5472 5473 return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0; 5474 } 5475 5476 /** 5477 * sys_sched_setaffinity - set the CPU affinity of a process 5478 * @pid: pid of the process 5479 * @len: length in bytes of the bitmask pointed to by user_mask_ptr 5480 * @user_mask_ptr: user-space pointer to the new CPU mask 5481 * 5482 * Return: 0 on success. An error code otherwise. 5483 */ 5484 SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len, 5485 unsigned long __user *, user_mask_ptr) 5486 { 5487 cpumask_var_t new_mask; 5488 int retval; 5489 5490 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) 5491 return -ENOMEM; 5492 5493 retval = get_user_cpu_mask(user_mask_ptr, len, new_mask); 5494 if (retval == 0) 5495 retval = sched_setaffinity(pid, new_mask); 5496 free_cpumask_var(new_mask); 5497 return retval; 5498 } 5499 5500 long sched_getaffinity(pid_t pid, struct cpumask *mask) 5501 { 5502 struct task_struct *p; 5503 unsigned long flags; 5504 int retval; 5505 5506 rcu_read_lock(); 5507 5508 retval = -ESRCH; 5509 p = find_process_by_pid(pid); 5510 if (!p) 5511 goto out_unlock; 5512 5513 retval = security_task_getscheduler(p); 5514 if (retval) 5515 goto out_unlock; 5516 5517 raw_spin_lock_irqsave(&p->pi_lock, flags); 5518 cpumask_and(mask, &p->cpus_mask, cpu_active_mask); 5519 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 5520 5521 out_unlock: 5522 rcu_read_unlock(); 5523 5524 return retval; 5525 } 5526 5527 /** 5528 * sys_sched_getaffinity - get the CPU affinity of a process 5529 * @pid: pid of the process 5530 * @len: length in bytes of the bitmask pointed to by user_mask_ptr 5531 * @user_mask_ptr: user-space pointer to hold the current CPU mask 5532 * 5533 * Return: size of CPU mask copied to user_mask_ptr on success. An 5534 * error code otherwise. 5535 */ 5536 SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len, 5537 unsigned long __user *, user_mask_ptr) 5538 { 5539 int ret; 5540 cpumask_var_t mask; 5541 5542 if ((len * BITS_PER_BYTE) < nr_cpu_ids) 5543 return -EINVAL; 5544 if (len & (sizeof(unsigned long)-1)) 5545 return -EINVAL; 5546 5547 if (!alloc_cpumask_var(&mask, GFP_KERNEL)) 5548 return -ENOMEM; 5549 5550 ret = sched_getaffinity(pid, mask); 5551 if (ret == 0) { 5552 unsigned int retlen = min(len, cpumask_size()); 5553 5554 if (copy_to_user(user_mask_ptr, mask, retlen)) 5555 ret = -EFAULT; 5556 else 5557 ret = retlen; 5558 } 5559 free_cpumask_var(mask); 5560 5561 return ret; 5562 } 5563 5564 /** 5565 * sys_sched_yield - yield the current processor to other threads. 5566 * 5567 * This function yields the current CPU to other tasks. If there are no 5568 * other threads running on this CPU then this function will return. 5569 * 5570 * Return: 0. 5571 */ 5572 static void do_sched_yield(void) 5573 { 5574 struct rq_flags rf; 5575 struct rq *rq; 5576 5577 rq = this_rq_lock_irq(&rf); 5578 5579 schedstat_inc(rq->yld_count); 5580 current->sched_class->yield_task(rq); 5581 5582 /* 5583 * Since we are going to call schedule() anyway, there's 5584 * no need to preempt or enable interrupts: 5585 */ 5586 preempt_disable(); 5587 rq_unlock(rq, &rf); 5588 sched_preempt_enable_no_resched(); 5589 5590 schedule(); 5591 } 5592 5593 SYSCALL_DEFINE0(sched_yield) 5594 { 5595 do_sched_yield(); 5596 return 0; 5597 } 5598 5599 #ifndef CONFIG_PREEMPTION 5600 int __sched _cond_resched(void) 5601 { 5602 if (should_resched(0)) { 5603 preempt_schedule_common(); 5604 return 1; 5605 } 5606 rcu_all_qs(); 5607 return 0; 5608 } 5609 EXPORT_SYMBOL(_cond_resched); 5610 #endif 5611 5612 /* 5613 * __cond_resched_lock() - if a reschedule is pending, drop the given lock, 5614 * call schedule, and on return reacquire the lock. 5615 * 5616 * This works OK both with and without CONFIG_PREEMPTION. We do strange low-level 5617 * operations here to prevent schedule() from being called twice (once via 5618 * spin_unlock(), once by hand). 5619 */ 5620 int __cond_resched_lock(spinlock_t *lock) 5621 { 5622 int resched = should_resched(PREEMPT_LOCK_OFFSET); 5623 int ret = 0; 5624 5625 lockdep_assert_held(lock); 5626 5627 if (spin_needbreak(lock) || resched) { 5628 spin_unlock(lock); 5629 if (resched) 5630 preempt_schedule_common(); 5631 else 5632 cpu_relax(); 5633 ret = 1; 5634 spin_lock(lock); 5635 } 5636 return ret; 5637 } 5638 EXPORT_SYMBOL(__cond_resched_lock); 5639 5640 /** 5641 * yield - yield the current processor to other threads. 5642 * 5643 * Do not ever use this function, there's a 99% chance you're doing it wrong. 5644 * 5645 * The scheduler is at all times free to pick the calling task as the most 5646 * eligible task to run, if removing the yield() call from your code breaks 5647 * it, its already broken. 5648 * 5649 * Typical broken usage is: 5650 * 5651 * while (!event) 5652 * yield(); 5653 * 5654 * where one assumes that yield() will let 'the other' process run that will 5655 * make event true. If the current task is a SCHED_FIFO task that will never 5656 * happen. Never use yield() as a progress guarantee!! 5657 * 5658 * If you want to use yield() to wait for something, use wait_event(). 5659 * If you want to use yield() to be 'nice' for others, use cond_resched(). 5660 * If you still want to use yield(), do not! 5661 */ 5662 void __sched yield(void) 5663 { 5664 set_current_state(TASK_RUNNING); 5665 do_sched_yield(); 5666 } 5667 EXPORT_SYMBOL(yield); 5668 5669 /** 5670 * yield_to - yield the current processor to another thread in 5671 * your thread group, or accelerate that thread toward the 5672 * processor it's on. 5673 * @p: target task 5674 * @preempt: whether task preemption is allowed or not 5675 * 5676 * It's the caller's job to ensure that the target task struct 5677 * can't go away on us before we can do any checks. 5678 * 5679 * Return: 5680 * true (>0) if we indeed boosted the target task. 5681 * false (0) if we failed to boost the target. 5682 * -ESRCH if there's no task to yield to. 5683 */ 5684 int __sched yield_to(struct task_struct *p, bool preempt) 5685 { 5686 struct task_struct *curr = current; 5687 struct rq *rq, *p_rq; 5688 unsigned long flags; 5689 int yielded = 0; 5690 5691 local_irq_save(flags); 5692 rq = this_rq(); 5693 5694 again: 5695 p_rq = task_rq(p); 5696 /* 5697 * If we're the only runnable task on the rq and target rq also 5698 * has only one task, there's absolutely no point in yielding. 5699 */ 5700 if (rq->nr_running == 1 && p_rq->nr_running == 1) { 5701 yielded = -ESRCH; 5702 goto out_irq; 5703 } 5704 5705 double_rq_lock(rq, p_rq); 5706 if (task_rq(p) != p_rq) { 5707 double_rq_unlock(rq, p_rq); 5708 goto again; 5709 } 5710 5711 if (!curr->sched_class->yield_to_task) 5712 goto out_unlock; 5713 5714 if (curr->sched_class != p->sched_class) 5715 goto out_unlock; 5716 5717 if (task_running(p_rq, p) || p->state) 5718 goto out_unlock; 5719 5720 yielded = curr->sched_class->yield_to_task(rq, p, preempt); 5721 if (yielded) { 5722 schedstat_inc(rq->yld_count); 5723 /* 5724 * Make p's CPU reschedule; pick_next_entity takes care of 5725 * fairness. 5726 */ 5727 if (preempt && rq != p_rq) 5728 resched_curr(p_rq); 5729 } 5730 5731 out_unlock: 5732 double_rq_unlock(rq, p_rq); 5733 out_irq: 5734 local_irq_restore(flags); 5735 5736 if (yielded > 0) 5737 schedule(); 5738 5739 return yielded; 5740 } 5741 EXPORT_SYMBOL_GPL(yield_to); 5742 5743 int io_schedule_prepare(void) 5744 { 5745 int old_iowait = current->in_iowait; 5746 5747 current->in_iowait = 1; 5748 blk_schedule_flush_plug(current); 5749 5750 return old_iowait; 5751 } 5752 5753 void io_schedule_finish(int token) 5754 { 5755 current->in_iowait = token; 5756 } 5757 5758 /* 5759 * This task is about to go to sleep on IO. Increment rq->nr_iowait so 5760 * that process accounting knows that this is a task in IO wait state. 5761 */ 5762 long __sched io_schedule_timeout(long timeout) 5763 { 5764 int token; 5765 long ret; 5766 5767 token = io_schedule_prepare(); 5768 ret = schedule_timeout(timeout); 5769 io_schedule_finish(token); 5770 5771 return ret; 5772 } 5773 EXPORT_SYMBOL(io_schedule_timeout); 5774 5775 void __sched io_schedule(void) 5776 { 5777 int token; 5778 5779 token = io_schedule_prepare(); 5780 schedule(); 5781 io_schedule_finish(token); 5782 } 5783 EXPORT_SYMBOL(io_schedule); 5784 5785 /** 5786 * sys_sched_get_priority_max - return maximum RT priority. 5787 * @policy: scheduling class. 5788 * 5789 * Return: On success, this syscall returns the maximum 5790 * rt_priority that can be used by a given scheduling class. 5791 * On failure, a negative error code is returned. 5792 */ 5793 SYSCALL_DEFINE1(sched_get_priority_max, int, policy) 5794 { 5795 int ret = -EINVAL; 5796 5797 switch (policy) { 5798 case SCHED_FIFO: 5799 case SCHED_RR: 5800 ret = MAX_USER_RT_PRIO-1; 5801 break; 5802 case SCHED_DEADLINE: 5803 case SCHED_NORMAL: 5804 case SCHED_BATCH: 5805 case SCHED_IDLE: 5806 ret = 0; 5807 break; 5808 } 5809 return ret; 5810 } 5811 5812 /** 5813 * sys_sched_get_priority_min - return minimum RT priority. 5814 * @policy: scheduling class. 5815 * 5816 * Return: On success, this syscall returns the minimum 5817 * rt_priority that can be used by a given scheduling class. 5818 * On failure, a negative error code is returned. 5819 */ 5820 SYSCALL_DEFINE1(sched_get_priority_min, int, policy) 5821 { 5822 int ret = -EINVAL; 5823 5824 switch (policy) { 5825 case SCHED_FIFO: 5826 case SCHED_RR: 5827 ret = 1; 5828 break; 5829 case SCHED_DEADLINE: 5830 case SCHED_NORMAL: 5831 case SCHED_BATCH: 5832 case SCHED_IDLE: 5833 ret = 0; 5834 } 5835 return ret; 5836 } 5837 5838 static int sched_rr_get_interval(pid_t pid, struct timespec64 *t) 5839 { 5840 struct task_struct *p; 5841 unsigned int time_slice; 5842 struct rq_flags rf; 5843 struct rq *rq; 5844 int retval; 5845 5846 if (pid < 0) 5847 return -EINVAL; 5848 5849 retval = -ESRCH; 5850 rcu_read_lock(); 5851 p = find_process_by_pid(pid); 5852 if (!p) 5853 goto out_unlock; 5854 5855 retval = security_task_getscheduler(p); 5856 if (retval) 5857 goto out_unlock; 5858 5859 rq = task_rq_lock(p, &rf); 5860 time_slice = 0; 5861 if (p->sched_class->get_rr_interval) 5862 time_slice = p->sched_class->get_rr_interval(rq, p); 5863 task_rq_unlock(rq, p, &rf); 5864 5865 rcu_read_unlock(); 5866 jiffies_to_timespec64(time_slice, t); 5867 return 0; 5868 5869 out_unlock: 5870 rcu_read_unlock(); 5871 return retval; 5872 } 5873 5874 /** 5875 * sys_sched_rr_get_interval - return the default timeslice of a process. 5876 * @pid: pid of the process. 5877 * @interval: userspace pointer to the timeslice value. 5878 * 5879 * this syscall writes the default timeslice value of a given process 5880 * into the user-space timespec buffer. A value of '0' means infinity. 5881 * 5882 * Return: On success, 0 and the timeslice is in @interval. Otherwise, 5883 * an error code. 5884 */ 5885 SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid, 5886 struct __kernel_timespec __user *, interval) 5887 { 5888 struct timespec64 t; 5889 int retval = sched_rr_get_interval(pid, &t); 5890 5891 if (retval == 0) 5892 retval = put_timespec64(&t, interval); 5893 5894 return retval; 5895 } 5896 5897 #ifdef CONFIG_COMPAT_32BIT_TIME 5898 SYSCALL_DEFINE2(sched_rr_get_interval_time32, pid_t, pid, 5899 struct old_timespec32 __user *, interval) 5900 { 5901 struct timespec64 t; 5902 int retval = sched_rr_get_interval(pid, &t); 5903 5904 if (retval == 0) 5905 retval = put_old_timespec32(&t, interval); 5906 return retval; 5907 } 5908 #endif 5909 5910 void sched_show_task(struct task_struct *p) 5911 { 5912 unsigned long free = 0; 5913 int ppid; 5914 5915 if (!try_get_task_stack(p)) 5916 return; 5917 5918 printk(KERN_INFO "%-15.15s %c", p->comm, task_state_to_char(p)); 5919 5920 if (p->state == TASK_RUNNING) 5921 printk(KERN_CONT " running task "); 5922 #ifdef CONFIG_DEBUG_STACK_USAGE 5923 free = stack_not_used(p); 5924 #endif 5925 ppid = 0; 5926 rcu_read_lock(); 5927 if (pid_alive(p)) 5928 ppid = task_pid_nr(rcu_dereference(p->real_parent)); 5929 rcu_read_unlock(); 5930 printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free, 5931 task_pid_nr(p), ppid, 5932 (unsigned long)task_thread_info(p)->flags); 5933 5934 print_worker_info(KERN_INFO, p); 5935 show_stack(p, NULL); 5936 put_task_stack(p); 5937 } 5938 EXPORT_SYMBOL_GPL(sched_show_task); 5939 5940 static inline bool 5941 state_filter_match(unsigned long state_filter, struct task_struct *p) 5942 { 5943 /* no filter, everything matches */ 5944 if (!state_filter) 5945 return true; 5946 5947 /* filter, but doesn't match */ 5948 if (!(p->state & state_filter)) 5949 return false; 5950 5951 /* 5952 * When looking for TASK_UNINTERRUPTIBLE skip TASK_IDLE (allows 5953 * TASK_KILLABLE). 5954 */ 5955 if (state_filter == TASK_UNINTERRUPTIBLE && p->state == TASK_IDLE) 5956 return false; 5957 5958 return true; 5959 } 5960 5961 5962 void show_state_filter(unsigned long state_filter) 5963 { 5964 struct task_struct *g, *p; 5965 5966 #if BITS_PER_LONG == 32 5967 printk(KERN_INFO 5968 " task PC stack pid father\n"); 5969 #else 5970 printk(KERN_INFO 5971 " task PC stack pid father\n"); 5972 #endif 5973 rcu_read_lock(); 5974 for_each_process_thread(g, p) { 5975 /* 5976 * reset the NMI-timeout, listing all files on a slow 5977 * console might take a lot of time: 5978 * Also, reset softlockup watchdogs on all CPUs, because 5979 * another CPU might be blocked waiting for us to process 5980 * an IPI. 5981 */ 5982 touch_nmi_watchdog(); 5983 touch_all_softlockup_watchdogs(); 5984 if (state_filter_match(state_filter, p)) 5985 sched_show_task(p); 5986 } 5987 5988 #ifdef CONFIG_SCHED_DEBUG 5989 if (!state_filter) 5990 sysrq_sched_debug_show(); 5991 #endif 5992 rcu_read_unlock(); 5993 /* 5994 * Only show locks if all tasks are dumped: 5995 */ 5996 if (!state_filter) 5997 debug_show_all_locks(); 5998 } 5999 6000 /** 6001 * init_idle - set up an idle thread for a given CPU 6002 * @idle: task in question 6003 * @cpu: CPU the idle task belongs to 6004 * 6005 * NOTE: this function does not set the idle thread's NEED_RESCHED 6006 * flag, to make booting more robust. 6007 */ 6008 void init_idle(struct task_struct *idle, int cpu) 6009 { 6010 struct rq *rq = cpu_rq(cpu); 6011 unsigned long flags; 6012 6013 raw_spin_lock_irqsave(&idle->pi_lock, flags); 6014 raw_spin_lock(&rq->lock); 6015 6016 __sched_fork(0, idle); 6017 idle->state = TASK_RUNNING; 6018 idle->se.exec_start = sched_clock(); 6019 idle->flags |= PF_IDLE; 6020 6021 kasan_unpoison_task_stack(idle); 6022 6023 #ifdef CONFIG_SMP 6024 /* 6025 * Its possible that init_idle() gets called multiple times on a task, 6026 * in that case do_set_cpus_allowed() will not do the right thing. 6027 * 6028 * And since this is boot we can forgo the serialization. 6029 */ 6030 set_cpus_allowed_common(idle, cpumask_of(cpu)); 6031 #endif 6032 /* 6033 * We're having a chicken and egg problem, even though we are 6034 * holding rq->lock, the CPU isn't yet set to this CPU so the 6035 * lockdep check in task_group() will fail. 6036 * 6037 * Similar case to sched_fork(). / Alternatively we could 6038 * use task_rq_lock() here and obtain the other rq->lock. 6039 * 6040 * Silence PROVE_RCU 6041 */ 6042 rcu_read_lock(); 6043 __set_task_cpu(idle, cpu); 6044 rcu_read_unlock(); 6045 6046 rq->idle = idle; 6047 rcu_assign_pointer(rq->curr, idle); 6048 idle->on_rq = TASK_ON_RQ_QUEUED; 6049 #ifdef CONFIG_SMP 6050 idle->on_cpu = 1; 6051 #endif 6052 raw_spin_unlock(&rq->lock); 6053 raw_spin_unlock_irqrestore(&idle->pi_lock, flags); 6054 6055 /* Set the preempt count _outside_ the spinlocks! */ 6056 init_idle_preempt_count(idle, cpu); 6057 6058 /* 6059 * The idle tasks have their own, simple scheduling class: 6060 */ 6061 idle->sched_class = &idle_sched_class; 6062 ftrace_graph_init_idle_task(idle, cpu); 6063 vtime_init_idle(idle, cpu); 6064 #ifdef CONFIG_SMP 6065 sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu); 6066 #endif 6067 } 6068 6069 #ifdef CONFIG_SMP 6070 6071 int cpuset_cpumask_can_shrink(const struct cpumask *cur, 6072 const struct cpumask *trial) 6073 { 6074 int ret = 1; 6075 6076 if (!cpumask_weight(cur)) 6077 return ret; 6078 6079 ret = dl_cpuset_cpumask_can_shrink(cur, trial); 6080 6081 return ret; 6082 } 6083 6084 int task_can_attach(struct task_struct *p, 6085 const struct cpumask *cs_cpus_allowed) 6086 { 6087 int ret = 0; 6088 6089 /* 6090 * Kthreads which disallow setaffinity shouldn't be moved 6091 * to a new cpuset; we don't want to change their CPU 6092 * affinity and isolating such threads by their set of 6093 * allowed nodes is unnecessary. Thus, cpusets are not 6094 * applicable for such threads. This prevents checking for 6095 * success of set_cpus_allowed_ptr() on all attached tasks 6096 * before cpus_mask may be changed. 6097 */ 6098 if (p->flags & PF_NO_SETAFFINITY) { 6099 ret = -EINVAL; 6100 goto out; 6101 } 6102 6103 if (dl_task(p) && !cpumask_intersects(task_rq(p)->rd->span, 6104 cs_cpus_allowed)) 6105 ret = dl_task_can_attach(p, cs_cpus_allowed); 6106 6107 out: 6108 return ret; 6109 } 6110 6111 bool sched_smp_initialized __read_mostly; 6112 6113 #ifdef CONFIG_NUMA_BALANCING 6114 /* Migrate current task p to target_cpu */ 6115 int migrate_task_to(struct task_struct *p, int target_cpu) 6116 { 6117 struct migration_arg arg = { p, target_cpu }; 6118 int curr_cpu = task_cpu(p); 6119 6120 if (curr_cpu == target_cpu) 6121 return 0; 6122 6123 if (!cpumask_test_cpu(target_cpu, p->cpus_ptr)) 6124 return -EINVAL; 6125 6126 /* TODO: This is not properly updating schedstats */ 6127 6128 trace_sched_move_numa(p, curr_cpu, target_cpu); 6129 return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg); 6130 } 6131 6132 /* 6133 * Requeue a task on a given node and accurately track the number of NUMA 6134 * tasks on the runqueues 6135 */ 6136 void sched_setnuma(struct task_struct *p, int nid) 6137 { 6138 bool queued, running; 6139 struct rq_flags rf; 6140 struct rq *rq; 6141 6142 rq = task_rq_lock(p, &rf); 6143 queued = task_on_rq_queued(p); 6144 running = task_current(rq, p); 6145 6146 if (queued) 6147 dequeue_task(rq, p, DEQUEUE_SAVE); 6148 if (running) 6149 put_prev_task(rq, p); 6150 6151 p->numa_preferred_nid = nid; 6152 6153 if (queued) 6154 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); 6155 if (running) 6156 set_next_task(rq, p); 6157 task_rq_unlock(rq, p, &rf); 6158 } 6159 #endif /* CONFIG_NUMA_BALANCING */ 6160 6161 #ifdef CONFIG_HOTPLUG_CPU 6162 /* 6163 * Ensure that the idle task is using init_mm right before its CPU goes 6164 * offline. 6165 */ 6166 void idle_task_exit(void) 6167 { 6168 struct mm_struct *mm = current->active_mm; 6169 6170 BUG_ON(cpu_online(smp_processor_id())); 6171 6172 if (mm != &init_mm) { 6173 switch_mm(mm, &init_mm, current); 6174 current->active_mm = &init_mm; 6175 finish_arch_post_lock_switch(); 6176 } 6177 mmdrop(mm); 6178 } 6179 6180 /* 6181 * Since this CPU is going 'away' for a while, fold any nr_active delta 6182 * we might have. Assumes we're called after migrate_tasks() so that the 6183 * nr_active count is stable. We need to take the teardown thread which 6184 * is calling this into account, so we hand in adjust = 1 to the load 6185 * calculation. 6186 * 6187 * Also see the comment "Global load-average calculations". 6188 */ 6189 static void calc_load_migrate(struct rq *rq) 6190 { 6191 long delta = calc_load_fold_active(rq, 1); 6192 if (delta) 6193 atomic_long_add(delta, &calc_load_tasks); 6194 } 6195 6196 static struct task_struct *__pick_migrate_task(struct rq *rq) 6197 { 6198 const struct sched_class *class; 6199 struct task_struct *next; 6200 6201 for_each_class(class) { 6202 next = class->pick_next_task(rq, NULL, NULL); 6203 if (next) { 6204 next->sched_class->put_prev_task(rq, next, NULL); 6205 return next; 6206 } 6207 } 6208 6209 /* The idle class should always have a runnable task */ 6210 BUG(); 6211 } 6212 6213 /* 6214 * Migrate all tasks from the rq, sleeping tasks will be migrated by 6215 * try_to_wake_up()->select_task_rq(). 6216 * 6217 * Called with rq->lock held even though we'er in stop_machine() and 6218 * there's no concurrency possible, we hold the required locks anyway 6219 * because of lock validation efforts. 6220 */ 6221 static void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf) 6222 { 6223 struct rq *rq = dead_rq; 6224 struct task_struct *next, *stop = rq->stop; 6225 struct rq_flags orf = *rf; 6226 int dest_cpu; 6227 6228 /* 6229 * Fudge the rq selection such that the below task selection loop 6230 * doesn't get stuck on the currently eligible stop task. 6231 * 6232 * We're currently inside stop_machine() and the rq is either stuck 6233 * in the stop_machine_cpu_stop() loop, or we're executing this code, 6234 * either way we should never end up calling schedule() until we're 6235 * done here. 6236 */ 6237 rq->stop = NULL; 6238 6239 /* 6240 * put_prev_task() and pick_next_task() sched 6241 * class method both need to have an up-to-date 6242 * value of rq->clock[_task] 6243 */ 6244 update_rq_clock(rq); 6245 6246 for (;;) { 6247 /* 6248 * There's this thread running, bail when that's the only 6249 * remaining thread: 6250 */ 6251 if (rq->nr_running == 1) 6252 break; 6253 6254 next = __pick_migrate_task(rq); 6255 6256 /* 6257 * Rules for changing task_struct::cpus_mask are holding 6258 * both pi_lock and rq->lock, such that holding either 6259 * stabilizes the mask. 6260 * 6261 * Drop rq->lock is not quite as disastrous as it usually is 6262 * because !cpu_active at this point, which means load-balance 6263 * will not interfere. Also, stop-machine. 6264 */ 6265 rq_unlock(rq, rf); 6266 raw_spin_lock(&next->pi_lock); 6267 rq_relock(rq, rf); 6268 6269 /* 6270 * Since we're inside stop-machine, _nothing_ should have 6271 * changed the task, WARN if weird stuff happened, because in 6272 * that case the above rq->lock drop is a fail too. 6273 */ 6274 if (WARN_ON(task_rq(next) != rq || !task_on_rq_queued(next))) { 6275 raw_spin_unlock(&next->pi_lock); 6276 continue; 6277 } 6278 6279 /* Find suitable destination for @next, with force if needed. */ 6280 dest_cpu = select_fallback_rq(dead_rq->cpu, next); 6281 rq = __migrate_task(rq, rf, next, dest_cpu); 6282 if (rq != dead_rq) { 6283 rq_unlock(rq, rf); 6284 rq = dead_rq; 6285 *rf = orf; 6286 rq_relock(rq, rf); 6287 } 6288 raw_spin_unlock(&next->pi_lock); 6289 } 6290 6291 rq->stop = stop; 6292 } 6293 #endif /* CONFIG_HOTPLUG_CPU */ 6294 6295 void set_rq_online(struct rq *rq) 6296 { 6297 if (!rq->online) { 6298 const struct sched_class *class; 6299 6300 cpumask_set_cpu(rq->cpu, rq->rd->online); 6301 rq->online = 1; 6302 6303 for_each_class(class) { 6304 if (class->rq_online) 6305 class->rq_online(rq); 6306 } 6307 } 6308 } 6309 6310 void set_rq_offline(struct rq *rq) 6311 { 6312 if (rq->online) { 6313 const struct sched_class *class; 6314 6315 for_each_class(class) { 6316 if (class->rq_offline) 6317 class->rq_offline(rq); 6318 } 6319 6320 cpumask_clear_cpu(rq->cpu, rq->rd->online); 6321 rq->online = 0; 6322 } 6323 } 6324 6325 /* 6326 * used to mark begin/end of suspend/resume: 6327 */ 6328 static int num_cpus_frozen; 6329 6330 /* 6331 * Update cpusets according to cpu_active mask. If cpusets are 6332 * disabled, cpuset_update_active_cpus() becomes a simple wrapper 6333 * around partition_sched_domains(). 6334 * 6335 * If we come here as part of a suspend/resume, don't touch cpusets because we 6336 * want to restore it back to its original state upon resume anyway. 6337 */ 6338 static void cpuset_cpu_active(void) 6339 { 6340 if (cpuhp_tasks_frozen) { 6341 /* 6342 * num_cpus_frozen tracks how many CPUs are involved in suspend 6343 * resume sequence. As long as this is not the last online 6344 * operation in the resume sequence, just build a single sched 6345 * domain, ignoring cpusets. 6346 */ 6347 partition_sched_domains(1, NULL, NULL); 6348 if (--num_cpus_frozen) 6349 return; 6350 /* 6351 * This is the last CPU online operation. So fall through and 6352 * restore the original sched domains by considering the 6353 * cpuset configurations. 6354 */ 6355 cpuset_force_rebuild(); 6356 } 6357 cpuset_update_active_cpus(); 6358 } 6359 6360 static int cpuset_cpu_inactive(unsigned int cpu) 6361 { 6362 if (!cpuhp_tasks_frozen) { 6363 if (dl_cpu_busy(cpu)) 6364 return -EBUSY; 6365 cpuset_update_active_cpus(); 6366 } else { 6367 num_cpus_frozen++; 6368 partition_sched_domains(1, NULL, NULL); 6369 } 6370 return 0; 6371 } 6372 6373 int sched_cpu_activate(unsigned int cpu) 6374 { 6375 struct rq *rq = cpu_rq(cpu); 6376 struct rq_flags rf; 6377 6378 #ifdef CONFIG_SCHED_SMT 6379 /* 6380 * When going up, increment the number of cores with SMT present. 6381 */ 6382 if (cpumask_weight(cpu_smt_mask(cpu)) == 2) 6383 static_branch_inc_cpuslocked(&sched_smt_present); 6384 #endif 6385 set_cpu_active(cpu, true); 6386 6387 if (sched_smp_initialized) { 6388 sched_domains_numa_masks_set(cpu); 6389 cpuset_cpu_active(); 6390 } 6391 6392 /* 6393 * Put the rq online, if not already. This happens: 6394 * 6395 * 1) In the early boot process, because we build the real domains 6396 * after all CPUs have been brought up. 6397 * 6398 * 2) At runtime, if cpuset_cpu_active() fails to rebuild the 6399 * domains. 6400 */ 6401 rq_lock_irqsave(rq, &rf); 6402 if (rq->rd) { 6403 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); 6404 set_rq_online(rq); 6405 } 6406 rq_unlock_irqrestore(rq, &rf); 6407 6408 return 0; 6409 } 6410 6411 int sched_cpu_deactivate(unsigned int cpu) 6412 { 6413 int ret; 6414 6415 set_cpu_active(cpu, false); 6416 /* 6417 * We've cleared cpu_active_mask, wait for all preempt-disabled and RCU 6418 * users of this state to go away such that all new such users will 6419 * observe it. 6420 * 6421 * Do sync before park smpboot threads to take care the rcu boost case. 6422 */ 6423 synchronize_rcu(); 6424 6425 #ifdef CONFIG_SCHED_SMT 6426 /* 6427 * When going down, decrement the number of cores with SMT present. 6428 */ 6429 if (cpumask_weight(cpu_smt_mask(cpu)) == 2) 6430 static_branch_dec_cpuslocked(&sched_smt_present); 6431 #endif 6432 6433 if (!sched_smp_initialized) 6434 return 0; 6435 6436 ret = cpuset_cpu_inactive(cpu); 6437 if (ret) { 6438 set_cpu_active(cpu, true); 6439 return ret; 6440 } 6441 sched_domains_numa_masks_clear(cpu); 6442 return 0; 6443 } 6444 6445 static void sched_rq_cpu_starting(unsigned int cpu) 6446 { 6447 struct rq *rq = cpu_rq(cpu); 6448 6449 rq->calc_load_update = calc_load_update; 6450 update_max_interval(); 6451 } 6452 6453 int sched_cpu_starting(unsigned int cpu) 6454 { 6455 sched_rq_cpu_starting(cpu); 6456 sched_tick_start(cpu); 6457 return 0; 6458 } 6459 6460 #ifdef CONFIG_HOTPLUG_CPU 6461 int sched_cpu_dying(unsigned int cpu) 6462 { 6463 struct rq *rq = cpu_rq(cpu); 6464 struct rq_flags rf; 6465 6466 /* Handle pending wakeups and then migrate everything off */ 6467 sched_ttwu_pending(); 6468 sched_tick_stop(cpu); 6469 6470 rq_lock_irqsave(rq, &rf); 6471 if (rq->rd) { 6472 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); 6473 set_rq_offline(rq); 6474 } 6475 migrate_tasks(rq, &rf); 6476 BUG_ON(rq->nr_running != 1); 6477 rq_unlock_irqrestore(rq, &rf); 6478 6479 calc_load_migrate(rq); 6480 update_max_interval(); 6481 nohz_balance_exit_idle(rq); 6482 hrtick_clear(rq); 6483 return 0; 6484 } 6485 #endif 6486 6487 void __init sched_init_smp(void) 6488 { 6489 sched_init_numa(); 6490 6491 /* 6492 * There's no userspace yet to cause hotplug operations; hence all the 6493 * CPU masks are stable and all blatant races in the below code cannot 6494 * happen. 6495 */ 6496 mutex_lock(&sched_domains_mutex); 6497 sched_init_domains(cpu_active_mask); 6498 mutex_unlock(&sched_domains_mutex); 6499 6500 /* Move init over to a non-isolated CPU */ 6501 if (set_cpus_allowed_ptr(current, housekeeping_cpumask(HK_FLAG_DOMAIN)) < 0) 6502 BUG(); 6503 sched_init_granularity(); 6504 6505 init_sched_rt_class(); 6506 init_sched_dl_class(); 6507 6508 sched_smp_initialized = true; 6509 } 6510 6511 static int __init migration_init(void) 6512 { 6513 sched_cpu_starting(smp_processor_id()); 6514 return 0; 6515 } 6516 early_initcall(migration_init); 6517 6518 #else 6519 void __init sched_init_smp(void) 6520 { 6521 sched_init_granularity(); 6522 } 6523 #endif /* CONFIG_SMP */ 6524 6525 int in_sched_functions(unsigned long addr) 6526 { 6527 return in_lock_functions(addr) || 6528 (addr >= (unsigned long)__sched_text_start 6529 && addr < (unsigned long)__sched_text_end); 6530 } 6531 6532 #ifdef CONFIG_CGROUP_SCHED 6533 /* 6534 * Default task group. 6535 * Every task in system belongs to this group at bootup. 6536 */ 6537 struct task_group root_task_group; 6538 LIST_HEAD(task_groups); 6539 6540 /* Cacheline aligned slab cache for task_group */ 6541 static struct kmem_cache *task_group_cache __read_mostly; 6542 #endif 6543 6544 DECLARE_PER_CPU(cpumask_var_t, load_balance_mask); 6545 DECLARE_PER_CPU(cpumask_var_t, select_idle_mask); 6546 6547 void __init sched_init(void) 6548 { 6549 unsigned long ptr = 0; 6550 int i; 6551 6552 wait_bit_init(); 6553 6554 #ifdef CONFIG_FAIR_GROUP_SCHED 6555 ptr += 2 * nr_cpu_ids * sizeof(void **); 6556 #endif 6557 #ifdef CONFIG_RT_GROUP_SCHED 6558 ptr += 2 * nr_cpu_ids * sizeof(void **); 6559 #endif 6560 if (ptr) { 6561 ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT); 6562 6563 #ifdef CONFIG_FAIR_GROUP_SCHED 6564 root_task_group.se = (struct sched_entity **)ptr; 6565 ptr += nr_cpu_ids * sizeof(void **); 6566 6567 root_task_group.cfs_rq = (struct cfs_rq **)ptr; 6568 ptr += nr_cpu_ids * sizeof(void **); 6569 6570 #endif /* CONFIG_FAIR_GROUP_SCHED */ 6571 #ifdef CONFIG_RT_GROUP_SCHED 6572 root_task_group.rt_se = (struct sched_rt_entity **)ptr; 6573 ptr += nr_cpu_ids * sizeof(void **); 6574 6575 root_task_group.rt_rq = (struct rt_rq **)ptr; 6576 ptr += nr_cpu_ids * sizeof(void **); 6577 6578 #endif /* CONFIG_RT_GROUP_SCHED */ 6579 } 6580 #ifdef CONFIG_CPUMASK_OFFSTACK 6581 for_each_possible_cpu(i) { 6582 per_cpu(load_balance_mask, i) = (cpumask_var_t)kzalloc_node( 6583 cpumask_size(), GFP_KERNEL, cpu_to_node(i)); 6584 per_cpu(select_idle_mask, i) = (cpumask_var_t)kzalloc_node( 6585 cpumask_size(), GFP_KERNEL, cpu_to_node(i)); 6586 } 6587 #endif /* CONFIG_CPUMASK_OFFSTACK */ 6588 6589 init_rt_bandwidth(&def_rt_bandwidth, global_rt_period(), global_rt_runtime()); 6590 init_dl_bandwidth(&def_dl_bandwidth, global_rt_period(), global_rt_runtime()); 6591 6592 #ifdef CONFIG_SMP 6593 init_defrootdomain(); 6594 #endif 6595 6596 #ifdef CONFIG_RT_GROUP_SCHED 6597 init_rt_bandwidth(&root_task_group.rt_bandwidth, 6598 global_rt_period(), global_rt_runtime()); 6599 #endif /* CONFIG_RT_GROUP_SCHED */ 6600 6601 #ifdef CONFIG_CGROUP_SCHED 6602 task_group_cache = KMEM_CACHE(task_group, 0); 6603 6604 list_add(&root_task_group.list, &task_groups); 6605 INIT_LIST_HEAD(&root_task_group.children); 6606 INIT_LIST_HEAD(&root_task_group.siblings); 6607 autogroup_init(&init_task); 6608 #endif /* CONFIG_CGROUP_SCHED */ 6609 6610 for_each_possible_cpu(i) { 6611 struct rq *rq; 6612 6613 rq = cpu_rq(i); 6614 raw_spin_lock_init(&rq->lock); 6615 rq->nr_running = 0; 6616 rq->calc_load_active = 0; 6617 rq->calc_load_update = jiffies + LOAD_FREQ; 6618 init_cfs_rq(&rq->cfs); 6619 init_rt_rq(&rq->rt); 6620 init_dl_rq(&rq->dl); 6621 #ifdef CONFIG_FAIR_GROUP_SCHED 6622 root_task_group.shares = ROOT_TASK_GROUP_LOAD; 6623 INIT_LIST_HEAD(&rq->leaf_cfs_rq_list); 6624 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 6625 /* 6626 * How much CPU bandwidth does root_task_group get? 6627 * 6628 * In case of task-groups formed thr' the cgroup filesystem, it 6629 * gets 100% of the CPU resources in the system. This overall 6630 * system CPU resource is divided among the tasks of 6631 * root_task_group and its child task-groups in a fair manner, 6632 * based on each entity's (task or task-group's) weight 6633 * (se->load.weight). 6634 * 6635 * In other words, if root_task_group has 10 tasks of weight 6636 * 1024) and two child groups A0 and A1 (of weight 1024 each), 6637 * then A0's share of the CPU resource is: 6638 * 6639 * A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33% 6640 * 6641 * We achieve this by letting root_task_group's tasks sit 6642 * directly in rq->cfs (i.e root_task_group->se[] = NULL). 6643 */ 6644 init_cfs_bandwidth(&root_task_group.cfs_bandwidth); 6645 init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL); 6646 #endif /* CONFIG_FAIR_GROUP_SCHED */ 6647 6648 rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime; 6649 #ifdef CONFIG_RT_GROUP_SCHED 6650 init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL); 6651 #endif 6652 #ifdef CONFIG_SMP 6653 rq->sd = NULL; 6654 rq->rd = NULL; 6655 rq->cpu_capacity = rq->cpu_capacity_orig = SCHED_CAPACITY_SCALE; 6656 rq->balance_callback = NULL; 6657 rq->active_balance = 0; 6658 rq->next_balance = jiffies; 6659 rq->push_cpu = 0; 6660 rq->cpu = i; 6661 rq->online = 0; 6662 rq->idle_stamp = 0; 6663 rq->avg_idle = 2*sysctl_sched_migration_cost; 6664 rq->max_idle_balance_cost = sysctl_sched_migration_cost; 6665 6666 INIT_LIST_HEAD(&rq->cfs_tasks); 6667 6668 rq_attach_root(rq, &def_root_domain); 6669 #ifdef CONFIG_NO_HZ_COMMON 6670 rq->last_load_update_tick = jiffies; 6671 rq->last_blocked_load_update_tick = jiffies; 6672 atomic_set(&rq->nohz_flags, 0); 6673 #endif 6674 #endif /* CONFIG_SMP */ 6675 hrtick_rq_init(rq); 6676 atomic_set(&rq->nr_iowait, 0); 6677 } 6678 6679 set_load_weight(&init_task, false); 6680 6681 /* 6682 * The boot idle thread does lazy MMU switching as well: 6683 */ 6684 mmgrab(&init_mm); 6685 enter_lazy_tlb(&init_mm, current); 6686 6687 /* 6688 * Make us the idle thread. Technically, schedule() should not be 6689 * called from this thread, however somewhere below it might be, 6690 * but because we are the idle thread, we just pick up running again 6691 * when this runqueue becomes "idle". 6692 */ 6693 init_idle(current, smp_processor_id()); 6694 6695 calc_load_update = jiffies + LOAD_FREQ; 6696 6697 #ifdef CONFIG_SMP 6698 idle_thread_set_boot_cpu(); 6699 #endif 6700 init_sched_fair_class(); 6701 6702 init_schedstats(); 6703 6704 psi_init(); 6705 6706 init_uclamp(); 6707 6708 scheduler_running = 1; 6709 } 6710 6711 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP 6712 static inline int preempt_count_equals(int preempt_offset) 6713 { 6714 int nested = preempt_count() + rcu_preempt_depth(); 6715 6716 return (nested == preempt_offset); 6717 } 6718 6719 void __might_sleep(const char *file, int line, int preempt_offset) 6720 { 6721 /* 6722 * Blocking primitives will set (and therefore destroy) current->state, 6723 * since we will exit with TASK_RUNNING make sure we enter with it, 6724 * otherwise we will destroy state. 6725 */ 6726 WARN_ONCE(current->state != TASK_RUNNING && current->task_state_change, 6727 "do not call blocking ops when !TASK_RUNNING; " 6728 "state=%lx set at [<%p>] %pS\n", 6729 current->state, 6730 (void *)current->task_state_change, 6731 (void *)current->task_state_change); 6732 6733 ___might_sleep(file, line, preempt_offset); 6734 } 6735 EXPORT_SYMBOL(__might_sleep); 6736 6737 void ___might_sleep(const char *file, int line, int preempt_offset) 6738 { 6739 /* Ratelimiting timestamp: */ 6740 static unsigned long prev_jiffy; 6741 6742 unsigned long preempt_disable_ip; 6743 6744 /* WARN_ON_ONCE() by default, no rate limit required: */ 6745 rcu_sleep_check(); 6746 6747 if ((preempt_count_equals(preempt_offset) && !irqs_disabled() && 6748 !is_idle_task(current) && !current->non_block_count) || 6749 system_state == SYSTEM_BOOTING || system_state > SYSTEM_RUNNING || 6750 oops_in_progress) 6751 return; 6752 6753 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) 6754 return; 6755 prev_jiffy = jiffies; 6756 6757 /* Save this before calling printk(), since that will clobber it: */ 6758 preempt_disable_ip = get_preempt_disable_ip(current); 6759 6760 printk(KERN_ERR 6761 "BUG: sleeping function called from invalid context at %s:%d\n", 6762 file, line); 6763 printk(KERN_ERR 6764 "in_atomic(): %d, irqs_disabled(): %d, non_block: %d, pid: %d, name: %s\n", 6765 in_atomic(), irqs_disabled(), current->non_block_count, 6766 current->pid, current->comm); 6767 6768 if (task_stack_end_corrupted(current)) 6769 printk(KERN_EMERG "Thread overran stack, or stack corrupted\n"); 6770 6771 debug_show_held_locks(current); 6772 if (irqs_disabled()) 6773 print_irqtrace_events(current); 6774 if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) 6775 && !preempt_count_equals(preempt_offset)) { 6776 pr_err("Preemption disabled at:"); 6777 print_ip_sym(preempt_disable_ip); 6778 pr_cont("\n"); 6779 } 6780 dump_stack(); 6781 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 6782 } 6783 EXPORT_SYMBOL(___might_sleep); 6784 6785 void __cant_sleep(const char *file, int line, int preempt_offset) 6786 { 6787 static unsigned long prev_jiffy; 6788 6789 if (irqs_disabled()) 6790 return; 6791 6792 if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) 6793 return; 6794 6795 if (preempt_count() > preempt_offset) 6796 return; 6797 6798 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) 6799 return; 6800 prev_jiffy = jiffies; 6801 6802 printk(KERN_ERR "BUG: assuming atomic context at %s:%d\n", file, line); 6803 printk(KERN_ERR "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n", 6804 in_atomic(), irqs_disabled(), 6805 current->pid, current->comm); 6806 6807 debug_show_held_locks(current); 6808 dump_stack(); 6809 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 6810 } 6811 EXPORT_SYMBOL_GPL(__cant_sleep); 6812 #endif 6813 6814 #ifdef CONFIG_MAGIC_SYSRQ 6815 void normalize_rt_tasks(void) 6816 { 6817 struct task_struct *g, *p; 6818 struct sched_attr attr = { 6819 .sched_policy = SCHED_NORMAL, 6820 }; 6821 6822 read_lock(&tasklist_lock); 6823 for_each_process_thread(g, p) { 6824 /* 6825 * Only normalize user tasks: 6826 */ 6827 if (p->flags & PF_KTHREAD) 6828 continue; 6829 6830 p->se.exec_start = 0; 6831 schedstat_set(p->se.statistics.wait_start, 0); 6832 schedstat_set(p->se.statistics.sleep_start, 0); 6833 schedstat_set(p->se.statistics.block_start, 0); 6834 6835 if (!dl_task(p) && !rt_task(p)) { 6836 /* 6837 * Renice negative nice level userspace 6838 * tasks back to 0: 6839 */ 6840 if (task_nice(p) < 0) 6841 set_user_nice(p, 0); 6842 continue; 6843 } 6844 6845 __sched_setscheduler(p, &attr, false, false); 6846 } 6847 read_unlock(&tasklist_lock); 6848 } 6849 6850 #endif /* CONFIG_MAGIC_SYSRQ */ 6851 6852 #if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) 6853 /* 6854 * These functions are only useful for the IA64 MCA handling, or kdb. 6855 * 6856 * They can only be called when the whole system has been 6857 * stopped - every CPU needs to be quiescent, and no scheduling 6858 * activity can take place. Using them for anything else would 6859 * be a serious bug, and as a result, they aren't even visible 6860 * under any other configuration. 6861 */ 6862 6863 /** 6864 * curr_task - return the current task for a given CPU. 6865 * @cpu: the processor in question. 6866 * 6867 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! 6868 * 6869 * Return: The current task for @cpu. 6870 */ 6871 struct task_struct *curr_task(int cpu) 6872 { 6873 return cpu_curr(cpu); 6874 } 6875 6876 #endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */ 6877 6878 #ifdef CONFIG_IA64 6879 /** 6880 * ia64_set_curr_task - set the current task for a given CPU. 6881 * @cpu: the processor in question. 6882 * @p: the task pointer to set. 6883 * 6884 * Description: This function must only be used when non-maskable interrupts 6885 * are serviced on a separate stack. It allows the architecture to switch the 6886 * notion of the current task on a CPU in a non-blocking manner. This function 6887 * must be called with all CPU's synchronized, and interrupts disabled, the 6888 * and caller must save the original value of the current task (see 6889 * curr_task() above) and restore that value before reenabling interrupts and 6890 * re-starting the system. 6891 * 6892 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! 6893 */ 6894 void ia64_set_curr_task(int cpu, struct task_struct *p) 6895 { 6896 cpu_curr(cpu) = p; 6897 } 6898 6899 #endif 6900 6901 #ifdef CONFIG_CGROUP_SCHED 6902 /* task_group_lock serializes the addition/removal of task groups */ 6903 static DEFINE_SPINLOCK(task_group_lock); 6904 6905 static inline void alloc_uclamp_sched_group(struct task_group *tg, 6906 struct task_group *parent) 6907 { 6908 #ifdef CONFIG_UCLAMP_TASK_GROUP 6909 enum uclamp_id clamp_id; 6910 6911 for_each_clamp_id(clamp_id) { 6912 uclamp_se_set(&tg->uclamp_req[clamp_id], 6913 uclamp_none(clamp_id), false); 6914 tg->uclamp[clamp_id] = parent->uclamp[clamp_id]; 6915 } 6916 #endif 6917 } 6918 6919 static void sched_free_group(struct task_group *tg) 6920 { 6921 free_fair_sched_group(tg); 6922 free_rt_sched_group(tg); 6923 autogroup_free(tg); 6924 kmem_cache_free(task_group_cache, tg); 6925 } 6926 6927 /* allocate runqueue etc for a new task group */ 6928 struct task_group *sched_create_group(struct task_group *parent) 6929 { 6930 struct task_group *tg; 6931 6932 tg = kmem_cache_alloc(task_group_cache, GFP_KERNEL | __GFP_ZERO); 6933 if (!tg) 6934 return ERR_PTR(-ENOMEM); 6935 6936 if (!alloc_fair_sched_group(tg, parent)) 6937 goto err; 6938 6939 if (!alloc_rt_sched_group(tg, parent)) 6940 goto err; 6941 6942 alloc_uclamp_sched_group(tg, parent); 6943 6944 return tg; 6945 6946 err: 6947 sched_free_group(tg); 6948 return ERR_PTR(-ENOMEM); 6949 } 6950 6951 void sched_online_group(struct task_group *tg, struct task_group *parent) 6952 { 6953 unsigned long flags; 6954 6955 spin_lock_irqsave(&task_group_lock, flags); 6956 list_add_rcu(&tg->list, &task_groups); 6957 6958 /* Root should already exist: */ 6959 WARN_ON(!parent); 6960 6961 tg->parent = parent; 6962 INIT_LIST_HEAD(&tg->children); 6963 list_add_rcu(&tg->siblings, &parent->children); 6964 spin_unlock_irqrestore(&task_group_lock, flags); 6965 6966 online_fair_sched_group(tg); 6967 } 6968 6969 /* rcu callback to free various structures associated with a task group */ 6970 static void sched_free_group_rcu(struct rcu_head *rhp) 6971 { 6972 /* Now it should be safe to free those cfs_rqs: */ 6973 sched_free_group(container_of(rhp, struct task_group, rcu)); 6974 } 6975 6976 void sched_destroy_group(struct task_group *tg) 6977 { 6978 /* Wait for possible concurrent references to cfs_rqs complete: */ 6979 call_rcu(&tg->rcu, sched_free_group_rcu); 6980 } 6981 6982 void sched_offline_group(struct task_group *tg) 6983 { 6984 unsigned long flags; 6985 6986 /* End participation in shares distribution: */ 6987 unregister_fair_sched_group(tg); 6988 6989 spin_lock_irqsave(&task_group_lock, flags); 6990 list_del_rcu(&tg->list); 6991 list_del_rcu(&tg->siblings); 6992 spin_unlock_irqrestore(&task_group_lock, flags); 6993 } 6994 6995 static void sched_change_group(struct task_struct *tsk, int type) 6996 { 6997 struct task_group *tg; 6998 6999 /* 7000 * All callers are synchronized by task_rq_lock(); we do not use RCU 7001 * which is pointless here. Thus, we pass "true" to task_css_check() 7002 * to prevent lockdep warnings. 7003 */ 7004 tg = container_of(task_css_check(tsk, cpu_cgrp_id, true), 7005 struct task_group, css); 7006 tg = autogroup_task_group(tsk, tg); 7007 tsk->sched_task_group = tg; 7008 7009 #ifdef CONFIG_FAIR_GROUP_SCHED 7010 if (tsk->sched_class->task_change_group) 7011 tsk->sched_class->task_change_group(tsk, type); 7012 else 7013 #endif 7014 set_task_rq(tsk, task_cpu(tsk)); 7015 } 7016 7017 /* 7018 * Change task's runqueue when it moves between groups. 7019 * 7020 * The caller of this function should have put the task in its new group by 7021 * now. This function just updates tsk->se.cfs_rq and tsk->se.parent to reflect 7022 * its new group. 7023 */ 7024 void sched_move_task(struct task_struct *tsk) 7025 { 7026 int queued, running, queue_flags = 7027 DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; 7028 struct rq_flags rf; 7029 struct rq *rq; 7030 7031 rq = task_rq_lock(tsk, &rf); 7032 update_rq_clock(rq); 7033 7034 running = task_current(rq, tsk); 7035 queued = task_on_rq_queued(tsk); 7036 7037 if (queued) 7038 dequeue_task(rq, tsk, queue_flags); 7039 if (running) 7040 put_prev_task(rq, tsk); 7041 7042 sched_change_group(tsk, TASK_MOVE_GROUP); 7043 7044 if (queued) 7045 enqueue_task(rq, tsk, queue_flags); 7046 if (running) 7047 set_next_task(rq, tsk); 7048 7049 task_rq_unlock(rq, tsk, &rf); 7050 } 7051 7052 static inline struct task_group *css_tg(struct cgroup_subsys_state *css) 7053 { 7054 return css ? container_of(css, struct task_group, css) : NULL; 7055 } 7056 7057 static struct cgroup_subsys_state * 7058 cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) 7059 { 7060 struct task_group *parent = css_tg(parent_css); 7061 struct task_group *tg; 7062 7063 if (!parent) { 7064 /* This is early initialization for the top cgroup */ 7065 return &root_task_group.css; 7066 } 7067 7068 tg = sched_create_group(parent); 7069 if (IS_ERR(tg)) 7070 return ERR_PTR(-ENOMEM); 7071 7072 return &tg->css; 7073 } 7074 7075 /* Expose task group only after completing cgroup initialization */ 7076 static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) 7077 { 7078 struct task_group *tg = css_tg(css); 7079 struct task_group *parent = css_tg(css->parent); 7080 7081 if (parent) 7082 sched_online_group(tg, parent); 7083 return 0; 7084 } 7085 7086 static void cpu_cgroup_css_released(struct cgroup_subsys_state *css) 7087 { 7088 struct task_group *tg = css_tg(css); 7089 7090 sched_offline_group(tg); 7091 } 7092 7093 static void cpu_cgroup_css_free(struct cgroup_subsys_state *css) 7094 { 7095 struct task_group *tg = css_tg(css); 7096 7097 /* 7098 * Relies on the RCU grace period between css_released() and this. 7099 */ 7100 sched_free_group(tg); 7101 } 7102 7103 /* 7104 * This is called before wake_up_new_task(), therefore we really only 7105 * have to set its group bits, all the other stuff does not apply. 7106 */ 7107 static void cpu_cgroup_fork(struct task_struct *task) 7108 { 7109 struct rq_flags rf; 7110 struct rq *rq; 7111 7112 rq = task_rq_lock(task, &rf); 7113 7114 update_rq_clock(rq); 7115 sched_change_group(task, TASK_SET_GROUP); 7116 7117 task_rq_unlock(rq, task, &rf); 7118 } 7119 7120 static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) 7121 { 7122 struct task_struct *task; 7123 struct cgroup_subsys_state *css; 7124 int ret = 0; 7125 7126 cgroup_taskset_for_each(task, css, tset) { 7127 #ifdef CONFIG_RT_GROUP_SCHED 7128 if (!sched_rt_can_attach(css_tg(css), task)) 7129 return -EINVAL; 7130 #endif 7131 /* 7132 * Serialize against wake_up_new_task() such that if its 7133 * running, we're sure to observe its full state. 7134 */ 7135 raw_spin_lock_irq(&task->pi_lock); 7136 /* 7137 * Avoid calling sched_move_task() before wake_up_new_task() 7138 * has happened. This would lead to problems with PELT, due to 7139 * move wanting to detach+attach while we're not attached yet. 7140 */ 7141 if (task->state == TASK_NEW) 7142 ret = -EINVAL; 7143 raw_spin_unlock_irq(&task->pi_lock); 7144 7145 if (ret) 7146 break; 7147 } 7148 return ret; 7149 } 7150 7151 static void cpu_cgroup_attach(struct cgroup_taskset *tset) 7152 { 7153 struct task_struct *task; 7154 struct cgroup_subsys_state *css; 7155 7156 cgroup_taskset_for_each(task, css, tset) 7157 sched_move_task(task); 7158 } 7159 7160 #ifdef CONFIG_UCLAMP_TASK_GROUP 7161 static void cpu_util_update_eff(struct cgroup_subsys_state *css) 7162 { 7163 struct cgroup_subsys_state *top_css = css; 7164 struct uclamp_se *uc_parent = NULL; 7165 struct uclamp_se *uc_se = NULL; 7166 unsigned int eff[UCLAMP_CNT]; 7167 enum uclamp_id clamp_id; 7168 unsigned int clamps; 7169 7170 css_for_each_descendant_pre(css, top_css) { 7171 uc_parent = css_tg(css)->parent 7172 ? css_tg(css)->parent->uclamp : NULL; 7173 7174 for_each_clamp_id(clamp_id) { 7175 /* Assume effective clamps matches requested clamps */ 7176 eff[clamp_id] = css_tg(css)->uclamp_req[clamp_id].value; 7177 /* Cap effective clamps with parent's effective clamps */ 7178 if (uc_parent && 7179 eff[clamp_id] > uc_parent[clamp_id].value) { 7180 eff[clamp_id] = uc_parent[clamp_id].value; 7181 } 7182 } 7183 /* Ensure protection is always capped by limit */ 7184 eff[UCLAMP_MIN] = min(eff[UCLAMP_MIN], eff[UCLAMP_MAX]); 7185 7186 /* Propagate most restrictive effective clamps */ 7187 clamps = 0x0; 7188 uc_se = css_tg(css)->uclamp; 7189 for_each_clamp_id(clamp_id) { 7190 if (eff[clamp_id] == uc_se[clamp_id].value) 7191 continue; 7192 uc_se[clamp_id].value = eff[clamp_id]; 7193 uc_se[clamp_id].bucket_id = uclamp_bucket_id(eff[clamp_id]); 7194 clamps |= (0x1 << clamp_id); 7195 } 7196 if (!clamps) { 7197 css = css_rightmost_descendant(css); 7198 continue; 7199 } 7200 7201 /* Immediately update descendants RUNNABLE tasks */ 7202 uclamp_update_active_tasks(css, clamps); 7203 } 7204 } 7205 7206 /* 7207 * Integer 10^N with a given N exponent by casting to integer the literal "1eN" 7208 * C expression. Since there is no way to convert a macro argument (N) into a 7209 * character constant, use two levels of macros. 7210 */ 7211 #define _POW10(exp) ((unsigned int)1e##exp) 7212 #define POW10(exp) _POW10(exp) 7213 7214 struct uclamp_request { 7215 #define UCLAMP_PERCENT_SHIFT 2 7216 #define UCLAMP_PERCENT_SCALE (100 * POW10(UCLAMP_PERCENT_SHIFT)) 7217 s64 percent; 7218 u64 util; 7219 int ret; 7220 }; 7221 7222 static inline struct uclamp_request 7223 capacity_from_percent(char *buf) 7224 { 7225 struct uclamp_request req = { 7226 .percent = UCLAMP_PERCENT_SCALE, 7227 .util = SCHED_CAPACITY_SCALE, 7228 .ret = 0, 7229 }; 7230 7231 buf = strim(buf); 7232 if (strcmp(buf, "max")) { 7233 req.ret = cgroup_parse_float(buf, UCLAMP_PERCENT_SHIFT, 7234 &req.percent); 7235 if (req.ret) 7236 return req; 7237 if (req.percent > UCLAMP_PERCENT_SCALE) { 7238 req.ret = -ERANGE; 7239 return req; 7240 } 7241 7242 req.util = req.percent << SCHED_CAPACITY_SHIFT; 7243 req.util = DIV_ROUND_CLOSEST_ULL(req.util, UCLAMP_PERCENT_SCALE); 7244 } 7245 7246 return req; 7247 } 7248 7249 static ssize_t cpu_uclamp_write(struct kernfs_open_file *of, char *buf, 7250 size_t nbytes, loff_t off, 7251 enum uclamp_id clamp_id) 7252 { 7253 struct uclamp_request req; 7254 struct task_group *tg; 7255 7256 req = capacity_from_percent(buf); 7257 if (req.ret) 7258 return req.ret; 7259 7260 mutex_lock(&uclamp_mutex); 7261 rcu_read_lock(); 7262 7263 tg = css_tg(of_css(of)); 7264 if (tg->uclamp_req[clamp_id].value != req.util) 7265 uclamp_se_set(&tg->uclamp_req[clamp_id], req.util, false); 7266 7267 /* 7268 * Because of not recoverable conversion rounding we keep track of the 7269 * exact requested value 7270 */ 7271 tg->uclamp_pct[clamp_id] = req.percent; 7272 7273 /* Update effective clamps to track the most restrictive value */ 7274 cpu_util_update_eff(of_css(of)); 7275 7276 rcu_read_unlock(); 7277 mutex_unlock(&uclamp_mutex); 7278 7279 return nbytes; 7280 } 7281 7282 static ssize_t cpu_uclamp_min_write(struct kernfs_open_file *of, 7283 char *buf, size_t nbytes, 7284 loff_t off) 7285 { 7286 return cpu_uclamp_write(of, buf, nbytes, off, UCLAMP_MIN); 7287 } 7288 7289 static ssize_t cpu_uclamp_max_write(struct kernfs_open_file *of, 7290 char *buf, size_t nbytes, 7291 loff_t off) 7292 { 7293 return cpu_uclamp_write(of, buf, nbytes, off, UCLAMP_MAX); 7294 } 7295 7296 static inline void cpu_uclamp_print(struct seq_file *sf, 7297 enum uclamp_id clamp_id) 7298 { 7299 struct task_group *tg; 7300 u64 util_clamp; 7301 u64 percent; 7302 u32 rem; 7303 7304 rcu_read_lock(); 7305 tg = css_tg(seq_css(sf)); 7306 util_clamp = tg->uclamp_req[clamp_id].value; 7307 rcu_read_unlock(); 7308 7309 if (util_clamp == SCHED_CAPACITY_SCALE) { 7310 seq_puts(sf, "max\n"); 7311 return; 7312 } 7313 7314 percent = tg->uclamp_pct[clamp_id]; 7315 percent = div_u64_rem(percent, POW10(UCLAMP_PERCENT_SHIFT), &rem); 7316 seq_printf(sf, "%llu.%0*u\n", percent, UCLAMP_PERCENT_SHIFT, rem); 7317 } 7318 7319 static int cpu_uclamp_min_show(struct seq_file *sf, void *v) 7320 { 7321 cpu_uclamp_print(sf, UCLAMP_MIN); 7322 return 0; 7323 } 7324 7325 static int cpu_uclamp_max_show(struct seq_file *sf, void *v) 7326 { 7327 cpu_uclamp_print(sf, UCLAMP_MAX); 7328 return 0; 7329 } 7330 #endif /* CONFIG_UCLAMP_TASK_GROUP */ 7331 7332 #ifdef CONFIG_FAIR_GROUP_SCHED 7333 static int cpu_shares_write_u64(struct cgroup_subsys_state *css, 7334 struct cftype *cftype, u64 shareval) 7335 { 7336 if (shareval > scale_load_down(ULONG_MAX)) 7337 shareval = MAX_SHARES; 7338 return sched_group_set_shares(css_tg(css), scale_load(shareval)); 7339 } 7340 7341 static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css, 7342 struct cftype *cft) 7343 { 7344 struct task_group *tg = css_tg(css); 7345 7346 return (u64) scale_load_down(tg->shares); 7347 } 7348 7349 #ifdef CONFIG_CFS_BANDWIDTH 7350 static DEFINE_MUTEX(cfs_constraints_mutex); 7351 7352 const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */ 7353 static const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */ 7354 7355 static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime); 7356 7357 static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota) 7358 { 7359 int i, ret = 0, runtime_enabled, runtime_was_enabled; 7360 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 7361 7362 if (tg == &root_task_group) 7363 return -EINVAL; 7364 7365 /* 7366 * Ensure we have at some amount of bandwidth every period. This is 7367 * to prevent reaching a state of large arrears when throttled via 7368 * entity_tick() resulting in prolonged exit starvation. 7369 */ 7370 if (quota < min_cfs_quota_period || period < min_cfs_quota_period) 7371 return -EINVAL; 7372 7373 /* 7374 * Likewise, bound things on the otherside by preventing insane quota 7375 * periods. This also allows us to normalize in computing quota 7376 * feasibility. 7377 */ 7378 if (period > max_cfs_quota_period) 7379 return -EINVAL; 7380 7381 /* 7382 * Prevent race between setting of cfs_rq->runtime_enabled and 7383 * unthrottle_offline_cfs_rqs(). 7384 */ 7385 get_online_cpus(); 7386 mutex_lock(&cfs_constraints_mutex); 7387 ret = __cfs_schedulable(tg, period, quota); 7388 if (ret) 7389 goto out_unlock; 7390 7391 runtime_enabled = quota != RUNTIME_INF; 7392 runtime_was_enabled = cfs_b->quota != RUNTIME_INF; 7393 /* 7394 * If we need to toggle cfs_bandwidth_used, off->on must occur 7395 * before making related changes, and on->off must occur afterwards 7396 */ 7397 if (runtime_enabled && !runtime_was_enabled) 7398 cfs_bandwidth_usage_inc(); 7399 raw_spin_lock_irq(&cfs_b->lock); 7400 cfs_b->period = ns_to_ktime(period); 7401 cfs_b->quota = quota; 7402 7403 __refill_cfs_bandwidth_runtime(cfs_b); 7404 7405 /* Restart the period timer (if active) to handle new period expiry: */ 7406 if (runtime_enabled) 7407 start_cfs_bandwidth(cfs_b); 7408 7409 raw_spin_unlock_irq(&cfs_b->lock); 7410 7411 for_each_online_cpu(i) { 7412 struct cfs_rq *cfs_rq = tg->cfs_rq[i]; 7413 struct rq *rq = cfs_rq->rq; 7414 struct rq_flags rf; 7415 7416 rq_lock_irq(rq, &rf); 7417 cfs_rq->runtime_enabled = runtime_enabled; 7418 cfs_rq->runtime_remaining = 0; 7419 7420 if (cfs_rq->throttled) 7421 unthrottle_cfs_rq(cfs_rq); 7422 rq_unlock_irq(rq, &rf); 7423 } 7424 if (runtime_was_enabled && !runtime_enabled) 7425 cfs_bandwidth_usage_dec(); 7426 out_unlock: 7427 mutex_unlock(&cfs_constraints_mutex); 7428 put_online_cpus(); 7429 7430 return ret; 7431 } 7432 7433 static int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us) 7434 { 7435 u64 quota, period; 7436 7437 period = ktime_to_ns(tg->cfs_bandwidth.period); 7438 if (cfs_quota_us < 0) 7439 quota = RUNTIME_INF; 7440 else if ((u64)cfs_quota_us <= U64_MAX / NSEC_PER_USEC) 7441 quota = (u64)cfs_quota_us * NSEC_PER_USEC; 7442 else 7443 return -EINVAL; 7444 7445 return tg_set_cfs_bandwidth(tg, period, quota); 7446 } 7447 7448 static long tg_get_cfs_quota(struct task_group *tg) 7449 { 7450 u64 quota_us; 7451 7452 if (tg->cfs_bandwidth.quota == RUNTIME_INF) 7453 return -1; 7454 7455 quota_us = tg->cfs_bandwidth.quota; 7456 do_div(quota_us, NSEC_PER_USEC); 7457 7458 return quota_us; 7459 } 7460 7461 static int tg_set_cfs_period(struct task_group *tg, long cfs_period_us) 7462 { 7463 u64 quota, period; 7464 7465 if ((u64)cfs_period_us > U64_MAX / NSEC_PER_USEC) 7466 return -EINVAL; 7467 7468 period = (u64)cfs_period_us * NSEC_PER_USEC; 7469 quota = tg->cfs_bandwidth.quota; 7470 7471 return tg_set_cfs_bandwidth(tg, period, quota); 7472 } 7473 7474 static long tg_get_cfs_period(struct task_group *tg) 7475 { 7476 u64 cfs_period_us; 7477 7478 cfs_period_us = ktime_to_ns(tg->cfs_bandwidth.period); 7479 do_div(cfs_period_us, NSEC_PER_USEC); 7480 7481 return cfs_period_us; 7482 } 7483 7484 static s64 cpu_cfs_quota_read_s64(struct cgroup_subsys_state *css, 7485 struct cftype *cft) 7486 { 7487 return tg_get_cfs_quota(css_tg(css)); 7488 } 7489 7490 static int cpu_cfs_quota_write_s64(struct cgroup_subsys_state *css, 7491 struct cftype *cftype, s64 cfs_quota_us) 7492 { 7493 return tg_set_cfs_quota(css_tg(css), cfs_quota_us); 7494 } 7495 7496 static u64 cpu_cfs_period_read_u64(struct cgroup_subsys_state *css, 7497 struct cftype *cft) 7498 { 7499 return tg_get_cfs_period(css_tg(css)); 7500 } 7501 7502 static int cpu_cfs_period_write_u64(struct cgroup_subsys_state *css, 7503 struct cftype *cftype, u64 cfs_period_us) 7504 { 7505 return tg_set_cfs_period(css_tg(css), cfs_period_us); 7506 } 7507 7508 struct cfs_schedulable_data { 7509 struct task_group *tg; 7510 u64 period, quota; 7511 }; 7512 7513 /* 7514 * normalize group quota/period to be quota/max_period 7515 * note: units are usecs 7516 */ 7517 static u64 normalize_cfs_quota(struct task_group *tg, 7518 struct cfs_schedulable_data *d) 7519 { 7520 u64 quota, period; 7521 7522 if (tg == d->tg) { 7523 period = d->period; 7524 quota = d->quota; 7525 } else { 7526 period = tg_get_cfs_period(tg); 7527 quota = tg_get_cfs_quota(tg); 7528 } 7529 7530 /* note: these should typically be equivalent */ 7531 if (quota == RUNTIME_INF || quota == -1) 7532 return RUNTIME_INF; 7533 7534 return to_ratio(period, quota); 7535 } 7536 7537 static int tg_cfs_schedulable_down(struct task_group *tg, void *data) 7538 { 7539 struct cfs_schedulable_data *d = data; 7540 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 7541 s64 quota = 0, parent_quota = -1; 7542 7543 if (!tg->parent) { 7544 quota = RUNTIME_INF; 7545 } else { 7546 struct cfs_bandwidth *parent_b = &tg->parent->cfs_bandwidth; 7547 7548 quota = normalize_cfs_quota(tg, d); 7549 parent_quota = parent_b->hierarchical_quota; 7550 7551 /* 7552 * Ensure max(child_quota) <= parent_quota. On cgroup2, 7553 * always take the min. On cgroup1, only inherit when no 7554 * limit is set: 7555 */ 7556 if (cgroup_subsys_on_dfl(cpu_cgrp_subsys)) { 7557 quota = min(quota, parent_quota); 7558 } else { 7559 if (quota == RUNTIME_INF) 7560 quota = parent_quota; 7561 else if (parent_quota != RUNTIME_INF && quota > parent_quota) 7562 return -EINVAL; 7563 } 7564 } 7565 cfs_b->hierarchical_quota = quota; 7566 7567 return 0; 7568 } 7569 7570 static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota) 7571 { 7572 int ret; 7573 struct cfs_schedulable_data data = { 7574 .tg = tg, 7575 .period = period, 7576 .quota = quota, 7577 }; 7578 7579 if (quota != RUNTIME_INF) { 7580 do_div(data.period, NSEC_PER_USEC); 7581 do_div(data.quota, NSEC_PER_USEC); 7582 } 7583 7584 rcu_read_lock(); 7585 ret = walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data); 7586 rcu_read_unlock(); 7587 7588 return ret; 7589 } 7590 7591 static int cpu_cfs_stat_show(struct seq_file *sf, void *v) 7592 { 7593 struct task_group *tg = css_tg(seq_css(sf)); 7594 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 7595 7596 seq_printf(sf, "nr_periods %d\n", cfs_b->nr_periods); 7597 seq_printf(sf, "nr_throttled %d\n", cfs_b->nr_throttled); 7598 seq_printf(sf, "throttled_time %llu\n", cfs_b->throttled_time); 7599 7600 if (schedstat_enabled() && tg != &root_task_group) { 7601 u64 ws = 0; 7602 int i; 7603 7604 for_each_possible_cpu(i) 7605 ws += schedstat_val(tg->se[i]->statistics.wait_sum); 7606 7607 seq_printf(sf, "wait_sum %llu\n", ws); 7608 } 7609 7610 return 0; 7611 } 7612 #endif /* CONFIG_CFS_BANDWIDTH */ 7613 #endif /* CONFIG_FAIR_GROUP_SCHED */ 7614 7615 #ifdef CONFIG_RT_GROUP_SCHED 7616 static int cpu_rt_runtime_write(struct cgroup_subsys_state *css, 7617 struct cftype *cft, s64 val) 7618 { 7619 return sched_group_set_rt_runtime(css_tg(css), val); 7620 } 7621 7622 static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css, 7623 struct cftype *cft) 7624 { 7625 return sched_group_rt_runtime(css_tg(css)); 7626 } 7627 7628 static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css, 7629 struct cftype *cftype, u64 rt_period_us) 7630 { 7631 return sched_group_set_rt_period(css_tg(css), rt_period_us); 7632 } 7633 7634 static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css, 7635 struct cftype *cft) 7636 { 7637 return sched_group_rt_period(css_tg(css)); 7638 } 7639 #endif /* CONFIG_RT_GROUP_SCHED */ 7640 7641 static struct cftype cpu_legacy_files[] = { 7642 #ifdef CONFIG_FAIR_GROUP_SCHED 7643 { 7644 .name = "shares", 7645 .read_u64 = cpu_shares_read_u64, 7646 .write_u64 = cpu_shares_write_u64, 7647 }, 7648 #endif 7649 #ifdef CONFIG_CFS_BANDWIDTH 7650 { 7651 .name = "cfs_quota_us", 7652 .read_s64 = cpu_cfs_quota_read_s64, 7653 .write_s64 = cpu_cfs_quota_write_s64, 7654 }, 7655 { 7656 .name = "cfs_period_us", 7657 .read_u64 = cpu_cfs_period_read_u64, 7658 .write_u64 = cpu_cfs_period_write_u64, 7659 }, 7660 { 7661 .name = "stat", 7662 .seq_show = cpu_cfs_stat_show, 7663 }, 7664 #endif 7665 #ifdef CONFIG_RT_GROUP_SCHED 7666 { 7667 .name = "rt_runtime_us", 7668 .read_s64 = cpu_rt_runtime_read, 7669 .write_s64 = cpu_rt_runtime_write, 7670 }, 7671 { 7672 .name = "rt_period_us", 7673 .read_u64 = cpu_rt_period_read_uint, 7674 .write_u64 = cpu_rt_period_write_uint, 7675 }, 7676 #endif 7677 #ifdef CONFIG_UCLAMP_TASK_GROUP 7678 { 7679 .name = "uclamp.min", 7680 .flags = CFTYPE_NOT_ON_ROOT, 7681 .seq_show = cpu_uclamp_min_show, 7682 .write = cpu_uclamp_min_write, 7683 }, 7684 { 7685 .name = "uclamp.max", 7686 .flags = CFTYPE_NOT_ON_ROOT, 7687 .seq_show = cpu_uclamp_max_show, 7688 .write = cpu_uclamp_max_write, 7689 }, 7690 #endif 7691 { } /* Terminate */ 7692 }; 7693 7694 static int cpu_extra_stat_show(struct seq_file *sf, 7695 struct cgroup_subsys_state *css) 7696 { 7697 #ifdef CONFIG_CFS_BANDWIDTH 7698 { 7699 struct task_group *tg = css_tg(css); 7700 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 7701 u64 throttled_usec; 7702 7703 throttled_usec = cfs_b->throttled_time; 7704 do_div(throttled_usec, NSEC_PER_USEC); 7705 7706 seq_printf(sf, "nr_periods %d\n" 7707 "nr_throttled %d\n" 7708 "throttled_usec %llu\n", 7709 cfs_b->nr_periods, cfs_b->nr_throttled, 7710 throttled_usec); 7711 } 7712 #endif 7713 return 0; 7714 } 7715 7716 #ifdef CONFIG_FAIR_GROUP_SCHED 7717 static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css, 7718 struct cftype *cft) 7719 { 7720 struct task_group *tg = css_tg(css); 7721 u64 weight = scale_load_down(tg->shares); 7722 7723 return DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024); 7724 } 7725 7726 static int cpu_weight_write_u64(struct cgroup_subsys_state *css, 7727 struct cftype *cft, u64 weight) 7728 { 7729 /* 7730 * cgroup weight knobs should use the common MIN, DFL and MAX 7731 * values which are 1, 100 and 10000 respectively. While it loses 7732 * a bit of range on both ends, it maps pretty well onto the shares 7733 * value used by scheduler and the round-trip conversions preserve 7734 * the original value over the entire range. 7735 */ 7736 if (weight < CGROUP_WEIGHT_MIN || weight > CGROUP_WEIGHT_MAX) 7737 return -ERANGE; 7738 7739 weight = DIV_ROUND_CLOSEST_ULL(weight * 1024, CGROUP_WEIGHT_DFL); 7740 7741 return sched_group_set_shares(css_tg(css), scale_load(weight)); 7742 } 7743 7744 static s64 cpu_weight_nice_read_s64(struct cgroup_subsys_state *css, 7745 struct cftype *cft) 7746 { 7747 unsigned long weight = scale_load_down(css_tg(css)->shares); 7748 int last_delta = INT_MAX; 7749 int prio, delta; 7750 7751 /* find the closest nice value to the current weight */ 7752 for (prio = 0; prio < ARRAY_SIZE(sched_prio_to_weight); prio++) { 7753 delta = abs(sched_prio_to_weight[prio] - weight); 7754 if (delta >= last_delta) 7755 break; 7756 last_delta = delta; 7757 } 7758 7759 return PRIO_TO_NICE(prio - 1 + MAX_RT_PRIO); 7760 } 7761 7762 static int cpu_weight_nice_write_s64(struct cgroup_subsys_state *css, 7763 struct cftype *cft, s64 nice) 7764 { 7765 unsigned long weight; 7766 int idx; 7767 7768 if (nice < MIN_NICE || nice > MAX_NICE) 7769 return -ERANGE; 7770 7771 idx = NICE_TO_PRIO(nice) - MAX_RT_PRIO; 7772 idx = array_index_nospec(idx, 40); 7773 weight = sched_prio_to_weight[idx]; 7774 7775 return sched_group_set_shares(css_tg(css), scale_load(weight)); 7776 } 7777 #endif 7778 7779 static void __maybe_unused cpu_period_quota_print(struct seq_file *sf, 7780 long period, long quota) 7781 { 7782 if (quota < 0) 7783 seq_puts(sf, "max"); 7784 else 7785 seq_printf(sf, "%ld", quota); 7786 7787 seq_printf(sf, " %ld\n", period); 7788 } 7789 7790 /* caller should put the current value in *@periodp before calling */ 7791 static int __maybe_unused cpu_period_quota_parse(char *buf, 7792 u64 *periodp, u64 *quotap) 7793 { 7794 char tok[21]; /* U64_MAX */ 7795 7796 if (sscanf(buf, "%20s %llu", tok, periodp) < 1) 7797 return -EINVAL; 7798 7799 *periodp *= NSEC_PER_USEC; 7800 7801 if (sscanf(tok, "%llu", quotap)) 7802 *quotap *= NSEC_PER_USEC; 7803 else if (!strcmp(tok, "max")) 7804 *quotap = RUNTIME_INF; 7805 else 7806 return -EINVAL; 7807 7808 return 0; 7809 } 7810 7811 #ifdef CONFIG_CFS_BANDWIDTH 7812 static int cpu_max_show(struct seq_file *sf, void *v) 7813 { 7814 struct task_group *tg = css_tg(seq_css(sf)); 7815 7816 cpu_period_quota_print(sf, tg_get_cfs_period(tg), tg_get_cfs_quota(tg)); 7817 return 0; 7818 } 7819 7820 static ssize_t cpu_max_write(struct kernfs_open_file *of, 7821 char *buf, size_t nbytes, loff_t off) 7822 { 7823 struct task_group *tg = css_tg(of_css(of)); 7824 u64 period = tg_get_cfs_period(tg); 7825 u64 quota; 7826 int ret; 7827 7828 ret = cpu_period_quota_parse(buf, &period, "a); 7829 if (!ret) 7830 ret = tg_set_cfs_bandwidth(tg, period, quota); 7831 return ret ?: nbytes; 7832 } 7833 #endif 7834 7835 static struct cftype cpu_files[] = { 7836 #ifdef CONFIG_FAIR_GROUP_SCHED 7837 { 7838 .name = "weight", 7839 .flags = CFTYPE_NOT_ON_ROOT, 7840 .read_u64 = cpu_weight_read_u64, 7841 .write_u64 = cpu_weight_write_u64, 7842 }, 7843 { 7844 .name = "weight.nice", 7845 .flags = CFTYPE_NOT_ON_ROOT, 7846 .read_s64 = cpu_weight_nice_read_s64, 7847 .write_s64 = cpu_weight_nice_write_s64, 7848 }, 7849 #endif 7850 #ifdef CONFIG_CFS_BANDWIDTH 7851 { 7852 .name = "max", 7853 .flags = CFTYPE_NOT_ON_ROOT, 7854 .seq_show = cpu_max_show, 7855 .write = cpu_max_write, 7856 }, 7857 #endif 7858 #ifdef CONFIG_UCLAMP_TASK_GROUP 7859 { 7860 .name = "uclamp.min", 7861 .flags = CFTYPE_NOT_ON_ROOT, 7862 .seq_show = cpu_uclamp_min_show, 7863 .write = cpu_uclamp_min_write, 7864 }, 7865 { 7866 .name = "uclamp.max", 7867 .flags = CFTYPE_NOT_ON_ROOT, 7868 .seq_show = cpu_uclamp_max_show, 7869 .write = cpu_uclamp_max_write, 7870 }, 7871 #endif 7872 { } /* terminate */ 7873 }; 7874 7875 struct cgroup_subsys cpu_cgrp_subsys = { 7876 .css_alloc = cpu_cgroup_css_alloc, 7877 .css_online = cpu_cgroup_css_online, 7878 .css_released = cpu_cgroup_css_released, 7879 .css_free = cpu_cgroup_css_free, 7880 .css_extra_stat_show = cpu_extra_stat_show, 7881 .fork = cpu_cgroup_fork, 7882 .can_attach = cpu_cgroup_can_attach, 7883 .attach = cpu_cgroup_attach, 7884 .legacy_cftypes = cpu_legacy_files, 7885 .dfl_cftypes = cpu_files, 7886 .early_init = true, 7887 .threaded = true, 7888 }; 7889 7890 #endif /* CONFIG_CGROUP_SCHED */ 7891 7892 void dump_cpu_task(int cpu) 7893 { 7894 pr_info("Task dump for CPU %d:\n", cpu); 7895 sched_show_task(cpu_curr(cpu)); 7896 } 7897 7898 /* 7899 * Nice levels are multiplicative, with a gentle 10% change for every 7900 * nice level changed. I.e. when a CPU-bound task goes from nice 0 to 7901 * nice 1, it will get ~10% less CPU time than another CPU-bound task 7902 * that remained on nice 0. 7903 * 7904 * The "10% effect" is relative and cumulative: from _any_ nice level, 7905 * if you go up 1 level, it's -10% CPU usage, if you go down 1 level 7906 * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25. 7907 * If a task goes up by ~10% and another task goes down by ~10% then 7908 * the relative distance between them is ~25%.) 7909 */ 7910 const int sched_prio_to_weight[40] = { 7911 /* -20 */ 88761, 71755, 56483, 46273, 36291, 7912 /* -15 */ 29154, 23254, 18705, 14949, 11916, 7913 /* -10 */ 9548, 7620, 6100, 4904, 3906, 7914 /* -5 */ 3121, 2501, 1991, 1586, 1277, 7915 /* 0 */ 1024, 820, 655, 526, 423, 7916 /* 5 */ 335, 272, 215, 172, 137, 7917 /* 10 */ 110, 87, 70, 56, 45, 7918 /* 15 */ 36, 29, 23, 18, 15, 7919 }; 7920 7921 /* 7922 * Inverse (2^32/x) values of the sched_prio_to_weight[] array, precalculated. 7923 * 7924 * In cases where the weight does not change often, we can use the 7925 * precalculated inverse to speed up arithmetics by turning divisions 7926 * into multiplications: 7927 */ 7928 const u32 sched_prio_to_wmult[40] = { 7929 /* -20 */ 48388, 59856, 76040, 92818, 118348, 7930 /* -15 */ 147320, 184698, 229616, 287308, 360437, 7931 /* -10 */ 449829, 563644, 704093, 875809, 1099582, 7932 /* -5 */ 1376151, 1717300, 2157191, 2708050, 3363326, 7933 /* 0 */ 4194304, 5237765, 6557202, 8165337, 10153587, 7934 /* 5 */ 12820798, 15790321, 19976592, 24970740, 31350126, 7935 /* 10 */ 39045157, 49367440, 61356676, 76695844, 95443717, 7936 /* 15 */ 119304647, 148102320, 186737708, 238609294, 286331153, 7937 }; 7938 7939 #undef CREATE_TRACE_POINTS 7940