1 /* 2 * kernel/sched/core.c 3 * 4 * Core kernel scheduler code and related syscalls 5 * 6 * Copyright (C) 1991-2002 Linus Torvalds 7 */ 8 #include "sched.h" 9 10 #include <linux/nospec.h> 11 12 #include <linux/kcov.h> 13 14 #include <asm/switch_to.h> 15 #include <asm/tlb.h> 16 17 #include "../workqueue_internal.h" 18 #include "../smpboot.h" 19 20 #include "pelt.h" 21 22 #define CREATE_TRACE_POINTS 23 #include <trace/events/sched.h> 24 25 DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); 26 27 #if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_JUMP_LABEL) 28 /* 29 * Debugging: various feature bits 30 * 31 * If SCHED_DEBUG is disabled, each compilation unit has its own copy of 32 * sysctl_sched_features, defined in sched.h, to allow constants propagation 33 * at compile time and compiler optimization based on features default. 34 */ 35 #define SCHED_FEAT(name, enabled) \ 36 (1UL << __SCHED_FEAT_##name) * enabled | 37 const_debug unsigned int sysctl_sched_features = 38 #include "features.h" 39 0; 40 #undef SCHED_FEAT 41 #endif 42 43 /* 44 * Number of tasks to iterate in a single balance run. 45 * Limited because this is done with IRQs disabled. 46 */ 47 const_debug unsigned int sysctl_sched_nr_migrate = 32; 48 49 /* 50 * period over which we measure -rt task CPU usage in us. 51 * default: 1s 52 */ 53 unsigned int sysctl_sched_rt_period = 1000000; 54 55 __read_mostly int scheduler_running; 56 57 /* 58 * part of the period that we allow rt tasks to run in us. 59 * default: 0.95s 60 */ 61 int sysctl_sched_rt_runtime = 950000; 62 63 /* 64 * __task_rq_lock - lock the rq @p resides on. 65 */ 66 struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf) 67 __acquires(rq->lock) 68 { 69 struct rq *rq; 70 71 lockdep_assert_held(&p->pi_lock); 72 73 for (;;) { 74 rq = task_rq(p); 75 raw_spin_lock(&rq->lock); 76 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) { 77 rq_pin_lock(rq, rf); 78 return rq; 79 } 80 raw_spin_unlock(&rq->lock); 81 82 while (unlikely(task_on_rq_migrating(p))) 83 cpu_relax(); 84 } 85 } 86 87 /* 88 * task_rq_lock - lock p->pi_lock and lock the rq @p resides on. 89 */ 90 struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf) 91 __acquires(p->pi_lock) 92 __acquires(rq->lock) 93 { 94 struct rq *rq; 95 96 for (;;) { 97 raw_spin_lock_irqsave(&p->pi_lock, rf->flags); 98 rq = task_rq(p); 99 raw_spin_lock(&rq->lock); 100 /* 101 * move_queued_task() task_rq_lock() 102 * 103 * ACQUIRE (rq->lock) 104 * [S] ->on_rq = MIGRATING [L] rq = task_rq() 105 * WMB (__set_task_cpu()) ACQUIRE (rq->lock); 106 * [S] ->cpu = new_cpu [L] task_rq() 107 * [L] ->on_rq 108 * RELEASE (rq->lock) 109 * 110 * If we observe the old CPU in task_rq_lock, the acquire of 111 * the old rq->lock will fully serialize against the stores. 112 * 113 * If we observe the new CPU in task_rq_lock, the acquire will 114 * pair with the WMB to ensure we must then also see migrating. 115 */ 116 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) { 117 rq_pin_lock(rq, rf); 118 return rq; 119 } 120 raw_spin_unlock(&rq->lock); 121 raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags); 122 123 while (unlikely(task_on_rq_migrating(p))) 124 cpu_relax(); 125 } 126 } 127 128 /* 129 * RQ-clock updating methods: 130 */ 131 132 static void update_rq_clock_task(struct rq *rq, s64 delta) 133 { 134 /* 135 * In theory, the compile should just see 0 here, and optimize out the call 136 * to sched_rt_avg_update. But I don't trust it... 137 */ 138 s64 __maybe_unused steal = 0, irq_delta = 0; 139 140 #ifdef CONFIG_IRQ_TIME_ACCOUNTING 141 irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time; 142 143 /* 144 * Since irq_time is only updated on {soft,}irq_exit, we might run into 145 * this case when a previous update_rq_clock() happened inside a 146 * {soft,}irq region. 147 * 148 * When this happens, we stop ->clock_task and only update the 149 * prev_irq_time stamp to account for the part that fit, so that a next 150 * update will consume the rest. This ensures ->clock_task is 151 * monotonic. 152 * 153 * It does however cause some slight miss-attribution of {soft,}irq 154 * time, a more accurate solution would be to update the irq_time using 155 * the current rq->clock timestamp, except that would require using 156 * atomic ops. 157 */ 158 if (irq_delta > delta) 159 irq_delta = delta; 160 161 rq->prev_irq_time += irq_delta; 162 delta -= irq_delta; 163 #endif 164 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING 165 if (static_key_false((¶virt_steal_rq_enabled))) { 166 steal = paravirt_steal_clock(cpu_of(rq)); 167 steal -= rq->prev_steal_time_rq; 168 169 if (unlikely(steal > delta)) 170 steal = delta; 171 172 rq->prev_steal_time_rq += steal; 173 delta -= steal; 174 } 175 #endif 176 177 rq->clock_task += delta; 178 179 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ 180 if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY)) 181 update_irq_load_avg(rq, irq_delta + steal); 182 #endif 183 } 184 185 void update_rq_clock(struct rq *rq) 186 { 187 s64 delta; 188 189 lockdep_assert_held(&rq->lock); 190 191 if (rq->clock_update_flags & RQCF_ACT_SKIP) 192 return; 193 194 #ifdef CONFIG_SCHED_DEBUG 195 if (sched_feat(WARN_DOUBLE_CLOCK)) 196 SCHED_WARN_ON(rq->clock_update_flags & RQCF_UPDATED); 197 rq->clock_update_flags |= RQCF_UPDATED; 198 #endif 199 200 delta = sched_clock_cpu(cpu_of(rq)) - rq->clock; 201 if (delta < 0) 202 return; 203 rq->clock += delta; 204 update_rq_clock_task(rq, delta); 205 } 206 207 208 #ifdef CONFIG_SCHED_HRTICK 209 /* 210 * Use HR-timers to deliver accurate preemption points. 211 */ 212 213 static void hrtick_clear(struct rq *rq) 214 { 215 if (hrtimer_active(&rq->hrtick_timer)) 216 hrtimer_cancel(&rq->hrtick_timer); 217 } 218 219 /* 220 * High-resolution timer tick. 221 * Runs from hardirq context with interrupts disabled. 222 */ 223 static enum hrtimer_restart hrtick(struct hrtimer *timer) 224 { 225 struct rq *rq = container_of(timer, struct rq, hrtick_timer); 226 struct rq_flags rf; 227 228 WARN_ON_ONCE(cpu_of(rq) != smp_processor_id()); 229 230 rq_lock(rq, &rf); 231 update_rq_clock(rq); 232 rq->curr->sched_class->task_tick(rq, rq->curr, 1); 233 rq_unlock(rq, &rf); 234 235 return HRTIMER_NORESTART; 236 } 237 238 #ifdef CONFIG_SMP 239 240 static void __hrtick_restart(struct rq *rq) 241 { 242 struct hrtimer *timer = &rq->hrtick_timer; 243 244 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED); 245 } 246 247 /* 248 * called from hardirq (IPI) context 249 */ 250 static void __hrtick_start(void *arg) 251 { 252 struct rq *rq = arg; 253 struct rq_flags rf; 254 255 rq_lock(rq, &rf); 256 __hrtick_restart(rq); 257 rq->hrtick_csd_pending = 0; 258 rq_unlock(rq, &rf); 259 } 260 261 /* 262 * Called to set the hrtick timer state. 263 * 264 * called with rq->lock held and irqs disabled 265 */ 266 void hrtick_start(struct rq *rq, u64 delay) 267 { 268 struct hrtimer *timer = &rq->hrtick_timer; 269 ktime_t time; 270 s64 delta; 271 272 /* 273 * Don't schedule slices shorter than 10000ns, that just 274 * doesn't make sense and can cause timer DoS. 275 */ 276 delta = max_t(s64, delay, 10000LL); 277 time = ktime_add_ns(timer->base->get_time(), delta); 278 279 hrtimer_set_expires(timer, time); 280 281 if (rq == this_rq()) { 282 __hrtick_restart(rq); 283 } else if (!rq->hrtick_csd_pending) { 284 smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd); 285 rq->hrtick_csd_pending = 1; 286 } 287 } 288 289 #else 290 /* 291 * Called to set the hrtick timer state. 292 * 293 * called with rq->lock held and irqs disabled 294 */ 295 void hrtick_start(struct rq *rq, u64 delay) 296 { 297 /* 298 * Don't schedule slices shorter than 10000ns, that just 299 * doesn't make sense. Rely on vruntime for fairness. 300 */ 301 delay = max_t(u64, delay, 10000LL); 302 hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay), 303 HRTIMER_MODE_REL_PINNED); 304 } 305 #endif /* CONFIG_SMP */ 306 307 static void hrtick_rq_init(struct rq *rq) 308 { 309 #ifdef CONFIG_SMP 310 rq->hrtick_csd_pending = 0; 311 312 rq->hrtick_csd.flags = 0; 313 rq->hrtick_csd.func = __hrtick_start; 314 rq->hrtick_csd.info = rq; 315 #endif 316 317 hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 318 rq->hrtick_timer.function = hrtick; 319 } 320 #else /* CONFIG_SCHED_HRTICK */ 321 static inline void hrtick_clear(struct rq *rq) 322 { 323 } 324 325 static inline void hrtick_rq_init(struct rq *rq) 326 { 327 } 328 #endif /* CONFIG_SCHED_HRTICK */ 329 330 /* 331 * cmpxchg based fetch_or, macro so it works for different integer types 332 */ 333 #define fetch_or(ptr, mask) \ 334 ({ \ 335 typeof(ptr) _ptr = (ptr); \ 336 typeof(mask) _mask = (mask); \ 337 typeof(*_ptr) _old, _val = *_ptr; \ 338 \ 339 for (;;) { \ 340 _old = cmpxchg(_ptr, _val, _val | _mask); \ 341 if (_old == _val) \ 342 break; \ 343 _val = _old; \ 344 } \ 345 _old; \ 346 }) 347 348 #if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG) 349 /* 350 * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG, 351 * this avoids any races wrt polling state changes and thereby avoids 352 * spurious IPIs. 353 */ 354 static bool set_nr_and_not_polling(struct task_struct *p) 355 { 356 struct thread_info *ti = task_thread_info(p); 357 return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG); 358 } 359 360 /* 361 * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set. 362 * 363 * If this returns true, then the idle task promises to call 364 * sched_ttwu_pending() and reschedule soon. 365 */ 366 static bool set_nr_if_polling(struct task_struct *p) 367 { 368 struct thread_info *ti = task_thread_info(p); 369 typeof(ti->flags) old, val = READ_ONCE(ti->flags); 370 371 for (;;) { 372 if (!(val & _TIF_POLLING_NRFLAG)) 373 return false; 374 if (val & _TIF_NEED_RESCHED) 375 return true; 376 old = cmpxchg(&ti->flags, val, val | _TIF_NEED_RESCHED); 377 if (old == val) 378 break; 379 val = old; 380 } 381 return true; 382 } 383 384 #else 385 static bool set_nr_and_not_polling(struct task_struct *p) 386 { 387 set_tsk_need_resched(p); 388 return true; 389 } 390 391 #ifdef CONFIG_SMP 392 static bool set_nr_if_polling(struct task_struct *p) 393 { 394 return false; 395 } 396 #endif 397 #endif 398 399 /** 400 * wake_q_add() - queue a wakeup for 'later' waking. 401 * @head: the wake_q_head to add @task to 402 * @task: the task to queue for 'later' wakeup 403 * 404 * Queue a task for later wakeup, most likely by the wake_up_q() call in the 405 * same context, _HOWEVER_ this is not guaranteed, the wakeup can come 406 * instantly. 407 * 408 * This function must be used as-if it were wake_up_process(); IOW the task 409 * must be ready to be woken at this location. 410 */ 411 void wake_q_add(struct wake_q_head *head, struct task_struct *task) 412 { 413 struct wake_q_node *node = &task->wake_q; 414 415 /* 416 * Atomically grab the task, if ->wake_q is !nil already it means 417 * its already queued (either by us or someone else) and will get the 418 * wakeup due to that. 419 * 420 * In order to ensure that a pending wakeup will observe our pending 421 * state, even in the failed case, an explicit smp_mb() must be used. 422 */ 423 smp_mb__before_atomic(); 424 if (cmpxchg_relaxed(&node->next, NULL, WAKE_Q_TAIL)) 425 return; 426 427 get_task_struct(task); 428 429 /* 430 * The head is context local, there can be no concurrency. 431 */ 432 *head->lastp = node; 433 head->lastp = &node->next; 434 } 435 436 void wake_up_q(struct wake_q_head *head) 437 { 438 struct wake_q_node *node = head->first; 439 440 while (node != WAKE_Q_TAIL) { 441 struct task_struct *task; 442 443 task = container_of(node, struct task_struct, wake_q); 444 BUG_ON(!task); 445 /* Task can safely be re-inserted now: */ 446 node = node->next; 447 task->wake_q.next = NULL; 448 449 /* 450 * wake_up_process() executes a full barrier, which pairs with 451 * the queueing in wake_q_add() so as not to miss wakeups. 452 */ 453 wake_up_process(task); 454 put_task_struct(task); 455 } 456 } 457 458 /* 459 * resched_curr - mark rq's current task 'to be rescheduled now'. 460 * 461 * On UP this means the setting of the need_resched flag, on SMP it 462 * might also involve a cross-CPU call to trigger the scheduler on 463 * the target CPU. 464 */ 465 void resched_curr(struct rq *rq) 466 { 467 struct task_struct *curr = rq->curr; 468 int cpu; 469 470 lockdep_assert_held(&rq->lock); 471 472 if (test_tsk_need_resched(curr)) 473 return; 474 475 cpu = cpu_of(rq); 476 477 if (cpu == smp_processor_id()) { 478 set_tsk_need_resched(curr); 479 set_preempt_need_resched(); 480 return; 481 } 482 483 if (set_nr_and_not_polling(curr)) 484 smp_send_reschedule(cpu); 485 else 486 trace_sched_wake_idle_without_ipi(cpu); 487 } 488 489 void resched_cpu(int cpu) 490 { 491 struct rq *rq = cpu_rq(cpu); 492 unsigned long flags; 493 494 raw_spin_lock_irqsave(&rq->lock, flags); 495 if (cpu_online(cpu) || cpu == smp_processor_id()) 496 resched_curr(rq); 497 raw_spin_unlock_irqrestore(&rq->lock, flags); 498 } 499 500 #ifdef CONFIG_SMP 501 #ifdef CONFIG_NO_HZ_COMMON 502 /* 503 * In the semi idle case, use the nearest busy CPU for migrating timers 504 * from an idle CPU. This is good for power-savings. 505 * 506 * We don't do similar optimization for completely idle system, as 507 * selecting an idle CPU will add more delays to the timers than intended 508 * (as that CPU's timer base may not be uptodate wrt jiffies etc). 509 */ 510 int get_nohz_timer_target(void) 511 { 512 int i, cpu = smp_processor_id(); 513 struct sched_domain *sd; 514 515 if (!idle_cpu(cpu) && housekeeping_cpu(cpu, HK_FLAG_TIMER)) 516 return cpu; 517 518 rcu_read_lock(); 519 for_each_domain(cpu, sd) { 520 for_each_cpu(i, sched_domain_span(sd)) { 521 if (cpu == i) 522 continue; 523 524 if (!idle_cpu(i) && housekeeping_cpu(i, HK_FLAG_TIMER)) { 525 cpu = i; 526 goto unlock; 527 } 528 } 529 } 530 531 if (!housekeeping_cpu(cpu, HK_FLAG_TIMER)) 532 cpu = housekeeping_any_cpu(HK_FLAG_TIMER); 533 unlock: 534 rcu_read_unlock(); 535 return cpu; 536 } 537 538 /* 539 * When add_timer_on() enqueues a timer into the timer wheel of an 540 * idle CPU then this timer might expire before the next timer event 541 * which is scheduled to wake up that CPU. In case of a completely 542 * idle system the next event might even be infinite time into the 543 * future. wake_up_idle_cpu() ensures that the CPU is woken up and 544 * leaves the inner idle loop so the newly added timer is taken into 545 * account when the CPU goes back to idle and evaluates the timer 546 * wheel for the next timer event. 547 */ 548 static void wake_up_idle_cpu(int cpu) 549 { 550 struct rq *rq = cpu_rq(cpu); 551 552 if (cpu == smp_processor_id()) 553 return; 554 555 if (set_nr_and_not_polling(rq->idle)) 556 smp_send_reschedule(cpu); 557 else 558 trace_sched_wake_idle_without_ipi(cpu); 559 } 560 561 static bool wake_up_full_nohz_cpu(int cpu) 562 { 563 /* 564 * We just need the target to call irq_exit() and re-evaluate 565 * the next tick. The nohz full kick at least implies that. 566 * If needed we can still optimize that later with an 567 * empty IRQ. 568 */ 569 if (cpu_is_offline(cpu)) 570 return true; /* Don't try to wake offline CPUs. */ 571 if (tick_nohz_full_cpu(cpu)) { 572 if (cpu != smp_processor_id() || 573 tick_nohz_tick_stopped()) 574 tick_nohz_full_kick_cpu(cpu); 575 return true; 576 } 577 578 return false; 579 } 580 581 /* 582 * Wake up the specified CPU. If the CPU is going offline, it is the 583 * caller's responsibility to deal with the lost wakeup, for example, 584 * by hooking into the CPU_DEAD notifier like timers and hrtimers do. 585 */ 586 void wake_up_nohz_cpu(int cpu) 587 { 588 if (!wake_up_full_nohz_cpu(cpu)) 589 wake_up_idle_cpu(cpu); 590 } 591 592 static inline bool got_nohz_idle_kick(void) 593 { 594 int cpu = smp_processor_id(); 595 596 if (!(atomic_read(nohz_flags(cpu)) & NOHZ_KICK_MASK)) 597 return false; 598 599 if (idle_cpu(cpu) && !need_resched()) 600 return true; 601 602 /* 603 * We can't run Idle Load Balance on this CPU for this time so we 604 * cancel it and clear NOHZ_BALANCE_KICK 605 */ 606 atomic_andnot(NOHZ_KICK_MASK, nohz_flags(cpu)); 607 return false; 608 } 609 610 #else /* CONFIG_NO_HZ_COMMON */ 611 612 static inline bool got_nohz_idle_kick(void) 613 { 614 return false; 615 } 616 617 #endif /* CONFIG_NO_HZ_COMMON */ 618 619 #ifdef CONFIG_NO_HZ_FULL 620 bool sched_can_stop_tick(struct rq *rq) 621 { 622 int fifo_nr_running; 623 624 /* Deadline tasks, even if single, need the tick */ 625 if (rq->dl.dl_nr_running) 626 return false; 627 628 /* 629 * If there are more than one RR tasks, we need the tick to effect the 630 * actual RR behaviour. 631 */ 632 if (rq->rt.rr_nr_running) { 633 if (rq->rt.rr_nr_running == 1) 634 return true; 635 else 636 return false; 637 } 638 639 /* 640 * If there's no RR tasks, but FIFO tasks, we can skip the tick, no 641 * forced preemption between FIFO tasks. 642 */ 643 fifo_nr_running = rq->rt.rt_nr_running - rq->rt.rr_nr_running; 644 if (fifo_nr_running) 645 return true; 646 647 /* 648 * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left; 649 * if there's more than one we need the tick for involuntary 650 * preemption. 651 */ 652 if (rq->nr_running > 1) 653 return false; 654 655 return true; 656 } 657 #endif /* CONFIG_NO_HZ_FULL */ 658 #endif /* CONFIG_SMP */ 659 660 #if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \ 661 (defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH))) 662 /* 663 * Iterate task_group tree rooted at *from, calling @down when first entering a 664 * node and @up when leaving it for the final time. 665 * 666 * Caller must hold rcu_lock or sufficient equivalent. 667 */ 668 int walk_tg_tree_from(struct task_group *from, 669 tg_visitor down, tg_visitor up, void *data) 670 { 671 struct task_group *parent, *child; 672 int ret; 673 674 parent = from; 675 676 down: 677 ret = (*down)(parent, data); 678 if (ret) 679 goto out; 680 list_for_each_entry_rcu(child, &parent->children, siblings) { 681 parent = child; 682 goto down; 683 684 up: 685 continue; 686 } 687 ret = (*up)(parent, data); 688 if (ret || parent == from) 689 goto out; 690 691 child = parent; 692 parent = parent->parent; 693 if (parent) 694 goto up; 695 out: 696 return ret; 697 } 698 699 int tg_nop(struct task_group *tg, void *data) 700 { 701 return 0; 702 } 703 #endif 704 705 static void set_load_weight(struct task_struct *p, bool update_load) 706 { 707 int prio = p->static_prio - MAX_RT_PRIO; 708 struct load_weight *load = &p->se.load; 709 710 /* 711 * SCHED_IDLE tasks get minimal weight: 712 */ 713 if (task_has_idle_policy(p)) { 714 load->weight = scale_load(WEIGHT_IDLEPRIO); 715 load->inv_weight = WMULT_IDLEPRIO; 716 p->se.runnable_weight = load->weight; 717 return; 718 } 719 720 /* 721 * SCHED_OTHER tasks have to update their load when changing their 722 * weight 723 */ 724 if (update_load && p->sched_class == &fair_sched_class) { 725 reweight_task(p, prio); 726 } else { 727 load->weight = scale_load(sched_prio_to_weight[prio]); 728 load->inv_weight = sched_prio_to_wmult[prio]; 729 p->se.runnable_weight = load->weight; 730 } 731 } 732 733 static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags) 734 { 735 if (!(flags & ENQUEUE_NOCLOCK)) 736 update_rq_clock(rq); 737 738 if (!(flags & ENQUEUE_RESTORE)) { 739 sched_info_queued(rq, p); 740 psi_enqueue(p, flags & ENQUEUE_WAKEUP); 741 } 742 743 p->sched_class->enqueue_task(rq, p, flags); 744 } 745 746 static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags) 747 { 748 if (!(flags & DEQUEUE_NOCLOCK)) 749 update_rq_clock(rq); 750 751 if (!(flags & DEQUEUE_SAVE)) { 752 sched_info_dequeued(rq, p); 753 psi_dequeue(p, flags & DEQUEUE_SLEEP); 754 } 755 756 p->sched_class->dequeue_task(rq, p, flags); 757 } 758 759 void activate_task(struct rq *rq, struct task_struct *p, int flags) 760 { 761 if (task_contributes_to_load(p)) 762 rq->nr_uninterruptible--; 763 764 enqueue_task(rq, p, flags); 765 } 766 767 void deactivate_task(struct rq *rq, struct task_struct *p, int flags) 768 { 769 if (task_contributes_to_load(p)) 770 rq->nr_uninterruptible++; 771 772 dequeue_task(rq, p, flags); 773 } 774 775 /* 776 * __normal_prio - return the priority that is based on the static prio 777 */ 778 static inline int __normal_prio(struct task_struct *p) 779 { 780 return p->static_prio; 781 } 782 783 /* 784 * Calculate the expected normal priority: i.e. priority 785 * without taking RT-inheritance into account. Might be 786 * boosted by interactivity modifiers. Changes upon fork, 787 * setprio syscalls, and whenever the interactivity 788 * estimator recalculates. 789 */ 790 static inline int normal_prio(struct task_struct *p) 791 { 792 int prio; 793 794 if (task_has_dl_policy(p)) 795 prio = MAX_DL_PRIO-1; 796 else if (task_has_rt_policy(p)) 797 prio = MAX_RT_PRIO-1 - p->rt_priority; 798 else 799 prio = __normal_prio(p); 800 return prio; 801 } 802 803 /* 804 * Calculate the current priority, i.e. the priority 805 * taken into account by the scheduler. This value might 806 * be boosted by RT tasks, or might be boosted by 807 * interactivity modifiers. Will be RT if the task got 808 * RT-boosted. If not then it returns p->normal_prio. 809 */ 810 static int effective_prio(struct task_struct *p) 811 { 812 p->normal_prio = normal_prio(p); 813 /* 814 * If we are RT tasks or we were boosted to RT priority, 815 * keep the priority unchanged. Otherwise, update priority 816 * to the normal priority: 817 */ 818 if (!rt_prio(p->prio)) 819 return p->normal_prio; 820 return p->prio; 821 } 822 823 /** 824 * task_curr - is this task currently executing on a CPU? 825 * @p: the task in question. 826 * 827 * Return: 1 if the task is currently executing. 0 otherwise. 828 */ 829 inline int task_curr(const struct task_struct *p) 830 { 831 return cpu_curr(task_cpu(p)) == p; 832 } 833 834 /* 835 * switched_from, switched_to and prio_changed must _NOT_ drop rq->lock, 836 * use the balance_callback list if you want balancing. 837 * 838 * this means any call to check_class_changed() must be followed by a call to 839 * balance_callback(). 840 */ 841 static inline void check_class_changed(struct rq *rq, struct task_struct *p, 842 const struct sched_class *prev_class, 843 int oldprio) 844 { 845 if (prev_class != p->sched_class) { 846 if (prev_class->switched_from) 847 prev_class->switched_from(rq, p); 848 849 p->sched_class->switched_to(rq, p); 850 } else if (oldprio != p->prio || dl_task(p)) 851 p->sched_class->prio_changed(rq, p, oldprio); 852 } 853 854 void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags) 855 { 856 const struct sched_class *class; 857 858 if (p->sched_class == rq->curr->sched_class) { 859 rq->curr->sched_class->check_preempt_curr(rq, p, flags); 860 } else { 861 for_each_class(class) { 862 if (class == rq->curr->sched_class) 863 break; 864 if (class == p->sched_class) { 865 resched_curr(rq); 866 break; 867 } 868 } 869 } 870 871 /* 872 * A queue event has occurred, and we're going to schedule. In 873 * this case, we can save a useless back to back clock update. 874 */ 875 if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr)) 876 rq_clock_skip_update(rq); 877 } 878 879 #ifdef CONFIG_SMP 880 881 static inline bool is_per_cpu_kthread(struct task_struct *p) 882 { 883 if (!(p->flags & PF_KTHREAD)) 884 return false; 885 886 if (p->nr_cpus_allowed != 1) 887 return false; 888 889 return true; 890 } 891 892 /* 893 * Per-CPU kthreads are allowed to run on !actie && online CPUs, see 894 * __set_cpus_allowed_ptr() and select_fallback_rq(). 895 */ 896 static inline bool is_cpu_allowed(struct task_struct *p, int cpu) 897 { 898 if (!cpumask_test_cpu(cpu, &p->cpus_allowed)) 899 return false; 900 901 if (is_per_cpu_kthread(p)) 902 return cpu_online(cpu); 903 904 return cpu_active(cpu); 905 } 906 907 /* 908 * This is how migration works: 909 * 910 * 1) we invoke migration_cpu_stop() on the target CPU using 911 * stop_one_cpu(). 912 * 2) stopper starts to run (implicitly forcing the migrated thread 913 * off the CPU) 914 * 3) it checks whether the migrated task is still in the wrong runqueue. 915 * 4) if it's in the wrong runqueue then the migration thread removes 916 * it and puts it into the right queue. 917 * 5) stopper completes and stop_one_cpu() returns and the migration 918 * is done. 919 */ 920 921 /* 922 * move_queued_task - move a queued task to new rq. 923 * 924 * Returns (locked) new rq. Old rq's lock is released. 925 */ 926 static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf, 927 struct task_struct *p, int new_cpu) 928 { 929 lockdep_assert_held(&rq->lock); 930 931 p->on_rq = TASK_ON_RQ_MIGRATING; 932 dequeue_task(rq, p, DEQUEUE_NOCLOCK); 933 set_task_cpu(p, new_cpu); 934 rq_unlock(rq, rf); 935 936 rq = cpu_rq(new_cpu); 937 938 rq_lock(rq, rf); 939 BUG_ON(task_cpu(p) != new_cpu); 940 enqueue_task(rq, p, 0); 941 p->on_rq = TASK_ON_RQ_QUEUED; 942 check_preempt_curr(rq, p, 0); 943 944 return rq; 945 } 946 947 struct migration_arg { 948 struct task_struct *task; 949 int dest_cpu; 950 }; 951 952 /* 953 * Move (not current) task off this CPU, onto the destination CPU. We're doing 954 * this because either it can't run here any more (set_cpus_allowed() 955 * away from this CPU, or CPU going down), or because we're 956 * attempting to rebalance this task on exec (sched_exec). 957 * 958 * So we race with normal scheduler movements, but that's OK, as long 959 * as the task is no longer on this CPU. 960 */ 961 static struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf, 962 struct task_struct *p, int dest_cpu) 963 { 964 /* Affinity changed (again). */ 965 if (!is_cpu_allowed(p, dest_cpu)) 966 return rq; 967 968 update_rq_clock(rq); 969 rq = move_queued_task(rq, rf, p, dest_cpu); 970 971 return rq; 972 } 973 974 /* 975 * migration_cpu_stop - this will be executed by a highprio stopper thread 976 * and performs thread migration by bumping thread off CPU then 977 * 'pushing' onto another runqueue. 978 */ 979 static int migration_cpu_stop(void *data) 980 { 981 struct migration_arg *arg = data; 982 struct task_struct *p = arg->task; 983 struct rq *rq = this_rq(); 984 struct rq_flags rf; 985 986 /* 987 * The original target CPU might have gone down and we might 988 * be on another CPU but it doesn't matter. 989 */ 990 local_irq_disable(); 991 /* 992 * We need to explicitly wake pending tasks before running 993 * __migrate_task() such that we will not miss enforcing cpus_allowed 994 * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test. 995 */ 996 sched_ttwu_pending(); 997 998 raw_spin_lock(&p->pi_lock); 999 rq_lock(rq, &rf); 1000 /* 1001 * If task_rq(p) != rq, it cannot be migrated here, because we're 1002 * holding rq->lock, if p->on_rq == 0 it cannot get enqueued because 1003 * we're holding p->pi_lock. 1004 */ 1005 if (task_rq(p) == rq) { 1006 if (task_on_rq_queued(p)) 1007 rq = __migrate_task(rq, &rf, p, arg->dest_cpu); 1008 else 1009 p->wake_cpu = arg->dest_cpu; 1010 } 1011 rq_unlock(rq, &rf); 1012 raw_spin_unlock(&p->pi_lock); 1013 1014 local_irq_enable(); 1015 return 0; 1016 } 1017 1018 /* 1019 * sched_class::set_cpus_allowed must do the below, but is not required to 1020 * actually call this function. 1021 */ 1022 void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask) 1023 { 1024 cpumask_copy(&p->cpus_allowed, new_mask); 1025 p->nr_cpus_allowed = cpumask_weight(new_mask); 1026 } 1027 1028 void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) 1029 { 1030 struct rq *rq = task_rq(p); 1031 bool queued, running; 1032 1033 lockdep_assert_held(&p->pi_lock); 1034 1035 queued = task_on_rq_queued(p); 1036 running = task_current(rq, p); 1037 1038 if (queued) { 1039 /* 1040 * Because __kthread_bind() calls this on blocked tasks without 1041 * holding rq->lock. 1042 */ 1043 lockdep_assert_held(&rq->lock); 1044 dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); 1045 } 1046 if (running) 1047 put_prev_task(rq, p); 1048 1049 p->sched_class->set_cpus_allowed(p, new_mask); 1050 1051 if (queued) 1052 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); 1053 if (running) 1054 set_curr_task(rq, p); 1055 } 1056 1057 /* 1058 * Change a given task's CPU affinity. Migrate the thread to a 1059 * proper CPU and schedule it away if the CPU it's executing on 1060 * is removed from the allowed bitmask. 1061 * 1062 * NOTE: the caller must have a valid reference to the task, the 1063 * task must not exit() & deallocate itself prematurely. The 1064 * call is not atomic; no spinlocks may be held. 1065 */ 1066 static int __set_cpus_allowed_ptr(struct task_struct *p, 1067 const struct cpumask *new_mask, bool check) 1068 { 1069 const struct cpumask *cpu_valid_mask = cpu_active_mask; 1070 unsigned int dest_cpu; 1071 struct rq_flags rf; 1072 struct rq *rq; 1073 int ret = 0; 1074 1075 rq = task_rq_lock(p, &rf); 1076 update_rq_clock(rq); 1077 1078 if (p->flags & PF_KTHREAD) { 1079 /* 1080 * Kernel threads are allowed on online && !active CPUs 1081 */ 1082 cpu_valid_mask = cpu_online_mask; 1083 } 1084 1085 /* 1086 * Must re-check here, to close a race against __kthread_bind(), 1087 * sched_setaffinity() is not guaranteed to observe the flag. 1088 */ 1089 if (check && (p->flags & PF_NO_SETAFFINITY)) { 1090 ret = -EINVAL; 1091 goto out; 1092 } 1093 1094 if (cpumask_equal(&p->cpus_allowed, new_mask)) 1095 goto out; 1096 1097 if (!cpumask_intersects(new_mask, cpu_valid_mask)) { 1098 ret = -EINVAL; 1099 goto out; 1100 } 1101 1102 do_set_cpus_allowed(p, new_mask); 1103 1104 if (p->flags & PF_KTHREAD) { 1105 /* 1106 * For kernel threads that do indeed end up on online && 1107 * !active we want to ensure they are strict per-CPU threads. 1108 */ 1109 WARN_ON(cpumask_intersects(new_mask, cpu_online_mask) && 1110 !cpumask_intersects(new_mask, cpu_active_mask) && 1111 p->nr_cpus_allowed != 1); 1112 } 1113 1114 /* Can the task run on the task's current CPU? If so, we're done */ 1115 if (cpumask_test_cpu(task_cpu(p), new_mask)) 1116 goto out; 1117 1118 dest_cpu = cpumask_any_and(cpu_valid_mask, new_mask); 1119 if (task_running(rq, p) || p->state == TASK_WAKING) { 1120 struct migration_arg arg = { p, dest_cpu }; 1121 /* Need help from migration thread: drop lock and wait. */ 1122 task_rq_unlock(rq, p, &rf); 1123 stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg); 1124 tlb_migrate_finish(p->mm); 1125 return 0; 1126 } else if (task_on_rq_queued(p)) { 1127 /* 1128 * OK, since we're going to drop the lock immediately 1129 * afterwards anyway. 1130 */ 1131 rq = move_queued_task(rq, &rf, p, dest_cpu); 1132 } 1133 out: 1134 task_rq_unlock(rq, p, &rf); 1135 1136 return ret; 1137 } 1138 1139 int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) 1140 { 1141 return __set_cpus_allowed_ptr(p, new_mask, false); 1142 } 1143 EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr); 1144 1145 void set_task_cpu(struct task_struct *p, unsigned int new_cpu) 1146 { 1147 #ifdef CONFIG_SCHED_DEBUG 1148 /* 1149 * We should never call set_task_cpu() on a blocked task, 1150 * ttwu() will sort out the placement. 1151 */ 1152 WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING && 1153 !p->on_rq); 1154 1155 /* 1156 * Migrating fair class task must have p->on_rq = TASK_ON_RQ_MIGRATING, 1157 * because schedstat_wait_{start,end} rebase migrating task's wait_start 1158 * time relying on p->on_rq. 1159 */ 1160 WARN_ON_ONCE(p->state == TASK_RUNNING && 1161 p->sched_class == &fair_sched_class && 1162 (p->on_rq && !task_on_rq_migrating(p))); 1163 1164 #ifdef CONFIG_LOCKDEP 1165 /* 1166 * The caller should hold either p->pi_lock or rq->lock, when changing 1167 * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks. 1168 * 1169 * sched_move_task() holds both and thus holding either pins the cgroup, 1170 * see task_group(). 1171 * 1172 * Furthermore, all task_rq users should acquire both locks, see 1173 * task_rq_lock(). 1174 */ 1175 WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) || 1176 lockdep_is_held(&task_rq(p)->lock))); 1177 #endif 1178 /* 1179 * Clearly, migrating tasks to offline CPUs is a fairly daft thing. 1180 */ 1181 WARN_ON_ONCE(!cpu_online(new_cpu)); 1182 #endif 1183 1184 trace_sched_migrate_task(p, new_cpu); 1185 1186 if (task_cpu(p) != new_cpu) { 1187 if (p->sched_class->migrate_task_rq) 1188 p->sched_class->migrate_task_rq(p, new_cpu); 1189 p->se.nr_migrations++; 1190 rseq_migrate(p); 1191 perf_event_task_migrate(p); 1192 } 1193 1194 __set_task_cpu(p, new_cpu); 1195 } 1196 1197 #ifdef CONFIG_NUMA_BALANCING 1198 static void __migrate_swap_task(struct task_struct *p, int cpu) 1199 { 1200 if (task_on_rq_queued(p)) { 1201 struct rq *src_rq, *dst_rq; 1202 struct rq_flags srf, drf; 1203 1204 src_rq = task_rq(p); 1205 dst_rq = cpu_rq(cpu); 1206 1207 rq_pin_lock(src_rq, &srf); 1208 rq_pin_lock(dst_rq, &drf); 1209 1210 p->on_rq = TASK_ON_RQ_MIGRATING; 1211 deactivate_task(src_rq, p, 0); 1212 set_task_cpu(p, cpu); 1213 activate_task(dst_rq, p, 0); 1214 p->on_rq = TASK_ON_RQ_QUEUED; 1215 check_preempt_curr(dst_rq, p, 0); 1216 1217 rq_unpin_lock(dst_rq, &drf); 1218 rq_unpin_lock(src_rq, &srf); 1219 1220 } else { 1221 /* 1222 * Task isn't running anymore; make it appear like we migrated 1223 * it before it went to sleep. This means on wakeup we make the 1224 * previous CPU our target instead of where it really is. 1225 */ 1226 p->wake_cpu = cpu; 1227 } 1228 } 1229 1230 struct migration_swap_arg { 1231 struct task_struct *src_task, *dst_task; 1232 int src_cpu, dst_cpu; 1233 }; 1234 1235 static int migrate_swap_stop(void *data) 1236 { 1237 struct migration_swap_arg *arg = data; 1238 struct rq *src_rq, *dst_rq; 1239 int ret = -EAGAIN; 1240 1241 if (!cpu_active(arg->src_cpu) || !cpu_active(arg->dst_cpu)) 1242 return -EAGAIN; 1243 1244 src_rq = cpu_rq(arg->src_cpu); 1245 dst_rq = cpu_rq(arg->dst_cpu); 1246 1247 double_raw_lock(&arg->src_task->pi_lock, 1248 &arg->dst_task->pi_lock); 1249 double_rq_lock(src_rq, dst_rq); 1250 1251 if (task_cpu(arg->dst_task) != arg->dst_cpu) 1252 goto unlock; 1253 1254 if (task_cpu(arg->src_task) != arg->src_cpu) 1255 goto unlock; 1256 1257 if (!cpumask_test_cpu(arg->dst_cpu, &arg->src_task->cpus_allowed)) 1258 goto unlock; 1259 1260 if (!cpumask_test_cpu(arg->src_cpu, &arg->dst_task->cpus_allowed)) 1261 goto unlock; 1262 1263 __migrate_swap_task(arg->src_task, arg->dst_cpu); 1264 __migrate_swap_task(arg->dst_task, arg->src_cpu); 1265 1266 ret = 0; 1267 1268 unlock: 1269 double_rq_unlock(src_rq, dst_rq); 1270 raw_spin_unlock(&arg->dst_task->pi_lock); 1271 raw_spin_unlock(&arg->src_task->pi_lock); 1272 1273 return ret; 1274 } 1275 1276 /* 1277 * Cross migrate two tasks 1278 */ 1279 int migrate_swap(struct task_struct *cur, struct task_struct *p, 1280 int target_cpu, int curr_cpu) 1281 { 1282 struct migration_swap_arg arg; 1283 int ret = -EINVAL; 1284 1285 arg = (struct migration_swap_arg){ 1286 .src_task = cur, 1287 .src_cpu = curr_cpu, 1288 .dst_task = p, 1289 .dst_cpu = target_cpu, 1290 }; 1291 1292 if (arg.src_cpu == arg.dst_cpu) 1293 goto out; 1294 1295 /* 1296 * These three tests are all lockless; this is OK since all of them 1297 * will be re-checked with proper locks held further down the line. 1298 */ 1299 if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu)) 1300 goto out; 1301 1302 if (!cpumask_test_cpu(arg.dst_cpu, &arg.src_task->cpus_allowed)) 1303 goto out; 1304 1305 if (!cpumask_test_cpu(arg.src_cpu, &arg.dst_task->cpus_allowed)) 1306 goto out; 1307 1308 trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu); 1309 ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg); 1310 1311 out: 1312 return ret; 1313 } 1314 #endif /* CONFIG_NUMA_BALANCING */ 1315 1316 /* 1317 * wait_task_inactive - wait for a thread to unschedule. 1318 * 1319 * If @match_state is nonzero, it's the @p->state value just checked and 1320 * not expected to change. If it changes, i.e. @p might have woken up, 1321 * then return zero. When we succeed in waiting for @p to be off its CPU, 1322 * we return a positive number (its total switch count). If a second call 1323 * a short while later returns the same number, the caller can be sure that 1324 * @p has remained unscheduled the whole time. 1325 * 1326 * The caller must ensure that the task *will* unschedule sometime soon, 1327 * else this function might spin for a *long* time. This function can't 1328 * be called with interrupts off, or it may introduce deadlock with 1329 * smp_call_function() if an IPI is sent by the same process we are 1330 * waiting to become inactive. 1331 */ 1332 unsigned long wait_task_inactive(struct task_struct *p, long match_state) 1333 { 1334 int running, queued; 1335 struct rq_flags rf; 1336 unsigned long ncsw; 1337 struct rq *rq; 1338 1339 for (;;) { 1340 /* 1341 * We do the initial early heuristics without holding 1342 * any task-queue locks at all. We'll only try to get 1343 * the runqueue lock when things look like they will 1344 * work out! 1345 */ 1346 rq = task_rq(p); 1347 1348 /* 1349 * If the task is actively running on another CPU 1350 * still, just relax and busy-wait without holding 1351 * any locks. 1352 * 1353 * NOTE! Since we don't hold any locks, it's not 1354 * even sure that "rq" stays as the right runqueue! 1355 * But we don't care, since "task_running()" will 1356 * return false if the runqueue has changed and p 1357 * is actually now running somewhere else! 1358 */ 1359 while (task_running(rq, p)) { 1360 if (match_state && unlikely(p->state != match_state)) 1361 return 0; 1362 cpu_relax(); 1363 } 1364 1365 /* 1366 * Ok, time to look more closely! We need the rq 1367 * lock now, to be *sure*. If we're wrong, we'll 1368 * just go back and repeat. 1369 */ 1370 rq = task_rq_lock(p, &rf); 1371 trace_sched_wait_task(p); 1372 running = task_running(rq, p); 1373 queued = task_on_rq_queued(p); 1374 ncsw = 0; 1375 if (!match_state || p->state == match_state) 1376 ncsw = p->nvcsw | LONG_MIN; /* sets MSB */ 1377 task_rq_unlock(rq, p, &rf); 1378 1379 /* 1380 * If it changed from the expected state, bail out now. 1381 */ 1382 if (unlikely(!ncsw)) 1383 break; 1384 1385 /* 1386 * Was it really running after all now that we 1387 * checked with the proper locks actually held? 1388 * 1389 * Oops. Go back and try again.. 1390 */ 1391 if (unlikely(running)) { 1392 cpu_relax(); 1393 continue; 1394 } 1395 1396 /* 1397 * It's not enough that it's not actively running, 1398 * it must be off the runqueue _entirely_, and not 1399 * preempted! 1400 * 1401 * So if it was still runnable (but just not actively 1402 * running right now), it's preempted, and we should 1403 * yield - it could be a while. 1404 */ 1405 if (unlikely(queued)) { 1406 ktime_t to = NSEC_PER_SEC / HZ; 1407 1408 set_current_state(TASK_UNINTERRUPTIBLE); 1409 schedule_hrtimeout(&to, HRTIMER_MODE_REL); 1410 continue; 1411 } 1412 1413 /* 1414 * Ahh, all good. It wasn't running, and it wasn't 1415 * runnable, which means that it will never become 1416 * running in the future either. We're all done! 1417 */ 1418 break; 1419 } 1420 1421 return ncsw; 1422 } 1423 1424 /*** 1425 * kick_process - kick a running thread to enter/exit the kernel 1426 * @p: the to-be-kicked thread 1427 * 1428 * Cause a process which is running on another CPU to enter 1429 * kernel-mode, without any delay. (to get signals handled.) 1430 * 1431 * NOTE: this function doesn't have to take the runqueue lock, 1432 * because all it wants to ensure is that the remote task enters 1433 * the kernel. If the IPI races and the task has been migrated 1434 * to another CPU then no harm is done and the purpose has been 1435 * achieved as well. 1436 */ 1437 void kick_process(struct task_struct *p) 1438 { 1439 int cpu; 1440 1441 preempt_disable(); 1442 cpu = task_cpu(p); 1443 if ((cpu != smp_processor_id()) && task_curr(p)) 1444 smp_send_reschedule(cpu); 1445 preempt_enable(); 1446 } 1447 EXPORT_SYMBOL_GPL(kick_process); 1448 1449 /* 1450 * ->cpus_allowed is protected by both rq->lock and p->pi_lock 1451 * 1452 * A few notes on cpu_active vs cpu_online: 1453 * 1454 * - cpu_active must be a subset of cpu_online 1455 * 1456 * - on CPU-up we allow per-CPU kthreads on the online && !active CPU, 1457 * see __set_cpus_allowed_ptr(). At this point the newly online 1458 * CPU isn't yet part of the sched domains, and balancing will not 1459 * see it. 1460 * 1461 * - on CPU-down we clear cpu_active() to mask the sched domains and 1462 * avoid the load balancer to place new tasks on the to be removed 1463 * CPU. Existing tasks will remain running there and will be taken 1464 * off. 1465 * 1466 * This means that fallback selection must not select !active CPUs. 1467 * And can assume that any active CPU must be online. Conversely 1468 * select_task_rq() below may allow selection of !active CPUs in order 1469 * to satisfy the above rules. 1470 */ 1471 static int select_fallback_rq(int cpu, struct task_struct *p) 1472 { 1473 int nid = cpu_to_node(cpu); 1474 const struct cpumask *nodemask = NULL; 1475 enum { cpuset, possible, fail } state = cpuset; 1476 int dest_cpu; 1477 1478 /* 1479 * If the node that the CPU is on has been offlined, cpu_to_node() 1480 * will return -1. There is no CPU on the node, and we should 1481 * select the CPU on the other node. 1482 */ 1483 if (nid != -1) { 1484 nodemask = cpumask_of_node(nid); 1485 1486 /* Look for allowed, online CPU in same node. */ 1487 for_each_cpu(dest_cpu, nodemask) { 1488 if (!cpu_active(dest_cpu)) 1489 continue; 1490 if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed)) 1491 return dest_cpu; 1492 } 1493 } 1494 1495 for (;;) { 1496 /* Any allowed, online CPU? */ 1497 for_each_cpu(dest_cpu, &p->cpus_allowed) { 1498 if (!is_cpu_allowed(p, dest_cpu)) 1499 continue; 1500 1501 goto out; 1502 } 1503 1504 /* No more Mr. Nice Guy. */ 1505 switch (state) { 1506 case cpuset: 1507 if (IS_ENABLED(CONFIG_CPUSETS)) { 1508 cpuset_cpus_allowed_fallback(p); 1509 state = possible; 1510 break; 1511 } 1512 /* Fall-through */ 1513 case possible: 1514 do_set_cpus_allowed(p, cpu_possible_mask); 1515 state = fail; 1516 break; 1517 1518 case fail: 1519 BUG(); 1520 break; 1521 } 1522 } 1523 1524 out: 1525 if (state != cpuset) { 1526 /* 1527 * Don't tell them about moving exiting tasks or 1528 * kernel threads (both mm NULL), since they never 1529 * leave kernel. 1530 */ 1531 if (p->mm && printk_ratelimit()) { 1532 printk_deferred("process %d (%s) no longer affine to cpu%d\n", 1533 task_pid_nr(p), p->comm, cpu); 1534 } 1535 } 1536 1537 return dest_cpu; 1538 } 1539 1540 /* 1541 * The caller (fork, wakeup) owns p->pi_lock, ->cpus_allowed is stable. 1542 */ 1543 static inline 1544 int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags) 1545 { 1546 lockdep_assert_held(&p->pi_lock); 1547 1548 if (p->nr_cpus_allowed > 1) 1549 cpu = p->sched_class->select_task_rq(p, cpu, sd_flags, wake_flags); 1550 else 1551 cpu = cpumask_any(&p->cpus_allowed); 1552 1553 /* 1554 * In order not to call set_task_cpu() on a blocking task we need 1555 * to rely on ttwu() to place the task on a valid ->cpus_allowed 1556 * CPU. 1557 * 1558 * Since this is common to all placement strategies, this lives here. 1559 * 1560 * [ this allows ->select_task() to simply return task_cpu(p) and 1561 * not worry about this generic constraint ] 1562 */ 1563 if (unlikely(!is_cpu_allowed(p, cpu))) 1564 cpu = select_fallback_rq(task_cpu(p), p); 1565 1566 return cpu; 1567 } 1568 1569 static void update_avg(u64 *avg, u64 sample) 1570 { 1571 s64 diff = sample - *avg; 1572 *avg += diff >> 3; 1573 } 1574 1575 void sched_set_stop_task(int cpu, struct task_struct *stop) 1576 { 1577 struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 }; 1578 struct task_struct *old_stop = cpu_rq(cpu)->stop; 1579 1580 if (stop) { 1581 /* 1582 * Make it appear like a SCHED_FIFO task, its something 1583 * userspace knows about and won't get confused about. 1584 * 1585 * Also, it will make PI more or less work without too 1586 * much confusion -- but then, stop work should not 1587 * rely on PI working anyway. 1588 */ 1589 sched_setscheduler_nocheck(stop, SCHED_FIFO, ¶m); 1590 1591 stop->sched_class = &stop_sched_class; 1592 } 1593 1594 cpu_rq(cpu)->stop = stop; 1595 1596 if (old_stop) { 1597 /* 1598 * Reset it back to a normal scheduling class so that 1599 * it can die in pieces. 1600 */ 1601 old_stop->sched_class = &rt_sched_class; 1602 } 1603 } 1604 1605 #else 1606 1607 static inline int __set_cpus_allowed_ptr(struct task_struct *p, 1608 const struct cpumask *new_mask, bool check) 1609 { 1610 return set_cpus_allowed_ptr(p, new_mask); 1611 } 1612 1613 #endif /* CONFIG_SMP */ 1614 1615 static void 1616 ttwu_stat(struct task_struct *p, int cpu, int wake_flags) 1617 { 1618 struct rq *rq; 1619 1620 if (!schedstat_enabled()) 1621 return; 1622 1623 rq = this_rq(); 1624 1625 #ifdef CONFIG_SMP 1626 if (cpu == rq->cpu) { 1627 __schedstat_inc(rq->ttwu_local); 1628 __schedstat_inc(p->se.statistics.nr_wakeups_local); 1629 } else { 1630 struct sched_domain *sd; 1631 1632 __schedstat_inc(p->se.statistics.nr_wakeups_remote); 1633 rcu_read_lock(); 1634 for_each_domain(rq->cpu, sd) { 1635 if (cpumask_test_cpu(cpu, sched_domain_span(sd))) { 1636 __schedstat_inc(sd->ttwu_wake_remote); 1637 break; 1638 } 1639 } 1640 rcu_read_unlock(); 1641 } 1642 1643 if (wake_flags & WF_MIGRATED) 1644 __schedstat_inc(p->se.statistics.nr_wakeups_migrate); 1645 #endif /* CONFIG_SMP */ 1646 1647 __schedstat_inc(rq->ttwu_count); 1648 __schedstat_inc(p->se.statistics.nr_wakeups); 1649 1650 if (wake_flags & WF_SYNC) 1651 __schedstat_inc(p->se.statistics.nr_wakeups_sync); 1652 } 1653 1654 static inline void ttwu_activate(struct rq *rq, struct task_struct *p, int en_flags) 1655 { 1656 activate_task(rq, p, en_flags); 1657 p->on_rq = TASK_ON_RQ_QUEUED; 1658 1659 /* If a worker is waking up, notify the workqueue: */ 1660 if (p->flags & PF_WQ_WORKER) 1661 wq_worker_waking_up(p, cpu_of(rq)); 1662 } 1663 1664 /* 1665 * Mark the task runnable and perform wakeup-preemption. 1666 */ 1667 static void ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags, 1668 struct rq_flags *rf) 1669 { 1670 check_preempt_curr(rq, p, wake_flags); 1671 p->state = TASK_RUNNING; 1672 trace_sched_wakeup(p); 1673 1674 #ifdef CONFIG_SMP 1675 if (p->sched_class->task_woken) { 1676 /* 1677 * Our task @p is fully woken up and running; so its safe to 1678 * drop the rq->lock, hereafter rq is only used for statistics. 1679 */ 1680 rq_unpin_lock(rq, rf); 1681 p->sched_class->task_woken(rq, p); 1682 rq_repin_lock(rq, rf); 1683 } 1684 1685 if (rq->idle_stamp) { 1686 u64 delta = rq_clock(rq) - rq->idle_stamp; 1687 u64 max = 2*rq->max_idle_balance_cost; 1688 1689 update_avg(&rq->avg_idle, delta); 1690 1691 if (rq->avg_idle > max) 1692 rq->avg_idle = max; 1693 1694 rq->idle_stamp = 0; 1695 } 1696 #endif 1697 } 1698 1699 static void 1700 ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags, 1701 struct rq_flags *rf) 1702 { 1703 int en_flags = ENQUEUE_WAKEUP | ENQUEUE_NOCLOCK; 1704 1705 lockdep_assert_held(&rq->lock); 1706 1707 #ifdef CONFIG_SMP 1708 if (p->sched_contributes_to_load) 1709 rq->nr_uninterruptible--; 1710 1711 if (wake_flags & WF_MIGRATED) 1712 en_flags |= ENQUEUE_MIGRATED; 1713 #endif 1714 1715 ttwu_activate(rq, p, en_flags); 1716 ttwu_do_wakeup(rq, p, wake_flags, rf); 1717 } 1718 1719 /* 1720 * Called in case the task @p isn't fully descheduled from its runqueue, 1721 * in this case we must do a remote wakeup. Its a 'light' wakeup though, 1722 * since all we need to do is flip p->state to TASK_RUNNING, since 1723 * the task is still ->on_rq. 1724 */ 1725 static int ttwu_remote(struct task_struct *p, int wake_flags) 1726 { 1727 struct rq_flags rf; 1728 struct rq *rq; 1729 int ret = 0; 1730 1731 rq = __task_rq_lock(p, &rf); 1732 if (task_on_rq_queued(p)) { 1733 /* check_preempt_curr() may use rq clock */ 1734 update_rq_clock(rq); 1735 ttwu_do_wakeup(rq, p, wake_flags, &rf); 1736 ret = 1; 1737 } 1738 __task_rq_unlock(rq, &rf); 1739 1740 return ret; 1741 } 1742 1743 #ifdef CONFIG_SMP 1744 void sched_ttwu_pending(void) 1745 { 1746 struct rq *rq = this_rq(); 1747 struct llist_node *llist = llist_del_all(&rq->wake_list); 1748 struct task_struct *p, *t; 1749 struct rq_flags rf; 1750 1751 if (!llist) 1752 return; 1753 1754 rq_lock_irqsave(rq, &rf); 1755 update_rq_clock(rq); 1756 1757 llist_for_each_entry_safe(p, t, llist, wake_entry) 1758 ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0, &rf); 1759 1760 rq_unlock_irqrestore(rq, &rf); 1761 } 1762 1763 void scheduler_ipi(void) 1764 { 1765 /* 1766 * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting 1767 * TIF_NEED_RESCHED remotely (for the first time) will also send 1768 * this IPI. 1769 */ 1770 preempt_fold_need_resched(); 1771 1772 if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick()) 1773 return; 1774 1775 /* 1776 * Not all reschedule IPI handlers call irq_enter/irq_exit, since 1777 * traditionally all their work was done from the interrupt return 1778 * path. Now that we actually do some work, we need to make sure 1779 * we do call them. 1780 * 1781 * Some archs already do call them, luckily irq_enter/exit nest 1782 * properly. 1783 * 1784 * Arguably we should visit all archs and update all handlers, 1785 * however a fair share of IPIs are still resched only so this would 1786 * somewhat pessimize the simple resched case. 1787 */ 1788 irq_enter(); 1789 sched_ttwu_pending(); 1790 1791 /* 1792 * Check if someone kicked us for doing the nohz idle load balance. 1793 */ 1794 if (unlikely(got_nohz_idle_kick())) { 1795 this_rq()->idle_balance = 1; 1796 raise_softirq_irqoff(SCHED_SOFTIRQ); 1797 } 1798 irq_exit(); 1799 } 1800 1801 static void ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags) 1802 { 1803 struct rq *rq = cpu_rq(cpu); 1804 1805 p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED); 1806 1807 if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) { 1808 if (!set_nr_if_polling(rq->idle)) 1809 smp_send_reschedule(cpu); 1810 else 1811 trace_sched_wake_idle_without_ipi(cpu); 1812 } 1813 } 1814 1815 void wake_up_if_idle(int cpu) 1816 { 1817 struct rq *rq = cpu_rq(cpu); 1818 struct rq_flags rf; 1819 1820 rcu_read_lock(); 1821 1822 if (!is_idle_task(rcu_dereference(rq->curr))) 1823 goto out; 1824 1825 if (set_nr_if_polling(rq->idle)) { 1826 trace_sched_wake_idle_without_ipi(cpu); 1827 } else { 1828 rq_lock_irqsave(rq, &rf); 1829 if (is_idle_task(rq->curr)) 1830 smp_send_reschedule(cpu); 1831 /* Else CPU is not idle, do nothing here: */ 1832 rq_unlock_irqrestore(rq, &rf); 1833 } 1834 1835 out: 1836 rcu_read_unlock(); 1837 } 1838 1839 bool cpus_share_cache(int this_cpu, int that_cpu) 1840 { 1841 return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu); 1842 } 1843 #endif /* CONFIG_SMP */ 1844 1845 static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags) 1846 { 1847 struct rq *rq = cpu_rq(cpu); 1848 struct rq_flags rf; 1849 1850 #if defined(CONFIG_SMP) 1851 if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) { 1852 sched_clock_cpu(cpu); /* Sync clocks across CPUs */ 1853 ttwu_queue_remote(p, cpu, wake_flags); 1854 return; 1855 } 1856 #endif 1857 1858 rq_lock(rq, &rf); 1859 update_rq_clock(rq); 1860 ttwu_do_activate(rq, p, wake_flags, &rf); 1861 rq_unlock(rq, &rf); 1862 } 1863 1864 /* 1865 * Notes on Program-Order guarantees on SMP systems. 1866 * 1867 * MIGRATION 1868 * 1869 * The basic program-order guarantee on SMP systems is that when a task [t] 1870 * migrates, all its activity on its old CPU [c0] happens-before any subsequent 1871 * execution on its new CPU [c1]. 1872 * 1873 * For migration (of runnable tasks) this is provided by the following means: 1874 * 1875 * A) UNLOCK of the rq(c0)->lock scheduling out task t 1876 * B) migration for t is required to synchronize *both* rq(c0)->lock and 1877 * rq(c1)->lock (if not at the same time, then in that order). 1878 * C) LOCK of the rq(c1)->lock scheduling in task 1879 * 1880 * Release/acquire chaining guarantees that B happens after A and C after B. 1881 * Note: the CPU doing B need not be c0 or c1 1882 * 1883 * Example: 1884 * 1885 * CPU0 CPU1 CPU2 1886 * 1887 * LOCK rq(0)->lock 1888 * sched-out X 1889 * sched-in Y 1890 * UNLOCK rq(0)->lock 1891 * 1892 * LOCK rq(0)->lock // orders against CPU0 1893 * dequeue X 1894 * UNLOCK rq(0)->lock 1895 * 1896 * LOCK rq(1)->lock 1897 * enqueue X 1898 * UNLOCK rq(1)->lock 1899 * 1900 * LOCK rq(1)->lock // orders against CPU2 1901 * sched-out Z 1902 * sched-in X 1903 * UNLOCK rq(1)->lock 1904 * 1905 * 1906 * BLOCKING -- aka. SLEEP + WAKEUP 1907 * 1908 * For blocking we (obviously) need to provide the same guarantee as for 1909 * migration. However the means are completely different as there is no lock 1910 * chain to provide order. Instead we do: 1911 * 1912 * 1) smp_store_release(X->on_cpu, 0) 1913 * 2) smp_cond_load_acquire(!X->on_cpu) 1914 * 1915 * Example: 1916 * 1917 * CPU0 (schedule) CPU1 (try_to_wake_up) CPU2 (schedule) 1918 * 1919 * LOCK rq(0)->lock LOCK X->pi_lock 1920 * dequeue X 1921 * sched-out X 1922 * smp_store_release(X->on_cpu, 0); 1923 * 1924 * smp_cond_load_acquire(&X->on_cpu, !VAL); 1925 * X->state = WAKING 1926 * set_task_cpu(X,2) 1927 * 1928 * LOCK rq(2)->lock 1929 * enqueue X 1930 * X->state = RUNNING 1931 * UNLOCK rq(2)->lock 1932 * 1933 * LOCK rq(2)->lock // orders against CPU1 1934 * sched-out Z 1935 * sched-in X 1936 * UNLOCK rq(2)->lock 1937 * 1938 * UNLOCK X->pi_lock 1939 * UNLOCK rq(0)->lock 1940 * 1941 * 1942 * However, for wakeups there is a second guarantee we must provide, namely we 1943 * must ensure that CONDITION=1 done by the caller can not be reordered with 1944 * accesses to the task state; see try_to_wake_up() and set_current_state(). 1945 */ 1946 1947 /** 1948 * try_to_wake_up - wake up a thread 1949 * @p: the thread to be awakened 1950 * @state: the mask of task states that can be woken 1951 * @wake_flags: wake modifier flags (WF_*) 1952 * 1953 * If (@state & @p->state) @p->state = TASK_RUNNING. 1954 * 1955 * If the task was not queued/runnable, also place it back on a runqueue. 1956 * 1957 * Atomic against schedule() which would dequeue a task, also see 1958 * set_current_state(). 1959 * 1960 * This function executes a full memory barrier before accessing the task 1961 * state; see set_current_state(). 1962 * 1963 * Return: %true if @p->state changes (an actual wakeup was done), 1964 * %false otherwise. 1965 */ 1966 static int 1967 try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) 1968 { 1969 unsigned long flags; 1970 int cpu, success = 0; 1971 1972 /* 1973 * If we are going to wake up a thread waiting for CONDITION we 1974 * need to ensure that CONDITION=1 done by the caller can not be 1975 * reordered with p->state check below. This pairs with mb() in 1976 * set_current_state() the waiting thread does. 1977 */ 1978 raw_spin_lock_irqsave(&p->pi_lock, flags); 1979 smp_mb__after_spinlock(); 1980 if (!(p->state & state)) 1981 goto out; 1982 1983 trace_sched_waking(p); 1984 1985 /* We're going to change ->state: */ 1986 success = 1; 1987 cpu = task_cpu(p); 1988 1989 /* 1990 * Ensure we load p->on_rq _after_ p->state, otherwise it would 1991 * be possible to, falsely, observe p->on_rq == 0 and get stuck 1992 * in smp_cond_load_acquire() below. 1993 * 1994 * sched_ttwu_pending() try_to_wake_up() 1995 * STORE p->on_rq = 1 LOAD p->state 1996 * UNLOCK rq->lock 1997 * 1998 * __schedule() (switch to task 'p') 1999 * LOCK rq->lock smp_rmb(); 2000 * smp_mb__after_spinlock(); 2001 * UNLOCK rq->lock 2002 * 2003 * [task p] 2004 * STORE p->state = UNINTERRUPTIBLE LOAD p->on_rq 2005 * 2006 * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in 2007 * __schedule(). See the comment for smp_mb__after_spinlock(). 2008 */ 2009 smp_rmb(); 2010 if (p->on_rq && ttwu_remote(p, wake_flags)) 2011 goto stat; 2012 2013 #ifdef CONFIG_SMP 2014 /* 2015 * Ensure we load p->on_cpu _after_ p->on_rq, otherwise it would be 2016 * possible to, falsely, observe p->on_cpu == 0. 2017 * 2018 * One must be running (->on_cpu == 1) in order to remove oneself 2019 * from the runqueue. 2020 * 2021 * __schedule() (switch to task 'p') try_to_wake_up() 2022 * STORE p->on_cpu = 1 LOAD p->on_rq 2023 * UNLOCK rq->lock 2024 * 2025 * __schedule() (put 'p' to sleep) 2026 * LOCK rq->lock smp_rmb(); 2027 * smp_mb__after_spinlock(); 2028 * STORE p->on_rq = 0 LOAD p->on_cpu 2029 * 2030 * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in 2031 * __schedule(). See the comment for smp_mb__after_spinlock(). 2032 */ 2033 smp_rmb(); 2034 2035 /* 2036 * If the owning (remote) CPU is still in the middle of schedule() with 2037 * this task as prev, wait until its done referencing the task. 2038 * 2039 * Pairs with the smp_store_release() in finish_task(). 2040 * 2041 * This ensures that tasks getting woken will be fully ordered against 2042 * their previous state and preserve Program Order. 2043 */ 2044 smp_cond_load_acquire(&p->on_cpu, !VAL); 2045 2046 p->sched_contributes_to_load = !!task_contributes_to_load(p); 2047 p->state = TASK_WAKING; 2048 2049 if (p->in_iowait) { 2050 delayacct_blkio_end(p); 2051 atomic_dec(&task_rq(p)->nr_iowait); 2052 } 2053 2054 cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags); 2055 if (task_cpu(p) != cpu) { 2056 wake_flags |= WF_MIGRATED; 2057 psi_ttwu_dequeue(p); 2058 set_task_cpu(p, cpu); 2059 } 2060 2061 #else /* CONFIG_SMP */ 2062 2063 if (p->in_iowait) { 2064 delayacct_blkio_end(p); 2065 atomic_dec(&task_rq(p)->nr_iowait); 2066 } 2067 2068 #endif /* CONFIG_SMP */ 2069 2070 ttwu_queue(p, cpu, wake_flags); 2071 stat: 2072 ttwu_stat(p, cpu, wake_flags); 2073 out: 2074 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 2075 2076 return success; 2077 } 2078 2079 /** 2080 * try_to_wake_up_local - try to wake up a local task with rq lock held 2081 * @p: the thread to be awakened 2082 * @rf: request-queue flags for pinning 2083 * 2084 * Put @p on the run-queue if it's not already there. The caller must 2085 * ensure that this_rq() is locked, @p is bound to this_rq() and not 2086 * the current task. 2087 */ 2088 static void try_to_wake_up_local(struct task_struct *p, struct rq_flags *rf) 2089 { 2090 struct rq *rq = task_rq(p); 2091 2092 if (WARN_ON_ONCE(rq != this_rq()) || 2093 WARN_ON_ONCE(p == current)) 2094 return; 2095 2096 lockdep_assert_held(&rq->lock); 2097 2098 if (!raw_spin_trylock(&p->pi_lock)) { 2099 /* 2100 * This is OK, because current is on_cpu, which avoids it being 2101 * picked for load-balance and preemption/IRQs are still 2102 * disabled avoiding further scheduler activity on it and we've 2103 * not yet picked a replacement task. 2104 */ 2105 rq_unlock(rq, rf); 2106 raw_spin_lock(&p->pi_lock); 2107 rq_relock(rq, rf); 2108 } 2109 2110 if (!(p->state & TASK_NORMAL)) 2111 goto out; 2112 2113 trace_sched_waking(p); 2114 2115 if (!task_on_rq_queued(p)) { 2116 if (p->in_iowait) { 2117 delayacct_blkio_end(p); 2118 atomic_dec(&rq->nr_iowait); 2119 } 2120 ttwu_activate(rq, p, ENQUEUE_WAKEUP | ENQUEUE_NOCLOCK); 2121 } 2122 2123 ttwu_do_wakeup(rq, p, 0, rf); 2124 ttwu_stat(p, smp_processor_id(), 0); 2125 out: 2126 raw_spin_unlock(&p->pi_lock); 2127 } 2128 2129 /** 2130 * wake_up_process - Wake up a specific process 2131 * @p: The process to be woken up. 2132 * 2133 * Attempt to wake up the nominated process and move it to the set of runnable 2134 * processes. 2135 * 2136 * Return: 1 if the process was woken up, 0 if it was already running. 2137 * 2138 * This function executes a full memory barrier before accessing the task state. 2139 */ 2140 int wake_up_process(struct task_struct *p) 2141 { 2142 return try_to_wake_up(p, TASK_NORMAL, 0); 2143 } 2144 EXPORT_SYMBOL(wake_up_process); 2145 2146 int wake_up_state(struct task_struct *p, unsigned int state) 2147 { 2148 return try_to_wake_up(p, state, 0); 2149 } 2150 2151 /* 2152 * Perform scheduler related setup for a newly forked process p. 2153 * p is forked by current. 2154 * 2155 * __sched_fork() is basic setup used by init_idle() too: 2156 */ 2157 static void __sched_fork(unsigned long clone_flags, struct task_struct *p) 2158 { 2159 p->on_rq = 0; 2160 2161 p->se.on_rq = 0; 2162 p->se.exec_start = 0; 2163 p->se.sum_exec_runtime = 0; 2164 p->se.prev_sum_exec_runtime = 0; 2165 p->se.nr_migrations = 0; 2166 p->se.vruntime = 0; 2167 INIT_LIST_HEAD(&p->se.group_node); 2168 2169 #ifdef CONFIG_FAIR_GROUP_SCHED 2170 p->se.cfs_rq = NULL; 2171 #endif 2172 2173 #ifdef CONFIG_SCHEDSTATS 2174 /* Even if schedstat is disabled, there should not be garbage */ 2175 memset(&p->se.statistics, 0, sizeof(p->se.statistics)); 2176 #endif 2177 2178 RB_CLEAR_NODE(&p->dl.rb_node); 2179 init_dl_task_timer(&p->dl); 2180 init_dl_inactive_task_timer(&p->dl); 2181 __dl_clear_params(p); 2182 2183 INIT_LIST_HEAD(&p->rt.run_list); 2184 p->rt.timeout = 0; 2185 p->rt.time_slice = sched_rr_timeslice; 2186 p->rt.on_rq = 0; 2187 p->rt.on_list = 0; 2188 2189 #ifdef CONFIG_PREEMPT_NOTIFIERS 2190 INIT_HLIST_HEAD(&p->preempt_notifiers); 2191 #endif 2192 2193 init_numa_balancing(clone_flags, p); 2194 } 2195 2196 DEFINE_STATIC_KEY_FALSE(sched_numa_balancing); 2197 2198 #ifdef CONFIG_NUMA_BALANCING 2199 2200 void set_numabalancing_state(bool enabled) 2201 { 2202 if (enabled) 2203 static_branch_enable(&sched_numa_balancing); 2204 else 2205 static_branch_disable(&sched_numa_balancing); 2206 } 2207 2208 #ifdef CONFIG_PROC_SYSCTL 2209 int sysctl_numa_balancing(struct ctl_table *table, int write, 2210 void __user *buffer, size_t *lenp, loff_t *ppos) 2211 { 2212 struct ctl_table t; 2213 int err; 2214 int state = static_branch_likely(&sched_numa_balancing); 2215 2216 if (write && !capable(CAP_SYS_ADMIN)) 2217 return -EPERM; 2218 2219 t = *table; 2220 t.data = &state; 2221 err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); 2222 if (err < 0) 2223 return err; 2224 if (write) 2225 set_numabalancing_state(state); 2226 return err; 2227 } 2228 #endif 2229 #endif 2230 2231 #ifdef CONFIG_SCHEDSTATS 2232 2233 DEFINE_STATIC_KEY_FALSE(sched_schedstats); 2234 static bool __initdata __sched_schedstats = false; 2235 2236 static void set_schedstats(bool enabled) 2237 { 2238 if (enabled) 2239 static_branch_enable(&sched_schedstats); 2240 else 2241 static_branch_disable(&sched_schedstats); 2242 } 2243 2244 void force_schedstat_enabled(void) 2245 { 2246 if (!schedstat_enabled()) { 2247 pr_info("kernel profiling enabled schedstats, disable via kernel.sched_schedstats.\n"); 2248 static_branch_enable(&sched_schedstats); 2249 } 2250 } 2251 2252 static int __init setup_schedstats(char *str) 2253 { 2254 int ret = 0; 2255 if (!str) 2256 goto out; 2257 2258 /* 2259 * This code is called before jump labels have been set up, so we can't 2260 * change the static branch directly just yet. Instead set a temporary 2261 * variable so init_schedstats() can do it later. 2262 */ 2263 if (!strcmp(str, "enable")) { 2264 __sched_schedstats = true; 2265 ret = 1; 2266 } else if (!strcmp(str, "disable")) { 2267 __sched_schedstats = false; 2268 ret = 1; 2269 } 2270 out: 2271 if (!ret) 2272 pr_warn("Unable to parse schedstats=\n"); 2273 2274 return ret; 2275 } 2276 __setup("schedstats=", setup_schedstats); 2277 2278 static void __init init_schedstats(void) 2279 { 2280 set_schedstats(__sched_schedstats); 2281 } 2282 2283 #ifdef CONFIG_PROC_SYSCTL 2284 int sysctl_schedstats(struct ctl_table *table, int write, 2285 void __user *buffer, size_t *lenp, loff_t *ppos) 2286 { 2287 struct ctl_table t; 2288 int err; 2289 int state = static_branch_likely(&sched_schedstats); 2290 2291 if (write && !capable(CAP_SYS_ADMIN)) 2292 return -EPERM; 2293 2294 t = *table; 2295 t.data = &state; 2296 err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); 2297 if (err < 0) 2298 return err; 2299 if (write) 2300 set_schedstats(state); 2301 return err; 2302 } 2303 #endif /* CONFIG_PROC_SYSCTL */ 2304 #else /* !CONFIG_SCHEDSTATS */ 2305 static inline void init_schedstats(void) {} 2306 #endif /* CONFIG_SCHEDSTATS */ 2307 2308 /* 2309 * fork()/clone()-time setup: 2310 */ 2311 int sched_fork(unsigned long clone_flags, struct task_struct *p) 2312 { 2313 unsigned long flags; 2314 2315 __sched_fork(clone_flags, p); 2316 /* 2317 * We mark the process as NEW here. This guarantees that 2318 * nobody will actually run it, and a signal or other external 2319 * event cannot wake it up and insert it on the runqueue either. 2320 */ 2321 p->state = TASK_NEW; 2322 2323 /* 2324 * Make sure we do not leak PI boosting priority to the child. 2325 */ 2326 p->prio = current->normal_prio; 2327 2328 /* 2329 * Revert to default priority/policy on fork if requested. 2330 */ 2331 if (unlikely(p->sched_reset_on_fork)) { 2332 if (task_has_dl_policy(p) || task_has_rt_policy(p)) { 2333 p->policy = SCHED_NORMAL; 2334 p->static_prio = NICE_TO_PRIO(0); 2335 p->rt_priority = 0; 2336 } else if (PRIO_TO_NICE(p->static_prio) < 0) 2337 p->static_prio = NICE_TO_PRIO(0); 2338 2339 p->prio = p->normal_prio = __normal_prio(p); 2340 set_load_weight(p, false); 2341 2342 /* 2343 * We don't need the reset flag anymore after the fork. It has 2344 * fulfilled its duty: 2345 */ 2346 p->sched_reset_on_fork = 0; 2347 } 2348 2349 if (dl_prio(p->prio)) 2350 return -EAGAIN; 2351 else if (rt_prio(p->prio)) 2352 p->sched_class = &rt_sched_class; 2353 else 2354 p->sched_class = &fair_sched_class; 2355 2356 init_entity_runnable_average(&p->se); 2357 2358 /* 2359 * The child is not yet in the pid-hash so no cgroup attach races, 2360 * and the cgroup is pinned to this child due to cgroup_fork() 2361 * is ran before sched_fork(). 2362 * 2363 * Silence PROVE_RCU. 2364 */ 2365 raw_spin_lock_irqsave(&p->pi_lock, flags); 2366 /* 2367 * We're setting the CPU for the first time, we don't migrate, 2368 * so use __set_task_cpu(). 2369 */ 2370 __set_task_cpu(p, smp_processor_id()); 2371 if (p->sched_class->task_fork) 2372 p->sched_class->task_fork(p); 2373 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 2374 2375 #ifdef CONFIG_SCHED_INFO 2376 if (likely(sched_info_on())) 2377 memset(&p->sched_info, 0, sizeof(p->sched_info)); 2378 #endif 2379 #if defined(CONFIG_SMP) 2380 p->on_cpu = 0; 2381 #endif 2382 init_task_preempt_count(p); 2383 #ifdef CONFIG_SMP 2384 plist_node_init(&p->pushable_tasks, MAX_PRIO); 2385 RB_CLEAR_NODE(&p->pushable_dl_tasks); 2386 #endif 2387 return 0; 2388 } 2389 2390 unsigned long to_ratio(u64 period, u64 runtime) 2391 { 2392 if (runtime == RUNTIME_INF) 2393 return BW_UNIT; 2394 2395 /* 2396 * Doing this here saves a lot of checks in all 2397 * the calling paths, and returning zero seems 2398 * safe for them anyway. 2399 */ 2400 if (period == 0) 2401 return 0; 2402 2403 return div64_u64(runtime << BW_SHIFT, period); 2404 } 2405 2406 /* 2407 * wake_up_new_task - wake up a newly created task for the first time. 2408 * 2409 * This function will do some initial scheduler statistics housekeeping 2410 * that must be done for every newly created context, then puts the task 2411 * on the runqueue and wakes it. 2412 */ 2413 void wake_up_new_task(struct task_struct *p) 2414 { 2415 struct rq_flags rf; 2416 struct rq *rq; 2417 2418 raw_spin_lock_irqsave(&p->pi_lock, rf.flags); 2419 p->state = TASK_RUNNING; 2420 #ifdef CONFIG_SMP 2421 /* 2422 * Fork balancing, do it here and not earlier because: 2423 * - cpus_allowed can change in the fork path 2424 * - any previously selected CPU might disappear through hotplug 2425 * 2426 * Use __set_task_cpu() to avoid calling sched_class::migrate_task_rq, 2427 * as we're not fully set-up yet. 2428 */ 2429 p->recent_used_cpu = task_cpu(p); 2430 __set_task_cpu(p, select_task_rq(p, task_cpu(p), SD_BALANCE_FORK, 0)); 2431 #endif 2432 rq = __task_rq_lock(p, &rf); 2433 update_rq_clock(rq); 2434 post_init_entity_util_avg(&p->se); 2435 2436 activate_task(rq, p, ENQUEUE_NOCLOCK); 2437 p->on_rq = TASK_ON_RQ_QUEUED; 2438 trace_sched_wakeup_new(p); 2439 check_preempt_curr(rq, p, WF_FORK); 2440 #ifdef CONFIG_SMP 2441 if (p->sched_class->task_woken) { 2442 /* 2443 * Nothing relies on rq->lock after this, so its fine to 2444 * drop it. 2445 */ 2446 rq_unpin_lock(rq, &rf); 2447 p->sched_class->task_woken(rq, p); 2448 rq_repin_lock(rq, &rf); 2449 } 2450 #endif 2451 task_rq_unlock(rq, p, &rf); 2452 } 2453 2454 #ifdef CONFIG_PREEMPT_NOTIFIERS 2455 2456 static DEFINE_STATIC_KEY_FALSE(preempt_notifier_key); 2457 2458 void preempt_notifier_inc(void) 2459 { 2460 static_branch_inc(&preempt_notifier_key); 2461 } 2462 EXPORT_SYMBOL_GPL(preempt_notifier_inc); 2463 2464 void preempt_notifier_dec(void) 2465 { 2466 static_branch_dec(&preempt_notifier_key); 2467 } 2468 EXPORT_SYMBOL_GPL(preempt_notifier_dec); 2469 2470 /** 2471 * preempt_notifier_register - tell me when current is being preempted & rescheduled 2472 * @notifier: notifier struct to register 2473 */ 2474 void preempt_notifier_register(struct preempt_notifier *notifier) 2475 { 2476 if (!static_branch_unlikely(&preempt_notifier_key)) 2477 WARN(1, "registering preempt_notifier while notifiers disabled\n"); 2478 2479 hlist_add_head(¬ifier->link, ¤t->preempt_notifiers); 2480 } 2481 EXPORT_SYMBOL_GPL(preempt_notifier_register); 2482 2483 /** 2484 * preempt_notifier_unregister - no longer interested in preemption notifications 2485 * @notifier: notifier struct to unregister 2486 * 2487 * This is *not* safe to call from within a preemption notifier. 2488 */ 2489 void preempt_notifier_unregister(struct preempt_notifier *notifier) 2490 { 2491 hlist_del(¬ifier->link); 2492 } 2493 EXPORT_SYMBOL_GPL(preempt_notifier_unregister); 2494 2495 static void __fire_sched_in_preempt_notifiers(struct task_struct *curr) 2496 { 2497 struct preempt_notifier *notifier; 2498 2499 hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) 2500 notifier->ops->sched_in(notifier, raw_smp_processor_id()); 2501 } 2502 2503 static __always_inline void fire_sched_in_preempt_notifiers(struct task_struct *curr) 2504 { 2505 if (static_branch_unlikely(&preempt_notifier_key)) 2506 __fire_sched_in_preempt_notifiers(curr); 2507 } 2508 2509 static void 2510 __fire_sched_out_preempt_notifiers(struct task_struct *curr, 2511 struct task_struct *next) 2512 { 2513 struct preempt_notifier *notifier; 2514 2515 hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) 2516 notifier->ops->sched_out(notifier, next); 2517 } 2518 2519 static __always_inline void 2520 fire_sched_out_preempt_notifiers(struct task_struct *curr, 2521 struct task_struct *next) 2522 { 2523 if (static_branch_unlikely(&preempt_notifier_key)) 2524 __fire_sched_out_preempt_notifiers(curr, next); 2525 } 2526 2527 #else /* !CONFIG_PREEMPT_NOTIFIERS */ 2528 2529 static inline void fire_sched_in_preempt_notifiers(struct task_struct *curr) 2530 { 2531 } 2532 2533 static inline void 2534 fire_sched_out_preempt_notifiers(struct task_struct *curr, 2535 struct task_struct *next) 2536 { 2537 } 2538 2539 #endif /* CONFIG_PREEMPT_NOTIFIERS */ 2540 2541 static inline void prepare_task(struct task_struct *next) 2542 { 2543 #ifdef CONFIG_SMP 2544 /* 2545 * Claim the task as running, we do this before switching to it 2546 * such that any running task will have this set. 2547 */ 2548 next->on_cpu = 1; 2549 #endif 2550 } 2551 2552 static inline void finish_task(struct task_struct *prev) 2553 { 2554 #ifdef CONFIG_SMP 2555 /* 2556 * After ->on_cpu is cleared, the task can be moved to a different CPU. 2557 * We must ensure this doesn't happen until the switch is completely 2558 * finished. 2559 * 2560 * In particular, the load of prev->state in finish_task_switch() must 2561 * happen before this. 2562 * 2563 * Pairs with the smp_cond_load_acquire() in try_to_wake_up(). 2564 */ 2565 smp_store_release(&prev->on_cpu, 0); 2566 #endif 2567 } 2568 2569 static inline void 2570 prepare_lock_switch(struct rq *rq, struct task_struct *next, struct rq_flags *rf) 2571 { 2572 /* 2573 * Since the runqueue lock will be released by the next 2574 * task (which is an invalid locking op but in the case 2575 * of the scheduler it's an obvious special-case), so we 2576 * do an early lockdep release here: 2577 */ 2578 rq_unpin_lock(rq, rf); 2579 spin_release(&rq->lock.dep_map, 1, _THIS_IP_); 2580 #ifdef CONFIG_DEBUG_SPINLOCK 2581 /* this is a valid case when another task releases the spinlock */ 2582 rq->lock.owner = next; 2583 #endif 2584 } 2585 2586 static inline void finish_lock_switch(struct rq *rq) 2587 { 2588 /* 2589 * If we are tracking spinlock dependencies then we have to 2590 * fix up the runqueue lock - which gets 'carried over' from 2591 * prev into current: 2592 */ 2593 spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_); 2594 raw_spin_unlock_irq(&rq->lock); 2595 } 2596 2597 /* 2598 * NOP if the arch has not defined these: 2599 */ 2600 2601 #ifndef prepare_arch_switch 2602 # define prepare_arch_switch(next) do { } while (0) 2603 #endif 2604 2605 #ifndef finish_arch_post_lock_switch 2606 # define finish_arch_post_lock_switch() do { } while (0) 2607 #endif 2608 2609 /** 2610 * prepare_task_switch - prepare to switch tasks 2611 * @rq: the runqueue preparing to switch 2612 * @prev: the current task that is being switched out 2613 * @next: the task we are going to switch to. 2614 * 2615 * This is called with the rq lock held and interrupts off. It must 2616 * be paired with a subsequent finish_task_switch after the context 2617 * switch. 2618 * 2619 * prepare_task_switch sets up locking and calls architecture specific 2620 * hooks. 2621 */ 2622 static inline void 2623 prepare_task_switch(struct rq *rq, struct task_struct *prev, 2624 struct task_struct *next) 2625 { 2626 kcov_prepare_switch(prev); 2627 sched_info_switch(rq, prev, next); 2628 perf_event_task_sched_out(prev, next); 2629 rseq_preempt(prev); 2630 fire_sched_out_preempt_notifiers(prev, next); 2631 prepare_task(next); 2632 prepare_arch_switch(next); 2633 } 2634 2635 /** 2636 * finish_task_switch - clean up after a task-switch 2637 * @prev: the thread we just switched away from. 2638 * 2639 * finish_task_switch must be called after the context switch, paired 2640 * with a prepare_task_switch call before the context switch. 2641 * finish_task_switch will reconcile locking set up by prepare_task_switch, 2642 * and do any other architecture-specific cleanup actions. 2643 * 2644 * Note that we may have delayed dropping an mm in context_switch(). If 2645 * so, we finish that here outside of the runqueue lock. (Doing it 2646 * with the lock held can cause deadlocks; see schedule() for 2647 * details.) 2648 * 2649 * The context switch have flipped the stack from under us and restored the 2650 * local variables which were saved when this task called schedule() in the 2651 * past. prev == current is still correct but we need to recalculate this_rq 2652 * because prev may have moved to another CPU. 2653 */ 2654 static struct rq *finish_task_switch(struct task_struct *prev) 2655 __releases(rq->lock) 2656 { 2657 struct rq *rq = this_rq(); 2658 struct mm_struct *mm = rq->prev_mm; 2659 long prev_state; 2660 2661 /* 2662 * The previous task will have left us with a preempt_count of 2 2663 * because it left us after: 2664 * 2665 * schedule() 2666 * preempt_disable(); // 1 2667 * __schedule() 2668 * raw_spin_lock_irq(&rq->lock) // 2 2669 * 2670 * Also, see FORK_PREEMPT_COUNT. 2671 */ 2672 if (WARN_ONCE(preempt_count() != 2*PREEMPT_DISABLE_OFFSET, 2673 "corrupted preempt_count: %s/%d/0x%x\n", 2674 current->comm, current->pid, preempt_count())) 2675 preempt_count_set(FORK_PREEMPT_COUNT); 2676 2677 rq->prev_mm = NULL; 2678 2679 /* 2680 * A task struct has one reference for the use as "current". 2681 * If a task dies, then it sets TASK_DEAD in tsk->state and calls 2682 * schedule one last time. The schedule call will never return, and 2683 * the scheduled task must drop that reference. 2684 * 2685 * We must observe prev->state before clearing prev->on_cpu (in 2686 * finish_task), otherwise a concurrent wakeup can get prev 2687 * running on another CPU and we could rave with its RUNNING -> DEAD 2688 * transition, resulting in a double drop. 2689 */ 2690 prev_state = prev->state; 2691 vtime_task_switch(prev); 2692 perf_event_task_sched_in(prev, current); 2693 finish_task(prev); 2694 finish_lock_switch(rq); 2695 finish_arch_post_lock_switch(); 2696 kcov_finish_switch(current); 2697 2698 fire_sched_in_preempt_notifiers(current); 2699 /* 2700 * When switching through a kernel thread, the loop in 2701 * membarrier_{private,global}_expedited() may have observed that 2702 * kernel thread and not issued an IPI. It is therefore possible to 2703 * schedule between user->kernel->user threads without passing though 2704 * switch_mm(). Membarrier requires a barrier after storing to 2705 * rq->curr, before returning to userspace, so provide them here: 2706 * 2707 * - a full memory barrier for {PRIVATE,GLOBAL}_EXPEDITED, implicitly 2708 * provided by mmdrop(), 2709 * - a sync_core for SYNC_CORE. 2710 */ 2711 if (mm) { 2712 membarrier_mm_sync_core_before_usermode(mm); 2713 mmdrop(mm); 2714 } 2715 if (unlikely(prev_state == TASK_DEAD)) { 2716 if (prev->sched_class->task_dead) 2717 prev->sched_class->task_dead(prev); 2718 2719 /* 2720 * Remove function-return probe instances associated with this 2721 * task and put them back on the free list. 2722 */ 2723 kprobe_flush_task(prev); 2724 2725 /* Task is done with its stack. */ 2726 put_task_stack(prev); 2727 2728 put_task_struct(prev); 2729 } 2730 2731 tick_nohz_task_switch(); 2732 return rq; 2733 } 2734 2735 #ifdef CONFIG_SMP 2736 2737 /* rq->lock is NOT held, but preemption is disabled */ 2738 static void __balance_callback(struct rq *rq) 2739 { 2740 struct callback_head *head, *next; 2741 void (*func)(struct rq *rq); 2742 unsigned long flags; 2743 2744 raw_spin_lock_irqsave(&rq->lock, flags); 2745 head = rq->balance_callback; 2746 rq->balance_callback = NULL; 2747 while (head) { 2748 func = (void (*)(struct rq *))head->func; 2749 next = head->next; 2750 head->next = NULL; 2751 head = next; 2752 2753 func(rq); 2754 } 2755 raw_spin_unlock_irqrestore(&rq->lock, flags); 2756 } 2757 2758 static inline void balance_callback(struct rq *rq) 2759 { 2760 if (unlikely(rq->balance_callback)) 2761 __balance_callback(rq); 2762 } 2763 2764 #else 2765 2766 static inline void balance_callback(struct rq *rq) 2767 { 2768 } 2769 2770 #endif 2771 2772 /** 2773 * schedule_tail - first thing a freshly forked thread must call. 2774 * @prev: the thread we just switched away from. 2775 */ 2776 asmlinkage __visible void schedule_tail(struct task_struct *prev) 2777 __releases(rq->lock) 2778 { 2779 struct rq *rq; 2780 2781 /* 2782 * New tasks start with FORK_PREEMPT_COUNT, see there and 2783 * finish_task_switch() for details. 2784 * 2785 * finish_task_switch() will drop rq->lock() and lower preempt_count 2786 * and the preempt_enable() will end up enabling preemption (on 2787 * PREEMPT_COUNT kernels). 2788 */ 2789 2790 rq = finish_task_switch(prev); 2791 balance_callback(rq); 2792 preempt_enable(); 2793 2794 if (current->set_child_tid) 2795 put_user(task_pid_vnr(current), current->set_child_tid); 2796 2797 calculate_sigpending(); 2798 } 2799 2800 /* 2801 * context_switch - switch to the new MM and the new thread's register state. 2802 */ 2803 static __always_inline struct rq * 2804 context_switch(struct rq *rq, struct task_struct *prev, 2805 struct task_struct *next, struct rq_flags *rf) 2806 { 2807 struct mm_struct *mm, *oldmm; 2808 2809 prepare_task_switch(rq, prev, next); 2810 2811 mm = next->mm; 2812 oldmm = prev->active_mm; 2813 /* 2814 * For paravirt, this is coupled with an exit in switch_to to 2815 * combine the page table reload and the switch backend into 2816 * one hypercall. 2817 */ 2818 arch_start_context_switch(prev); 2819 2820 /* 2821 * If mm is non-NULL, we pass through switch_mm(). If mm is 2822 * NULL, we will pass through mmdrop() in finish_task_switch(). 2823 * Both of these contain the full memory barrier required by 2824 * membarrier after storing to rq->curr, before returning to 2825 * user-space. 2826 */ 2827 if (!mm) { 2828 next->active_mm = oldmm; 2829 mmgrab(oldmm); 2830 enter_lazy_tlb(oldmm, next); 2831 } else 2832 switch_mm_irqs_off(oldmm, mm, next); 2833 2834 if (!prev->mm) { 2835 prev->active_mm = NULL; 2836 rq->prev_mm = oldmm; 2837 } 2838 2839 rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP); 2840 2841 prepare_lock_switch(rq, next, rf); 2842 2843 /* Here we just switch the register state and the stack. */ 2844 switch_to(prev, next, prev); 2845 barrier(); 2846 2847 return finish_task_switch(prev); 2848 } 2849 2850 /* 2851 * nr_running and nr_context_switches: 2852 * 2853 * externally visible scheduler statistics: current number of runnable 2854 * threads, total number of context switches performed since bootup. 2855 */ 2856 unsigned long nr_running(void) 2857 { 2858 unsigned long i, sum = 0; 2859 2860 for_each_online_cpu(i) 2861 sum += cpu_rq(i)->nr_running; 2862 2863 return sum; 2864 } 2865 2866 /* 2867 * Check if only the current task is running on the CPU. 2868 * 2869 * Caution: this function does not check that the caller has disabled 2870 * preemption, thus the result might have a time-of-check-to-time-of-use 2871 * race. The caller is responsible to use it correctly, for example: 2872 * 2873 * - from a non-preemptible section (of course) 2874 * 2875 * - from a thread that is bound to a single CPU 2876 * 2877 * - in a loop with very short iterations (e.g. a polling loop) 2878 */ 2879 bool single_task_running(void) 2880 { 2881 return raw_rq()->nr_running == 1; 2882 } 2883 EXPORT_SYMBOL(single_task_running); 2884 2885 unsigned long long nr_context_switches(void) 2886 { 2887 int i; 2888 unsigned long long sum = 0; 2889 2890 for_each_possible_cpu(i) 2891 sum += cpu_rq(i)->nr_switches; 2892 2893 return sum; 2894 } 2895 2896 /* 2897 * Consumers of these two interfaces, like for example the cpuidle menu 2898 * governor, are using nonsensical data. Preferring shallow idle state selection 2899 * for a CPU that has IO-wait which might not even end up running the task when 2900 * it does become runnable. 2901 */ 2902 2903 unsigned long nr_iowait_cpu(int cpu) 2904 { 2905 return atomic_read(&cpu_rq(cpu)->nr_iowait); 2906 } 2907 2908 /* 2909 * IO-wait accounting, and how its mostly bollocks (on SMP). 2910 * 2911 * The idea behind IO-wait account is to account the idle time that we could 2912 * have spend running if it were not for IO. That is, if we were to improve the 2913 * storage performance, we'd have a proportional reduction in IO-wait time. 2914 * 2915 * This all works nicely on UP, where, when a task blocks on IO, we account 2916 * idle time as IO-wait, because if the storage were faster, it could've been 2917 * running and we'd not be idle. 2918 * 2919 * This has been extended to SMP, by doing the same for each CPU. This however 2920 * is broken. 2921 * 2922 * Imagine for instance the case where two tasks block on one CPU, only the one 2923 * CPU will have IO-wait accounted, while the other has regular idle. Even 2924 * though, if the storage were faster, both could've ran at the same time, 2925 * utilising both CPUs. 2926 * 2927 * This means, that when looking globally, the current IO-wait accounting on 2928 * SMP is a lower bound, by reason of under accounting. 2929 * 2930 * Worse, since the numbers are provided per CPU, they are sometimes 2931 * interpreted per CPU, and that is nonsensical. A blocked task isn't strictly 2932 * associated with any one particular CPU, it can wake to another CPU than it 2933 * blocked on. This means the per CPU IO-wait number is meaningless. 2934 * 2935 * Task CPU affinities can make all that even more 'interesting'. 2936 */ 2937 2938 unsigned long nr_iowait(void) 2939 { 2940 unsigned long i, sum = 0; 2941 2942 for_each_possible_cpu(i) 2943 sum += nr_iowait_cpu(i); 2944 2945 return sum; 2946 } 2947 2948 #ifdef CONFIG_SMP 2949 2950 /* 2951 * sched_exec - execve() is a valuable balancing opportunity, because at 2952 * this point the task has the smallest effective memory and cache footprint. 2953 */ 2954 void sched_exec(void) 2955 { 2956 struct task_struct *p = current; 2957 unsigned long flags; 2958 int dest_cpu; 2959 2960 raw_spin_lock_irqsave(&p->pi_lock, flags); 2961 dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), SD_BALANCE_EXEC, 0); 2962 if (dest_cpu == smp_processor_id()) 2963 goto unlock; 2964 2965 if (likely(cpu_active(dest_cpu))) { 2966 struct migration_arg arg = { p, dest_cpu }; 2967 2968 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 2969 stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg); 2970 return; 2971 } 2972 unlock: 2973 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 2974 } 2975 2976 #endif 2977 2978 DEFINE_PER_CPU(struct kernel_stat, kstat); 2979 DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat); 2980 2981 EXPORT_PER_CPU_SYMBOL(kstat); 2982 EXPORT_PER_CPU_SYMBOL(kernel_cpustat); 2983 2984 /* 2985 * The function fair_sched_class.update_curr accesses the struct curr 2986 * and its field curr->exec_start; when called from task_sched_runtime(), 2987 * we observe a high rate of cache misses in practice. 2988 * Prefetching this data results in improved performance. 2989 */ 2990 static inline void prefetch_curr_exec_start(struct task_struct *p) 2991 { 2992 #ifdef CONFIG_FAIR_GROUP_SCHED 2993 struct sched_entity *curr = (&p->se)->cfs_rq->curr; 2994 #else 2995 struct sched_entity *curr = (&task_rq(p)->cfs)->curr; 2996 #endif 2997 prefetch(curr); 2998 prefetch(&curr->exec_start); 2999 } 3000 3001 /* 3002 * Return accounted runtime for the task. 3003 * In case the task is currently running, return the runtime plus current's 3004 * pending runtime that have not been accounted yet. 3005 */ 3006 unsigned long long task_sched_runtime(struct task_struct *p) 3007 { 3008 struct rq_flags rf; 3009 struct rq *rq; 3010 u64 ns; 3011 3012 #if defined(CONFIG_64BIT) && defined(CONFIG_SMP) 3013 /* 3014 * 64-bit doesn't need locks to atomically read a 64-bit value. 3015 * So we have a optimization chance when the task's delta_exec is 0. 3016 * Reading ->on_cpu is racy, but this is ok. 3017 * 3018 * If we race with it leaving CPU, we'll take a lock. So we're correct. 3019 * If we race with it entering CPU, unaccounted time is 0. This is 3020 * indistinguishable from the read occurring a few cycles earlier. 3021 * If we see ->on_cpu without ->on_rq, the task is leaving, and has 3022 * been accounted, so we're correct here as well. 3023 */ 3024 if (!p->on_cpu || !task_on_rq_queued(p)) 3025 return p->se.sum_exec_runtime; 3026 #endif 3027 3028 rq = task_rq_lock(p, &rf); 3029 /* 3030 * Must be ->curr _and_ ->on_rq. If dequeued, we would 3031 * project cycles that may never be accounted to this 3032 * thread, breaking clock_gettime(). 3033 */ 3034 if (task_current(rq, p) && task_on_rq_queued(p)) { 3035 prefetch_curr_exec_start(p); 3036 update_rq_clock(rq); 3037 p->sched_class->update_curr(rq); 3038 } 3039 ns = p->se.sum_exec_runtime; 3040 task_rq_unlock(rq, p, &rf); 3041 3042 return ns; 3043 } 3044 3045 /* 3046 * This function gets called by the timer code, with HZ frequency. 3047 * We call it with interrupts disabled. 3048 */ 3049 void scheduler_tick(void) 3050 { 3051 int cpu = smp_processor_id(); 3052 struct rq *rq = cpu_rq(cpu); 3053 struct task_struct *curr = rq->curr; 3054 struct rq_flags rf; 3055 3056 sched_clock_tick(); 3057 3058 rq_lock(rq, &rf); 3059 3060 update_rq_clock(rq); 3061 curr->sched_class->task_tick(rq, curr, 0); 3062 cpu_load_update_active(rq); 3063 calc_global_load_tick(rq); 3064 psi_task_tick(rq); 3065 3066 rq_unlock(rq, &rf); 3067 3068 perf_event_task_tick(); 3069 3070 #ifdef CONFIG_SMP 3071 rq->idle_balance = idle_cpu(cpu); 3072 trigger_load_balance(rq); 3073 #endif 3074 } 3075 3076 #ifdef CONFIG_NO_HZ_FULL 3077 3078 struct tick_work { 3079 int cpu; 3080 struct delayed_work work; 3081 }; 3082 3083 static struct tick_work __percpu *tick_work_cpu; 3084 3085 static void sched_tick_remote(struct work_struct *work) 3086 { 3087 struct delayed_work *dwork = to_delayed_work(work); 3088 struct tick_work *twork = container_of(dwork, struct tick_work, work); 3089 int cpu = twork->cpu; 3090 struct rq *rq = cpu_rq(cpu); 3091 struct task_struct *curr; 3092 struct rq_flags rf; 3093 u64 delta; 3094 3095 /* 3096 * Handle the tick only if it appears the remote CPU is running in full 3097 * dynticks mode. The check is racy by nature, but missing a tick or 3098 * having one too much is no big deal because the scheduler tick updates 3099 * statistics and checks timeslices in a time-independent way, regardless 3100 * of when exactly it is running. 3101 */ 3102 if (idle_cpu(cpu) || !tick_nohz_tick_stopped_cpu(cpu)) 3103 goto out_requeue; 3104 3105 rq_lock_irq(rq, &rf); 3106 curr = rq->curr; 3107 if (is_idle_task(curr)) 3108 goto out_unlock; 3109 3110 update_rq_clock(rq); 3111 delta = rq_clock_task(rq) - curr->se.exec_start; 3112 3113 /* 3114 * Make sure the next tick runs within a reasonable 3115 * amount of time. 3116 */ 3117 WARN_ON_ONCE(delta > (u64)NSEC_PER_SEC * 3); 3118 curr->sched_class->task_tick(rq, curr, 0); 3119 3120 out_unlock: 3121 rq_unlock_irq(rq, &rf); 3122 3123 out_requeue: 3124 /* 3125 * Run the remote tick once per second (1Hz). This arbitrary 3126 * frequency is large enough to avoid overload but short enough 3127 * to keep scheduler internal stats reasonably up to date. 3128 */ 3129 queue_delayed_work(system_unbound_wq, dwork, HZ); 3130 } 3131 3132 static void sched_tick_start(int cpu) 3133 { 3134 struct tick_work *twork; 3135 3136 if (housekeeping_cpu(cpu, HK_FLAG_TICK)) 3137 return; 3138 3139 WARN_ON_ONCE(!tick_work_cpu); 3140 3141 twork = per_cpu_ptr(tick_work_cpu, cpu); 3142 twork->cpu = cpu; 3143 INIT_DELAYED_WORK(&twork->work, sched_tick_remote); 3144 queue_delayed_work(system_unbound_wq, &twork->work, HZ); 3145 } 3146 3147 #ifdef CONFIG_HOTPLUG_CPU 3148 static void sched_tick_stop(int cpu) 3149 { 3150 struct tick_work *twork; 3151 3152 if (housekeeping_cpu(cpu, HK_FLAG_TICK)) 3153 return; 3154 3155 WARN_ON_ONCE(!tick_work_cpu); 3156 3157 twork = per_cpu_ptr(tick_work_cpu, cpu); 3158 cancel_delayed_work_sync(&twork->work); 3159 } 3160 #endif /* CONFIG_HOTPLUG_CPU */ 3161 3162 int __init sched_tick_offload_init(void) 3163 { 3164 tick_work_cpu = alloc_percpu(struct tick_work); 3165 BUG_ON(!tick_work_cpu); 3166 3167 return 0; 3168 } 3169 3170 #else /* !CONFIG_NO_HZ_FULL */ 3171 static inline void sched_tick_start(int cpu) { } 3172 static inline void sched_tick_stop(int cpu) { } 3173 #endif 3174 3175 #if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \ 3176 defined(CONFIG_TRACE_PREEMPT_TOGGLE)) 3177 /* 3178 * If the value passed in is equal to the current preempt count 3179 * then we just disabled preemption. Start timing the latency. 3180 */ 3181 static inline void preempt_latency_start(int val) 3182 { 3183 if (preempt_count() == val) { 3184 unsigned long ip = get_lock_parent_ip(); 3185 #ifdef CONFIG_DEBUG_PREEMPT 3186 current->preempt_disable_ip = ip; 3187 #endif 3188 trace_preempt_off(CALLER_ADDR0, ip); 3189 } 3190 } 3191 3192 void preempt_count_add(int val) 3193 { 3194 #ifdef CONFIG_DEBUG_PREEMPT 3195 /* 3196 * Underflow? 3197 */ 3198 if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) 3199 return; 3200 #endif 3201 __preempt_count_add(val); 3202 #ifdef CONFIG_DEBUG_PREEMPT 3203 /* 3204 * Spinlock count overflowing soon? 3205 */ 3206 DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >= 3207 PREEMPT_MASK - 10); 3208 #endif 3209 preempt_latency_start(val); 3210 } 3211 EXPORT_SYMBOL(preempt_count_add); 3212 NOKPROBE_SYMBOL(preempt_count_add); 3213 3214 /* 3215 * If the value passed in equals to the current preempt count 3216 * then we just enabled preemption. Stop timing the latency. 3217 */ 3218 static inline void preempt_latency_stop(int val) 3219 { 3220 if (preempt_count() == val) 3221 trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip()); 3222 } 3223 3224 void preempt_count_sub(int val) 3225 { 3226 #ifdef CONFIG_DEBUG_PREEMPT 3227 /* 3228 * Underflow? 3229 */ 3230 if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) 3231 return; 3232 /* 3233 * Is the spinlock portion underflowing? 3234 */ 3235 if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) && 3236 !(preempt_count() & PREEMPT_MASK))) 3237 return; 3238 #endif 3239 3240 preempt_latency_stop(val); 3241 __preempt_count_sub(val); 3242 } 3243 EXPORT_SYMBOL(preempt_count_sub); 3244 NOKPROBE_SYMBOL(preempt_count_sub); 3245 3246 #else 3247 static inline void preempt_latency_start(int val) { } 3248 static inline void preempt_latency_stop(int val) { } 3249 #endif 3250 3251 static inline unsigned long get_preempt_disable_ip(struct task_struct *p) 3252 { 3253 #ifdef CONFIG_DEBUG_PREEMPT 3254 return p->preempt_disable_ip; 3255 #else 3256 return 0; 3257 #endif 3258 } 3259 3260 /* 3261 * Print scheduling while atomic bug: 3262 */ 3263 static noinline void __schedule_bug(struct task_struct *prev) 3264 { 3265 /* Save this before calling printk(), since that will clobber it */ 3266 unsigned long preempt_disable_ip = get_preempt_disable_ip(current); 3267 3268 if (oops_in_progress) 3269 return; 3270 3271 printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n", 3272 prev->comm, prev->pid, preempt_count()); 3273 3274 debug_show_held_locks(prev); 3275 print_modules(); 3276 if (irqs_disabled()) 3277 print_irqtrace_events(prev); 3278 if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) 3279 && in_atomic_preempt_off()) { 3280 pr_err("Preemption disabled at:"); 3281 print_ip_sym(preempt_disable_ip); 3282 pr_cont("\n"); 3283 } 3284 if (panic_on_warn) 3285 panic("scheduling while atomic\n"); 3286 3287 dump_stack(); 3288 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 3289 } 3290 3291 /* 3292 * Various schedule()-time debugging checks and statistics: 3293 */ 3294 static inline void schedule_debug(struct task_struct *prev) 3295 { 3296 #ifdef CONFIG_SCHED_STACK_END_CHECK 3297 if (task_stack_end_corrupted(prev)) 3298 panic("corrupted stack end detected inside scheduler\n"); 3299 #endif 3300 3301 if (unlikely(in_atomic_preempt_off())) { 3302 __schedule_bug(prev); 3303 preempt_count_set(PREEMPT_DISABLED); 3304 } 3305 rcu_sleep_check(); 3306 3307 profile_hit(SCHED_PROFILING, __builtin_return_address(0)); 3308 3309 schedstat_inc(this_rq()->sched_count); 3310 } 3311 3312 /* 3313 * Pick up the highest-prio task: 3314 */ 3315 static inline struct task_struct * 3316 pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 3317 { 3318 const struct sched_class *class; 3319 struct task_struct *p; 3320 3321 /* 3322 * Optimization: we know that if all tasks are in the fair class we can 3323 * call that function directly, but only if the @prev task wasn't of a 3324 * higher scheduling class, because otherwise those loose the 3325 * opportunity to pull in more work from other CPUs. 3326 */ 3327 if (likely((prev->sched_class == &idle_sched_class || 3328 prev->sched_class == &fair_sched_class) && 3329 rq->nr_running == rq->cfs.h_nr_running)) { 3330 3331 p = fair_sched_class.pick_next_task(rq, prev, rf); 3332 if (unlikely(p == RETRY_TASK)) 3333 goto again; 3334 3335 /* Assumes fair_sched_class->next == idle_sched_class */ 3336 if (unlikely(!p)) 3337 p = idle_sched_class.pick_next_task(rq, prev, rf); 3338 3339 return p; 3340 } 3341 3342 again: 3343 for_each_class(class) { 3344 p = class->pick_next_task(rq, prev, rf); 3345 if (p) { 3346 if (unlikely(p == RETRY_TASK)) 3347 goto again; 3348 return p; 3349 } 3350 } 3351 3352 /* The idle class should always have a runnable task: */ 3353 BUG(); 3354 } 3355 3356 /* 3357 * __schedule() is the main scheduler function. 3358 * 3359 * The main means of driving the scheduler and thus entering this function are: 3360 * 3361 * 1. Explicit blocking: mutex, semaphore, waitqueue, etc. 3362 * 3363 * 2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return 3364 * paths. For example, see arch/x86/entry_64.S. 3365 * 3366 * To drive preemption between tasks, the scheduler sets the flag in timer 3367 * interrupt handler scheduler_tick(). 3368 * 3369 * 3. Wakeups don't really cause entry into schedule(). They add a 3370 * task to the run-queue and that's it. 3371 * 3372 * Now, if the new task added to the run-queue preempts the current 3373 * task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets 3374 * called on the nearest possible occasion: 3375 * 3376 * - If the kernel is preemptible (CONFIG_PREEMPT=y): 3377 * 3378 * - in syscall or exception context, at the next outmost 3379 * preempt_enable(). (this might be as soon as the wake_up()'s 3380 * spin_unlock()!) 3381 * 3382 * - in IRQ context, return from interrupt-handler to 3383 * preemptible context 3384 * 3385 * - If the kernel is not preemptible (CONFIG_PREEMPT is not set) 3386 * then at the next: 3387 * 3388 * - cond_resched() call 3389 * - explicit schedule() call 3390 * - return from syscall or exception to user-space 3391 * - return from interrupt-handler to user-space 3392 * 3393 * WARNING: must be called with preemption disabled! 3394 */ 3395 static void __sched notrace __schedule(bool preempt) 3396 { 3397 struct task_struct *prev, *next; 3398 unsigned long *switch_count; 3399 struct rq_flags rf; 3400 struct rq *rq; 3401 int cpu; 3402 3403 cpu = smp_processor_id(); 3404 rq = cpu_rq(cpu); 3405 prev = rq->curr; 3406 3407 schedule_debug(prev); 3408 3409 if (sched_feat(HRTICK)) 3410 hrtick_clear(rq); 3411 3412 local_irq_disable(); 3413 rcu_note_context_switch(preempt); 3414 3415 /* 3416 * Make sure that signal_pending_state()->signal_pending() below 3417 * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE) 3418 * done by the caller to avoid the race with signal_wake_up(). 3419 * 3420 * The membarrier system call requires a full memory barrier 3421 * after coming from user-space, before storing to rq->curr. 3422 */ 3423 rq_lock(rq, &rf); 3424 smp_mb__after_spinlock(); 3425 3426 /* Promote REQ to ACT */ 3427 rq->clock_update_flags <<= 1; 3428 update_rq_clock(rq); 3429 3430 switch_count = &prev->nivcsw; 3431 if (!preempt && prev->state) { 3432 if (signal_pending_state(prev->state, prev)) { 3433 prev->state = TASK_RUNNING; 3434 } else { 3435 deactivate_task(rq, prev, DEQUEUE_SLEEP | DEQUEUE_NOCLOCK); 3436 prev->on_rq = 0; 3437 3438 if (prev->in_iowait) { 3439 atomic_inc(&rq->nr_iowait); 3440 delayacct_blkio_start(); 3441 } 3442 3443 /* 3444 * If a worker went to sleep, notify and ask workqueue 3445 * whether it wants to wake up a task to maintain 3446 * concurrency. 3447 */ 3448 if (prev->flags & PF_WQ_WORKER) { 3449 struct task_struct *to_wakeup; 3450 3451 to_wakeup = wq_worker_sleeping(prev); 3452 if (to_wakeup) 3453 try_to_wake_up_local(to_wakeup, &rf); 3454 } 3455 } 3456 switch_count = &prev->nvcsw; 3457 } 3458 3459 next = pick_next_task(rq, prev, &rf); 3460 clear_tsk_need_resched(prev); 3461 clear_preempt_need_resched(); 3462 3463 if (likely(prev != next)) { 3464 rq->nr_switches++; 3465 rq->curr = next; 3466 /* 3467 * The membarrier system call requires each architecture 3468 * to have a full memory barrier after updating 3469 * rq->curr, before returning to user-space. 3470 * 3471 * Here are the schemes providing that barrier on the 3472 * various architectures: 3473 * - mm ? switch_mm() : mmdrop() for x86, s390, sparc, PowerPC. 3474 * switch_mm() rely on membarrier_arch_switch_mm() on PowerPC. 3475 * - finish_lock_switch() for weakly-ordered 3476 * architectures where spin_unlock is a full barrier, 3477 * - switch_to() for arm64 (weakly-ordered, spin_unlock 3478 * is a RELEASE barrier), 3479 */ 3480 ++*switch_count; 3481 3482 trace_sched_switch(preempt, prev, next); 3483 3484 /* Also unlocks the rq: */ 3485 rq = context_switch(rq, prev, next, &rf); 3486 } else { 3487 rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP); 3488 rq_unlock_irq(rq, &rf); 3489 } 3490 3491 balance_callback(rq); 3492 } 3493 3494 void __noreturn do_task_dead(void) 3495 { 3496 /* Causes final put_task_struct in finish_task_switch(): */ 3497 set_special_state(TASK_DEAD); 3498 3499 /* Tell freezer to ignore us: */ 3500 current->flags |= PF_NOFREEZE; 3501 3502 __schedule(false); 3503 BUG(); 3504 3505 /* Avoid "noreturn function does return" - but don't continue if BUG() is a NOP: */ 3506 for (;;) 3507 cpu_relax(); 3508 } 3509 3510 static inline void sched_submit_work(struct task_struct *tsk) 3511 { 3512 if (!tsk->state || tsk_is_pi_blocked(tsk)) 3513 return; 3514 /* 3515 * If we are going to sleep and we have plugged IO queued, 3516 * make sure to submit it to avoid deadlocks. 3517 */ 3518 if (blk_needs_flush_plug(tsk)) 3519 blk_schedule_flush_plug(tsk); 3520 } 3521 3522 asmlinkage __visible void __sched schedule(void) 3523 { 3524 struct task_struct *tsk = current; 3525 3526 sched_submit_work(tsk); 3527 do { 3528 preempt_disable(); 3529 __schedule(false); 3530 sched_preempt_enable_no_resched(); 3531 } while (need_resched()); 3532 } 3533 EXPORT_SYMBOL(schedule); 3534 3535 /* 3536 * synchronize_rcu_tasks() makes sure that no task is stuck in preempted 3537 * state (have scheduled out non-voluntarily) by making sure that all 3538 * tasks have either left the run queue or have gone into user space. 3539 * As idle tasks do not do either, they must not ever be preempted 3540 * (schedule out non-voluntarily). 3541 * 3542 * schedule_idle() is similar to schedule_preempt_disable() except that it 3543 * never enables preemption because it does not call sched_submit_work(). 3544 */ 3545 void __sched schedule_idle(void) 3546 { 3547 /* 3548 * As this skips calling sched_submit_work(), which the idle task does 3549 * regardless because that function is a nop when the task is in a 3550 * TASK_RUNNING state, make sure this isn't used someplace that the 3551 * current task can be in any other state. Note, idle is always in the 3552 * TASK_RUNNING state. 3553 */ 3554 WARN_ON_ONCE(current->state); 3555 do { 3556 __schedule(false); 3557 } while (need_resched()); 3558 } 3559 3560 #ifdef CONFIG_CONTEXT_TRACKING 3561 asmlinkage __visible void __sched schedule_user(void) 3562 { 3563 /* 3564 * If we come here after a random call to set_need_resched(), 3565 * or we have been woken up remotely but the IPI has not yet arrived, 3566 * we haven't yet exited the RCU idle mode. Do it here manually until 3567 * we find a better solution. 3568 * 3569 * NB: There are buggy callers of this function. Ideally we 3570 * should warn if prev_state != CONTEXT_USER, but that will trigger 3571 * too frequently to make sense yet. 3572 */ 3573 enum ctx_state prev_state = exception_enter(); 3574 schedule(); 3575 exception_exit(prev_state); 3576 } 3577 #endif 3578 3579 /** 3580 * schedule_preempt_disabled - called with preemption disabled 3581 * 3582 * Returns with preemption disabled. Note: preempt_count must be 1 3583 */ 3584 void __sched schedule_preempt_disabled(void) 3585 { 3586 sched_preempt_enable_no_resched(); 3587 schedule(); 3588 preempt_disable(); 3589 } 3590 3591 static void __sched notrace preempt_schedule_common(void) 3592 { 3593 do { 3594 /* 3595 * Because the function tracer can trace preempt_count_sub() 3596 * and it also uses preempt_enable/disable_notrace(), if 3597 * NEED_RESCHED is set, the preempt_enable_notrace() called 3598 * by the function tracer will call this function again and 3599 * cause infinite recursion. 3600 * 3601 * Preemption must be disabled here before the function 3602 * tracer can trace. Break up preempt_disable() into two 3603 * calls. One to disable preemption without fear of being 3604 * traced. The other to still record the preemption latency, 3605 * which can also be traced by the function tracer. 3606 */ 3607 preempt_disable_notrace(); 3608 preempt_latency_start(1); 3609 __schedule(true); 3610 preempt_latency_stop(1); 3611 preempt_enable_no_resched_notrace(); 3612 3613 /* 3614 * Check again in case we missed a preemption opportunity 3615 * between schedule and now. 3616 */ 3617 } while (need_resched()); 3618 } 3619 3620 #ifdef CONFIG_PREEMPT 3621 /* 3622 * this is the entry point to schedule() from in-kernel preemption 3623 * off of preempt_enable. Kernel preemptions off return from interrupt 3624 * occur there and call schedule directly. 3625 */ 3626 asmlinkage __visible void __sched notrace preempt_schedule(void) 3627 { 3628 /* 3629 * If there is a non-zero preempt_count or interrupts are disabled, 3630 * we do not want to preempt the current task. Just return.. 3631 */ 3632 if (likely(!preemptible())) 3633 return; 3634 3635 preempt_schedule_common(); 3636 } 3637 NOKPROBE_SYMBOL(preempt_schedule); 3638 EXPORT_SYMBOL(preempt_schedule); 3639 3640 /** 3641 * preempt_schedule_notrace - preempt_schedule called by tracing 3642 * 3643 * The tracing infrastructure uses preempt_enable_notrace to prevent 3644 * recursion and tracing preempt enabling caused by the tracing 3645 * infrastructure itself. But as tracing can happen in areas coming 3646 * from userspace or just about to enter userspace, a preempt enable 3647 * can occur before user_exit() is called. This will cause the scheduler 3648 * to be called when the system is still in usermode. 3649 * 3650 * To prevent this, the preempt_enable_notrace will use this function 3651 * instead of preempt_schedule() to exit user context if needed before 3652 * calling the scheduler. 3653 */ 3654 asmlinkage __visible void __sched notrace preempt_schedule_notrace(void) 3655 { 3656 enum ctx_state prev_ctx; 3657 3658 if (likely(!preemptible())) 3659 return; 3660 3661 do { 3662 /* 3663 * Because the function tracer can trace preempt_count_sub() 3664 * and it also uses preempt_enable/disable_notrace(), if 3665 * NEED_RESCHED is set, the preempt_enable_notrace() called 3666 * by the function tracer will call this function again and 3667 * cause infinite recursion. 3668 * 3669 * Preemption must be disabled here before the function 3670 * tracer can trace. Break up preempt_disable() into two 3671 * calls. One to disable preemption without fear of being 3672 * traced. The other to still record the preemption latency, 3673 * which can also be traced by the function tracer. 3674 */ 3675 preempt_disable_notrace(); 3676 preempt_latency_start(1); 3677 /* 3678 * Needs preempt disabled in case user_exit() is traced 3679 * and the tracer calls preempt_enable_notrace() causing 3680 * an infinite recursion. 3681 */ 3682 prev_ctx = exception_enter(); 3683 __schedule(true); 3684 exception_exit(prev_ctx); 3685 3686 preempt_latency_stop(1); 3687 preempt_enable_no_resched_notrace(); 3688 } while (need_resched()); 3689 } 3690 EXPORT_SYMBOL_GPL(preempt_schedule_notrace); 3691 3692 #endif /* CONFIG_PREEMPT */ 3693 3694 /* 3695 * this is the entry point to schedule() from kernel preemption 3696 * off of irq context. 3697 * Note, that this is called and return with irqs disabled. This will 3698 * protect us against recursive calling from irq. 3699 */ 3700 asmlinkage __visible void __sched preempt_schedule_irq(void) 3701 { 3702 enum ctx_state prev_state; 3703 3704 /* Catch callers which need to be fixed */ 3705 BUG_ON(preempt_count() || !irqs_disabled()); 3706 3707 prev_state = exception_enter(); 3708 3709 do { 3710 preempt_disable(); 3711 local_irq_enable(); 3712 __schedule(true); 3713 local_irq_disable(); 3714 sched_preempt_enable_no_resched(); 3715 } while (need_resched()); 3716 3717 exception_exit(prev_state); 3718 } 3719 3720 int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flags, 3721 void *key) 3722 { 3723 return try_to_wake_up(curr->private, mode, wake_flags); 3724 } 3725 EXPORT_SYMBOL(default_wake_function); 3726 3727 #ifdef CONFIG_RT_MUTEXES 3728 3729 static inline int __rt_effective_prio(struct task_struct *pi_task, int prio) 3730 { 3731 if (pi_task) 3732 prio = min(prio, pi_task->prio); 3733 3734 return prio; 3735 } 3736 3737 static inline int rt_effective_prio(struct task_struct *p, int prio) 3738 { 3739 struct task_struct *pi_task = rt_mutex_get_top_task(p); 3740 3741 return __rt_effective_prio(pi_task, prio); 3742 } 3743 3744 /* 3745 * rt_mutex_setprio - set the current priority of a task 3746 * @p: task to boost 3747 * @pi_task: donor task 3748 * 3749 * This function changes the 'effective' priority of a task. It does 3750 * not touch ->normal_prio like __setscheduler(). 3751 * 3752 * Used by the rt_mutex code to implement priority inheritance 3753 * logic. Call site only calls if the priority of the task changed. 3754 */ 3755 void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) 3756 { 3757 int prio, oldprio, queued, running, queue_flag = 3758 DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; 3759 const struct sched_class *prev_class; 3760 struct rq_flags rf; 3761 struct rq *rq; 3762 3763 /* XXX used to be waiter->prio, not waiter->task->prio */ 3764 prio = __rt_effective_prio(pi_task, p->normal_prio); 3765 3766 /* 3767 * If nothing changed; bail early. 3768 */ 3769 if (p->pi_top_task == pi_task && prio == p->prio && !dl_prio(prio)) 3770 return; 3771 3772 rq = __task_rq_lock(p, &rf); 3773 update_rq_clock(rq); 3774 /* 3775 * Set under pi_lock && rq->lock, such that the value can be used under 3776 * either lock. 3777 * 3778 * Note that there is loads of tricky to make this pointer cache work 3779 * right. rt_mutex_slowunlock()+rt_mutex_postunlock() work together to 3780 * ensure a task is de-boosted (pi_task is set to NULL) before the 3781 * task is allowed to run again (and can exit). This ensures the pointer 3782 * points to a blocked task -- which guaratees the task is present. 3783 */ 3784 p->pi_top_task = pi_task; 3785 3786 /* 3787 * For FIFO/RR we only need to set prio, if that matches we're done. 3788 */ 3789 if (prio == p->prio && !dl_prio(prio)) 3790 goto out_unlock; 3791 3792 /* 3793 * Idle task boosting is a nono in general. There is one 3794 * exception, when PREEMPT_RT and NOHZ is active: 3795 * 3796 * The idle task calls get_next_timer_interrupt() and holds 3797 * the timer wheel base->lock on the CPU and another CPU wants 3798 * to access the timer (probably to cancel it). We can safely 3799 * ignore the boosting request, as the idle CPU runs this code 3800 * with interrupts disabled and will complete the lock 3801 * protected section without being interrupted. So there is no 3802 * real need to boost. 3803 */ 3804 if (unlikely(p == rq->idle)) { 3805 WARN_ON(p != rq->curr); 3806 WARN_ON(p->pi_blocked_on); 3807 goto out_unlock; 3808 } 3809 3810 trace_sched_pi_setprio(p, pi_task); 3811 oldprio = p->prio; 3812 3813 if (oldprio == prio) 3814 queue_flag &= ~DEQUEUE_MOVE; 3815 3816 prev_class = p->sched_class; 3817 queued = task_on_rq_queued(p); 3818 running = task_current(rq, p); 3819 if (queued) 3820 dequeue_task(rq, p, queue_flag); 3821 if (running) 3822 put_prev_task(rq, p); 3823 3824 /* 3825 * Boosting condition are: 3826 * 1. -rt task is running and holds mutex A 3827 * --> -dl task blocks on mutex A 3828 * 3829 * 2. -dl task is running and holds mutex A 3830 * --> -dl task blocks on mutex A and could preempt the 3831 * running task 3832 */ 3833 if (dl_prio(prio)) { 3834 if (!dl_prio(p->normal_prio) || 3835 (pi_task && dl_entity_preempt(&pi_task->dl, &p->dl))) { 3836 p->dl.dl_boosted = 1; 3837 queue_flag |= ENQUEUE_REPLENISH; 3838 } else 3839 p->dl.dl_boosted = 0; 3840 p->sched_class = &dl_sched_class; 3841 } else if (rt_prio(prio)) { 3842 if (dl_prio(oldprio)) 3843 p->dl.dl_boosted = 0; 3844 if (oldprio < prio) 3845 queue_flag |= ENQUEUE_HEAD; 3846 p->sched_class = &rt_sched_class; 3847 } else { 3848 if (dl_prio(oldprio)) 3849 p->dl.dl_boosted = 0; 3850 if (rt_prio(oldprio)) 3851 p->rt.timeout = 0; 3852 p->sched_class = &fair_sched_class; 3853 } 3854 3855 p->prio = prio; 3856 3857 if (queued) 3858 enqueue_task(rq, p, queue_flag); 3859 if (running) 3860 set_curr_task(rq, p); 3861 3862 check_class_changed(rq, p, prev_class, oldprio); 3863 out_unlock: 3864 /* Avoid rq from going away on us: */ 3865 preempt_disable(); 3866 __task_rq_unlock(rq, &rf); 3867 3868 balance_callback(rq); 3869 preempt_enable(); 3870 } 3871 #else 3872 static inline int rt_effective_prio(struct task_struct *p, int prio) 3873 { 3874 return prio; 3875 } 3876 #endif 3877 3878 void set_user_nice(struct task_struct *p, long nice) 3879 { 3880 bool queued, running; 3881 int old_prio, delta; 3882 struct rq_flags rf; 3883 struct rq *rq; 3884 3885 if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE) 3886 return; 3887 /* 3888 * We have to be careful, if called from sys_setpriority(), 3889 * the task might be in the middle of scheduling on another CPU. 3890 */ 3891 rq = task_rq_lock(p, &rf); 3892 update_rq_clock(rq); 3893 3894 /* 3895 * The RT priorities are set via sched_setscheduler(), but we still 3896 * allow the 'normal' nice value to be set - but as expected 3897 * it wont have any effect on scheduling until the task is 3898 * SCHED_DEADLINE, SCHED_FIFO or SCHED_RR: 3899 */ 3900 if (task_has_dl_policy(p) || task_has_rt_policy(p)) { 3901 p->static_prio = NICE_TO_PRIO(nice); 3902 goto out_unlock; 3903 } 3904 queued = task_on_rq_queued(p); 3905 running = task_current(rq, p); 3906 if (queued) 3907 dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK); 3908 if (running) 3909 put_prev_task(rq, p); 3910 3911 p->static_prio = NICE_TO_PRIO(nice); 3912 set_load_weight(p, true); 3913 old_prio = p->prio; 3914 p->prio = effective_prio(p); 3915 delta = p->prio - old_prio; 3916 3917 if (queued) { 3918 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); 3919 /* 3920 * If the task increased its priority or is running and 3921 * lowered its priority, then reschedule its CPU: 3922 */ 3923 if (delta < 0 || (delta > 0 && task_running(rq, p))) 3924 resched_curr(rq); 3925 } 3926 if (running) 3927 set_curr_task(rq, p); 3928 out_unlock: 3929 task_rq_unlock(rq, p, &rf); 3930 } 3931 EXPORT_SYMBOL(set_user_nice); 3932 3933 /* 3934 * can_nice - check if a task can reduce its nice value 3935 * @p: task 3936 * @nice: nice value 3937 */ 3938 int can_nice(const struct task_struct *p, const int nice) 3939 { 3940 /* Convert nice value [19,-20] to rlimit style value [1,40]: */ 3941 int nice_rlim = nice_to_rlimit(nice); 3942 3943 return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) || 3944 capable(CAP_SYS_NICE)); 3945 } 3946 3947 #ifdef __ARCH_WANT_SYS_NICE 3948 3949 /* 3950 * sys_nice - change the priority of the current process. 3951 * @increment: priority increment 3952 * 3953 * sys_setpriority is a more generic, but much slower function that 3954 * does similar things. 3955 */ 3956 SYSCALL_DEFINE1(nice, int, increment) 3957 { 3958 long nice, retval; 3959 3960 /* 3961 * Setpriority might change our priority at the same moment. 3962 * We don't have to worry. Conceptually one call occurs first 3963 * and we have a single winner. 3964 */ 3965 increment = clamp(increment, -NICE_WIDTH, NICE_WIDTH); 3966 nice = task_nice(current) + increment; 3967 3968 nice = clamp_val(nice, MIN_NICE, MAX_NICE); 3969 if (increment < 0 && !can_nice(current, nice)) 3970 return -EPERM; 3971 3972 retval = security_task_setnice(current, nice); 3973 if (retval) 3974 return retval; 3975 3976 set_user_nice(current, nice); 3977 return 0; 3978 } 3979 3980 #endif 3981 3982 /** 3983 * task_prio - return the priority value of a given task. 3984 * @p: the task in question. 3985 * 3986 * Return: The priority value as seen by users in /proc. 3987 * RT tasks are offset by -200. Normal tasks are centered 3988 * around 0, value goes from -16 to +15. 3989 */ 3990 int task_prio(const struct task_struct *p) 3991 { 3992 return p->prio - MAX_RT_PRIO; 3993 } 3994 3995 /** 3996 * idle_cpu - is a given CPU idle currently? 3997 * @cpu: the processor in question. 3998 * 3999 * Return: 1 if the CPU is currently idle. 0 otherwise. 4000 */ 4001 int idle_cpu(int cpu) 4002 { 4003 struct rq *rq = cpu_rq(cpu); 4004 4005 if (rq->curr != rq->idle) 4006 return 0; 4007 4008 if (rq->nr_running) 4009 return 0; 4010 4011 #ifdef CONFIG_SMP 4012 if (!llist_empty(&rq->wake_list)) 4013 return 0; 4014 #endif 4015 4016 return 1; 4017 } 4018 4019 /** 4020 * available_idle_cpu - is a given CPU idle for enqueuing work. 4021 * @cpu: the CPU in question. 4022 * 4023 * Return: 1 if the CPU is currently idle. 0 otherwise. 4024 */ 4025 int available_idle_cpu(int cpu) 4026 { 4027 if (!idle_cpu(cpu)) 4028 return 0; 4029 4030 if (vcpu_is_preempted(cpu)) 4031 return 0; 4032 4033 return 1; 4034 } 4035 4036 /** 4037 * idle_task - return the idle task for a given CPU. 4038 * @cpu: the processor in question. 4039 * 4040 * Return: The idle task for the CPU @cpu. 4041 */ 4042 struct task_struct *idle_task(int cpu) 4043 { 4044 return cpu_rq(cpu)->idle; 4045 } 4046 4047 /** 4048 * find_process_by_pid - find a process with a matching PID value. 4049 * @pid: the pid in question. 4050 * 4051 * The task of @pid, if found. %NULL otherwise. 4052 */ 4053 static struct task_struct *find_process_by_pid(pid_t pid) 4054 { 4055 return pid ? find_task_by_vpid(pid) : current; 4056 } 4057 4058 /* 4059 * sched_setparam() passes in -1 for its policy, to let the functions 4060 * it calls know not to change it. 4061 */ 4062 #define SETPARAM_POLICY -1 4063 4064 static void __setscheduler_params(struct task_struct *p, 4065 const struct sched_attr *attr) 4066 { 4067 int policy = attr->sched_policy; 4068 4069 if (policy == SETPARAM_POLICY) 4070 policy = p->policy; 4071 4072 p->policy = policy; 4073 4074 if (dl_policy(policy)) 4075 __setparam_dl(p, attr); 4076 else if (fair_policy(policy)) 4077 p->static_prio = NICE_TO_PRIO(attr->sched_nice); 4078 4079 /* 4080 * __sched_setscheduler() ensures attr->sched_priority == 0 when 4081 * !rt_policy. Always setting this ensures that things like 4082 * getparam()/getattr() don't report silly values for !rt tasks. 4083 */ 4084 p->rt_priority = attr->sched_priority; 4085 p->normal_prio = normal_prio(p); 4086 set_load_weight(p, true); 4087 } 4088 4089 /* Actually do priority change: must hold pi & rq lock. */ 4090 static void __setscheduler(struct rq *rq, struct task_struct *p, 4091 const struct sched_attr *attr, bool keep_boost) 4092 { 4093 __setscheduler_params(p, attr); 4094 4095 /* 4096 * Keep a potential priority boosting if called from 4097 * sched_setscheduler(). 4098 */ 4099 p->prio = normal_prio(p); 4100 if (keep_boost) 4101 p->prio = rt_effective_prio(p, p->prio); 4102 4103 if (dl_prio(p->prio)) 4104 p->sched_class = &dl_sched_class; 4105 else if (rt_prio(p->prio)) 4106 p->sched_class = &rt_sched_class; 4107 else 4108 p->sched_class = &fair_sched_class; 4109 } 4110 4111 /* 4112 * Check the target process has a UID that matches the current process's: 4113 */ 4114 static bool check_same_owner(struct task_struct *p) 4115 { 4116 const struct cred *cred = current_cred(), *pcred; 4117 bool match; 4118 4119 rcu_read_lock(); 4120 pcred = __task_cred(p); 4121 match = (uid_eq(cred->euid, pcred->euid) || 4122 uid_eq(cred->euid, pcred->uid)); 4123 rcu_read_unlock(); 4124 return match; 4125 } 4126 4127 static int __sched_setscheduler(struct task_struct *p, 4128 const struct sched_attr *attr, 4129 bool user, bool pi) 4130 { 4131 int newprio = dl_policy(attr->sched_policy) ? MAX_DL_PRIO - 1 : 4132 MAX_RT_PRIO - 1 - attr->sched_priority; 4133 int retval, oldprio, oldpolicy = -1, queued, running; 4134 int new_effective_prio, policy = attr->sched_policy; 4135 const struct sched_class *prev_class; 4136 struct rq_flags rf; 4137 int reset_on_fork; 4138 int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; 4139 struct rq *rq; 4140 4141 /* The pi code expects interrupts enabled */ 4142 BUG_ON(pi && in_interrupt()); 4143 recheck: 4144 /* Double check policy once rq lock held: */ 4145 if (policy < 0) { 4146 reset_on_fork = p->sched_reset_on_fork; 4147 policy = oldpolicy = p->policy; 4148 } else { 4149 reset_on_fork = !!(attr->sched_flags & SCHED_FLAG_RESET_ON_FORK); 4150 4151 if (!valid_policy(policy)) 4152 return -EINVAL; 4153 } 4154 4155 if (attr->sched_flags & ~(SCHED_FLAG_ALL | SCHED_FLAG_SUGOV)) 4156 return -EINVAL; 4157 4158 /* 4159 * Valid priorities for SCHED_FIFO and SCHED_RR are 4160 * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL, 4161 * SCHED_BATCH and SCHED_IDLE is 0. 4162 */ 4163 if ((p->mm && attr->sched_priority > MAX_USER_RT_PRIO-1) || 4164 (!p->mm && attr->sched_priority > MAX_RT_PRIO-1)) 4165 return -EINVAL; 4166 if ((dl_policy(policy) && !__checkparam_dl(attr)) || 4167 (rt_policy(policy) != (attr->sched_priority != 0))) 4168 return -EINVAL; 4169 4170 /* 4171 * Allow unprivileged RT tasks to decrease priority: 4172 */ 4173 if (user && !capable(CAP_SYS_NICE)) { 4174 if (fair_policy(policy)) { 4175 if (attr->sched_nice < task_nice(p) && 4176 !can_nice(p, attr->sched_nice)) 4177 return -EPERM; 4178 } 4179 4180 if (rt_policy(policy)) { 4181 unsigned long rlim_rtprio = 4182 task_rlimit(p, RLIMIT_RTPRIO); 4183 4184 /* Can't set/change the rt policy: */ 4185 if (policy != p->policy && !rlim_rtprio) 4186 return -EPERM; 4187 4188 /* Can't increase priority: */ 4189 if (attr->sched_priority > p->rt_priority && 4190 attr->sched_priority > rlim_rtprio) 4191 return -EPERM; 4192 } 4193 4194 /* 4195 * Can't set/change SCHED_DEADLINE policy at all for now 4196 * (safest behavior); in the future we would like to allow 4197 * unprivileged DL tasks to increase their relative deadline 4198 * or reduce their runtime (both ways reducing utilization) 4199 */ 4200 if (dl_policy(policy)) 4201 return -EPERM; 4202 4203 /* 4204 * Treat SCHED_IDLE as nice 20. Only allow a switch to 4205 * SCHED_NORMAL if the RLIMIT_NICE would normally permit it. 4206 */ 4207 if (task_has_idle_policy(p) && !idle_policy(policy)) { 4208 if (!can_nice(p, task_nice(p))) 4209 return -EPERM; 4210 } 4211 4212 /* Can't change other user's priorities: */ 4213 if (!check_same_owner(p)) 4214 return -EPERM; 4215 4216 /* Normal users shall not reset the sched_reset_on_fork flag: */ 4217 if (p->sched_reset_on_fork && !reset_on_fork) 4218 return -EPERM; 4219 } 4220 4221 if (user) { 4222 if (attr->sched_flags & SCHED_FLAG_SUGOV) 4223 return -EINVAL; 4224 4225 retval = security_task_setscheduler(p); 4226 if (retval) 4227 return retval; 4228 } 4229 4230 /* 4231 * Make sure no PI-waiters arrive (or leave) while we are 4232 * changing the priority of the task: 4233 * 4234 * To be able to change p->policy safely, the appropriate 4235 * runqueue lock must be held. 4236 */ 4237 rq = task_rq_lock(p, &rf); 4238 update_rq_clock(rq); 4239 4240 /* 4241 * Changing the policy of the stop threads its a very bad idea: 4242 */ 4243 if (p == rq->stop) { 4244 task_rq_unlock(rq, p, &rf); 4245 return -EINVAL; 4246 } 4247 4248 /* 4249 * If not changing anything there's no need to proceed further, 4250 * but store a possible modification of reset_on_fork. 4251 */ 4252 if (unlikely(policy == p->policy)) { 4253 if (fair_policy(policy) && attr->sched_nice != task_nice(p)) 4254 goto change; 4255 if (rt_policy(policy) && attr->sched_priority != p->rt_priority) 4256 goto change; 4257 if (dl_policy(policy) && dl_param_changed(p, attr)) 4258 goto change; 4259 4260 p->sched_reset_on_fork = reset_on_fork; 4261 task_rq_unlock(rq, p, &rf); 4262 return 0; 4263 } 4264 change: 4265 4266 if (user) { 4267 #ifdef CONFIG_RT_GROUP_SCHED 4268 /* 4269 * Do not allow realtime tasks into groups that have no runtime 4270 * assigned. 4271 */ 4272 if (rt_bandwidth_enabled() && rt_policy(policy) && 4273 task_group(p)->rt_bandwidth.rt_runtime == 0 && 4274 !task_group_is_autogroup(task_group(p))) { 4275 task_rq_unlock(rq, p, &rf); 4276 return -EPERM; 4277 } 4278 #endif 4279 #ifdef CONFIG_SMP 4280 if (dl_bandwidth_enabled() && dl_policy(policy) && 4281 !(attr->sched_flags & SCHED_FLAG_SUGOV)) { 4282 cpumask_t *span = rq->rd->span; 4283 4284 /* 4285 * Don't allow tasks with an affinity mask smaller than 4286 * the entire root_domain to become SCHED_DEADLINE. We 4287 * will also fail if there's no bandwidth available. 4288 */ 4289 if (!cpumask_subset(span, &p->cpus_allowed) || 4290 rq->rd->dl_bw.bw == 0) { 4291 task_rq_unlock(rq, p, &rf); 4292 return -EPERM; 4293 } 4294 } 4295 #endif 4296 } 4297 4298 /* Re-check policy now with rq lock held: */ 4299 if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { 4300 policy = oldpolicy = -1; 4301 task_rq_unlock(rq, p, &rf); 4302 goto recheck; 4303 } 4304 4305 /* 4306 * If setscheduling to SCHED_DEADLINE (or changing the parameters 4307 * of a SCHED_DEADLINE task) we need to check if enough bandwidth 4308 * is available. 4309 */ 4310 if ((dl_policy(policy) || dl_task(p)) && sched_dl_overflow(p, policy, attr)) { 4311 task_rq_unlock(rq, p, &rf); 4312 return -EBUSY; 4313 } 4314 4315 p->sched_reset_on_fork = reset_on_fork; 4316 oldprio = p->prio; 4317 4318 if (pi) { 4319 /* 4320 * Take priority boosted tasks into account. If the new 4321 * effective priority is unchanged, we just store the new 4322 * normal parameters and do not touch the scheduler class and 4323 * the runqueue. This will be done when the task deboost 4324 * itself. 4325 */ 4326 new_effective_prio = rt_effective_prio(p, newprio); 4327 if (new_effective_prio == oldprio) 4328 queue_flags &= ~DEQUEUE_MOVE; 4329 } 4330 4331 queued = task_on_rq_queued(p); 4332 running = task_current(rq, p); 4333 if (queued) 4334 dequeue_task(rq, p, queue_flags); 4335 if (running) 4336 put_prev_task(rq, p); 4337 4338 prev_class = p->sched_class; 4339 __setscheduler(rq, p, attr, pi); 4340 4341 if (queued) { 4342 /* 4343 * We enqueue to tail when the priority of a task is 4344 * increased (user space view). 4345 */ 4346 if (oldprio < p->prio) 4347 queue_flags |= ENQUEUE_HEAD; 4348 4349 enqueue_task(rq, p, queue_flags); 4350 } 4351 if (running) 4352 set_curr_task(rq, p); 4353 4354 check_class_changed(rq, p, prev_class, oldprio); 4355 4356 /* Avoid rq from going away on us: */ 4357 preempt_disable(); 4358 task_rq_unlock(rq, p, &rf); 4359 4360 if (pi) 4361 rt_mutex_adjust_pi(p); 4362 4363 /* Run balance callbacks after we've adjusted the PI chain: */ 4364 balance_callback(rq); 4365 preempt_enable(); 4366 4367 return 0; 4368 } 4369 4370 static int _sched_setscheduler(struct task_struct *p, int policy, 4371 const struct sched_param *param, bool check) 4372 { 4373 struct sched_attr attr = { 4374 .sched_policy = policy, 4375 .sched_priority = param->sched_priority, 4376 .sched_nice = PRIO_TO_NICE(p->static_prio), 4377 }; 4378 4379 /* Fixup the legacy SCHED_RESET_ON_FORK hack. */ 4380 if ((policy != SETPARAM_POLICY) && (policy & SCHED_RESET_ON_FORK)) { 4381 attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK; 4382 policy &= ~SCHED_RESET_ON_FORK; 4383 attr.sched_policy = policy; 4384 } 4385 4386 return __sched_setscheduler(p, &attr, check, true); 4387 } 4388 /** 4389 * sched_setscheduler - change the scheduling policy and/or RT priority of a thread. 4390 * @p: the task in question. 4391 * @policy: new policy. 4392 * @param: structure containing the new RT priority. 4393 * 4394 * Return: 0 on success. An error code otherwise. 4395 * 4396 * NOTE that the task may be already dead. 4397 */ 4398 int sched_setscheduler(struct task_struct *p, int policy, 4399 const struct sched_param *param) 4400 { 4401 return _sched_setscheduler(p, policy, param, true); 4402 } 4403 EXPORT_SYMBOL_GPL(sched_setscheduler); 4404 4405 int sched_setattr(struct task_struct *p, const struct sched_attr *attr) 4406 { 4407 return __sched_setscheduler(p, attr, true, true); 4408 } 4409 EXPORT_SYMBOL_GPL(sched_setattr); 4410 4411 int sched_setattr_nocheck(struct task_struct *p, const struct sched_attr *attr) 4412 { 4413 return __sched_setscheduler(p, attr, false, true); 4414 } 4415 4416 /** 4417 * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace. 4418 * @p: the task in question. 4419 * @policy: new policy. 4420 * @param: structure containing the new RT priority. 4421 * 4422 * Just like sched_setscheduler, only don't bother checking if the 4423 * current context has permission. For example, this is needed in 4424 * stop_machine(): we create temporary high priority worker threads, 4425 * but our caller might not have that capability. 4426 * 4427 * Return: 0 on success. An error code otherwise. 4428 */ 4429 int sched_setscheduler_nocheck(struct task_struct *p, int policy, 4430 const struct sched_param *param) 4431 { 4432 return _sched_setscheduler(p, policy, param, false); 4433 } 4434 EXPORT_SYMBOL_GPL(sched_setscheduler_nocheck); 4435 4436 static int 4437 do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param) 4438 { 4439 struct sched_param lparam; 4440 struct task_struct *p; 4441 int retval; 4442 4443 if (!param || pid < 0) 4444 return -EINVAL; 4445 if (copy_from_user(&lparam, param, sizeof(struct sched_param))) 4446 return -EFAULT; 4447 4448 rcu_read_lock(); 4449 retval = -ESRCH; 4450 p = find_process_by_pid(pid); 4451 if (p != NULL) 4452 retval = sched_setscheduler(p, policy, &lparam); 4453 rcu_read_unlock(); 4454 4455 return retval; 4456 } 4457 4458 /* 4459 * Mimics kernel/events/core.c perf_copy_attr(). 4460 */ 4461 static int sched_copy_attr(struct sched_attr __user *uattr, struct sched_attr *attr) 4462 { 4463 u32 size; 4464 int ret; 4465 4466 if (!access_ok(uattr, SCHED_ATTR_SIZE_VER0)) 4467 return -EFAULT; 4468 4469 /* Zero the full structure, so that a short copy will be nice: */ 4470 memset(attr, 0, sizeof(*attr)); 4471 4472 ret = get_user(size, &uattr->size); 4473 if (ret) 4474 return ret; 4475 4476 /* Bail out on silly large: */ 4477 if (size > PAGE_SIZE) 4478 goto err_size; 4479 4480 /* ABI compatibility quirk: */ 4481 if (!size) 4482 size = SCHED_ATTR_SIZE_VER0; 4483 4484 if (size < SCHED_ATTR_SIZE_VER0) 4485 goto err_size; 4486 4487 /* 4488 * If we're handed a bigger struct than we know of, 4489 * ensure all the unknown bits are 0 - i.e. new 4490 * user-space does not rely on any kernel feature 4491 * extensions we dont know about yet. 4492 */ 4493 if (size > sizeof(*attr)) { 4494 unsigned char __user *addr; 4495 unsigned char __user *end; 4496 unsigned char val; 4497 4498 addr = (void __user *)uattr + sizeof(*attr); 4499 end = (void __user *)uattr + size; 4500 4501 for (; addr < end; addr++) { 4502 ret = get_user(val, addr); 4503 if (ret) 4504 return ret; 4505 if (val) 4506 goto err_size; 4507 } 4508 size = sizeof(*attr); 4509 } 4510 4511 ret = copy_from_user(attr, uattr, size); 4512 if (ret) 4513 return -EFAULT; 4514 4515 /* 4516 * XXX: Do we want to be lenient like existing syscalls; or do we want 4517 * to be strict and return an error on out-of-bounds values? 4518 */ 4519 attr->sched_nice = clamp(attr->sched_nice, MIN_NICE, MAX_NICE); 4520 4521 return 0; 4522 4523 err_size: 4524 put_user(sizeof(*attr), &uattr->size); 4525 return -E2BIG; 4526 } 4527 4528 /** 4529 * sys_sched_setscheduler - set/change the scheduler policy and RT priority 4530 * @pid: the pid in question. 4531 * @policy: new policy. 4532 * @param: structure containing the new RT priority. 4533 * 4534 * Return: 0 on success. An error code otherwise. 4535 */ 4536 SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy, struct sched_param __user *, param) 4537 { 4538 if (policy < 0) 4539 return -EINVAL; 4540 4541 return do_sched_setscheduler(pid, policy, param); 4542 } 4543 4544 /** 4545 * sys_sched_setparam - set/change the RT priority of a thread 4546 * @pid: the pid in question. 4547 * @param: structure containing the new RT priority. 4548 * 4549 * Return: 0 on success. An error code otherwise. 4550 */ 4551 SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param) 4552 { 4553 return do_sched_setscheduler(pid, SETPARAM_POLICY, param); 4554 } 4555 4556 /** 4557 * sys_sched_setattr - same as above, but with extended sched_attr 4558 * @pid: the pid in question. 4559 * @uattr: structure containing the extended parameters. 4560 * @flags: for future extension. 4561 */ 4562 SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr, 4563 unsigned int, flags) 4564 { 4565 struct sched_attr attr; 4566 struct task_struct *p; 4567 int retval; 4568 4569 if (!uattr || pid < 0 || flags) 4570 return -EINVAL; 4571 4572 retval = sched_copy_attr(uattr, &attr); 4573 if (retval) 4574 return retval; 4575 4576 if ((int)attr.sched_policy < 0) 4577 return -EINVAL; 4578 4579 rcu_read_lock(); 4580 retval = -ESRCH; 4581 p = find_process_by_pid(pid); 4582 if (p != NULL) 4583 retval = sched_setattr(p, &attr); 4584 rcu_read_unlock(); 4585 4586 return retval; 4587 } 4588 4589 /** 4590 * sys_sched_getscheduler - get the policy (scheduling class) of a thread 4591 * @pid: the pid in question. 4592 * 4593 * Return: On success, the policy of the thread. Otherwise, a negative error 4594 * code. 4595 */ 4596 SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid) 4597 { 4598 struct task_struct *p; 4599 int retval; 4600 4601 if (pid < 0) 4602 return -EINVAL; 4603 4604 retval = -ESRCH; 4605 rcu_read_lock(); 4606 p = find_process_by_pid(pid); 4607 if (p) { 4608 retval = security_task_getscheduler(p); 4609 if (!retval) 4610 retval = p->policy 4611 | (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0); 4612 } 4613 rcu_read_unlock(); 4614 return retval; 4615 } 4616 4617 /** 4618 * sys_sched_getparam - get the RT priority of a thread 4619 * @pid: the pid in question. 4620 * @param: structure containing the RT priority. 4621 * 4622 * Return: On success, 0 and the RT priority is in @param. Otherwise, an error 4623 * code. 4624 */ 4625 SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param) 4626 { 4627 struct sched_param lp = { .sched_priority = 0 }; 4628 struct task_struct *p; 4629 int retval; 4630 4631 if (!param || pid < 0) 4632 return -EINVAL; 4633 4634 rcu_read_lock(); 4635 p = find_process_by_pid(pid); 4636 retval = -ESRCH; 4637 if (!p) 4638 goto out_unlock; 4639 4640 retval = security_task_getscheduler(p); 4641 if (retval) 4642 goto out_unlock; 4643 4644 if (task_has_rt_policy(p)) 4645 lp.sched_priority = p->rt_priority; 4646 rcu_read_unlock(); 4647 4648 /* 4649 * This one might sleep, we cannot do it with a spinlock held ... 4650 */ 4651 retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0; 4652 4653 return retval; 4654 4655 out_unlock: 4656 rcu_read_unlock(); 4657 return retval; 4658 } 4659 4660 static int sched_read_attr(struct sched_attr __user *uattr, 4661 struct sched_attr *attr, 4662 unsigned int usize) 4663 { 4664 int ret; 4665 4666 if (!access_ok(uattr, usize)) 4667 return -EFAULT; 4668 4669 /* 4670 * If we're handed a smaller struct than we know of, 4671 * ensure all the unknown bits are 0 - i.e. old 4672 * user-space does not get uncomplete information. 4673 */ 4674 if (usize < sizeof(*attr)) { 4675 unsigned char *addr; 4676 unsigned char *end; 4677 4678 addr = (void *)attr + usize; 4679 end = (void *)attr + sizeof(*attr); 4680 4681 for (; addr < end; addr++) { 4682 if (*addr) 4683 return -EFBIG; 4684 } 4685 4686 attr->size = usize; 4687 } 4688 4689 ret = copy_to_user(uattr, attr, attr->size); 4690 if (ret) 4691 return -EFAULT; 4692 4693 return 0; 4694 } 4695 4696 /** 4697 * sys_sched_getattr - similar to sched_getparam, but with sched_attr 4698 * @pid: the pid in question. 4699 * @uattr: structure containing the extended parameters. 4700 * @size: sizeof(attr) for fwd/bwd comp. 4701 * @flags: for future extension. 4702 */ 4703 SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr, 4704 unsigned int, size, unsigned int, flags) 4705 { 4706 struct sched_attr attr = { 4707 .size = sizeof(struct sched_attr), 4708 }; 4709 struct task_struct *p; 4710 int retval; 4711 4712 if (!uattr || pid < 0 || size > PAGE_SIZE || 4713 size < SCHED_ATTR_SIZE_VER0 || flags) 4714 return -EINVAL; 4715 4716 rcu_read_lock(); 4717 p = find_process_by_pid(pid); 4718 retval = -ESRCH; 4719 if (!p) 4720 goto out_unlock; 4721 4722 retval = security_task_getscheduler(p); 4723 if (retval) 4724 goto out_unlock; 4725 4726 attr.sched_policy = p->policy; 4727 if (p->sched_reset_on_fork) 4728 attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK; 4729 if (task_has_dl_policy(p)) 4730 __getparam_dl(p, &attr); 4731 else if (task_has_rt_policy(p)) 4732 attr.sched_priority = p->rt_priority; 4733 else 4734 attr.sched_nice = task_nice(p); 4735 4736 rcu_read_unlock(); 4737 4738 retval = sched_read_attr(uattr, &attr, size); 4739 return retval; 4740 4741 out_unlock: 4742 rcu_read_unlock(); 4743 return retval; 4744 } 4745 4746 long sched_setaffinity(pid_t pid, const struct cpumask *in_mask) 4747 { 4748 cpumask_var_t cpus_allowed, new_mask; 4749 struct task_struct *p; 4750 int retval; 4751 4752 rcu_read_lock(); 4753 4754 p = find_process_by_pid(pid); 4755 if (!p) { 4756 rcu_read_unlock(); 4757 return -ESRCH; 4758 } 4759 4760 /* Prevent p going away */ 4761 get_task_struct(p); 4762 rcu_read_unlock(); 4763 4764 if (p->flags & PF_NO_SETAFFINITY) { 4765 retval = -EINVAL; 4766 goto out_put_task; 4767 } 4768 if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) { 4769 retval = -ENOMEM; 4770 goto out_put_task; 4771 } 4772 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) { 4773 retval = -ENOMEM; 4774 goto out_free_cpus_allowed; 4775 } 4776 retval = -EPERM; 4777 if (!check_same_owner(p)) { 4778 rcu_read_lock(); 4779 if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) { 4780 rcu_read_unlock(); 4781 goto out_free_new_mask; 4782 } 4783 rcu_read_unlock(); 4784 } 4785 4786 retval = security_task_setscheduler(p); 4787 if (retval) 4788 goto out_free_new_mask; 4789 4790 4791 cpuset_cpus_allowed(p, cpus_allowed); 4792 cpumask_and(new_mask, in_mask, cpus_allowed); 4793 4794 /* 4795 * Since bandwidth control happens on root_domain basis, 4796 * if admission test is enabled, we only admit -deadline 4797 * tasks allowed to run on all the CPUs in the task's 4798 * root_domain. 4799 */ 4800 #ifdef CONFIG_SMP 4801 if (task_has_dl_policy(p) && dl_bandwidth_enabled()) { 4802 rcu_read_lock(); 4803 if (!cpumask_subset(task_rq(p)->rd->span, new_mask)) { 4804 retval = -EBUSY; 4805 rcu_read_unlock(); 4806 goto out_free_new_mask; 4807 } 4808 rcu_read_unlock(); 4809 } 4810 #endif 4811 again: 4812 retval = __set_cpus_allowed_ptr(p, new_mask, true); 4813 4814 if (!retval) { 4815 cpuset_cpus_allowed(p, cpus_allowed); 4816 if (!cpumask_subset(new_mask, cpus_allowed)) { 4817 /* 4818 * We must have raced with a concurrent cpuset 4819 * update. Just reset the cpus_allowed to the 4820 * cpuset's cpus_allowed 4821 */ 4822 cpumask_copy(new_mask, cpus_allowed); 4823 goto again; 4824 } 4825 } 4826 out_free_new_mask: 4827 free_cpumask_var(new_mask); 4828 out_free_cpus_allowed: 4829 free_cpumask_var(cpus_allowed); 4830 out_put_task: 4831 put_task_struct(p); 4832 return retval; 4833 } 4834 4835 static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len, 4836 struct cpumask *new_mask) 4837 { 4838 if (len < cpumask_size()) 4839 cpumask_clear(new_mask); 4840 else if (len > cpumask_size()) 4841 len = cpumask_size(); 4842 4843 return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0; 4844 } 4845 4846 /** 4847 * sys_sched_setaffinity - set the CPU affinity of a process 4848 * @pid: pid of the process 4849 * @len: length in bytes of the bitmask pointed to by user_mask_ptr 4850 * @user_mask_ptr: user-space pointer to the new CPU mask 4851 * 4852 * Return: 0 on success. An error code otherwise. 4853 */ 4854 SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len, 4855 unsigned long __user *, user_mask_ptr) 4856 { 4857 cpumask_var_t new_mask; 4858 int retval; 4859 4860 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) 4861 return -ENOMEM; 4862 4863 retval = get_user_cpu_mask(user_mask_ptr, len, new_mask); 4864 if (retval == 0) 4865 retval = sched_setaffinity(pid, new_mask); 4866 free_cpumask_var(new_mask); 4867 return retval; 4868 } 4869 4870 long sched_getaffinity(pid_t pid, struct cpumask *mask) 4871 { 4872 struct task_struct *p; 4873 unsigned long flags; 4874 int retval; 4875 4876 rcu_read_lock(); 4877 4878 retval = -ESRCH; 4879 p = find_process_by_pid(pid); 4880 if (!p) 4881 goto out_unlock; 4882 4883 retval = security_task_getscheduler(p); 4884 if (retval) 4885 goto out_unlock; 4886 4887 raw_spin_lock_irqsave(&p->pi_lock, flags); 4888 cpumask_and(mask, &p->cpus_allowed, cpu_active_mask); 4889 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 4890 4891 out_unlock: 4892 rcu_read_unlock(); 4893 4894 return retval; 4895 } 4896 4897 /** 4898 * sys_sched_getaffinity - get the CPU affinity of a process 4899 * @pid: pid of the process 4900 * @len: length in bytes of the bitmask pointed to by user_mask_ptr 4901 * @user_mask_ptr: user-space pointer to hold the current CPU mask 4902 * 4903 * Return: size of CPU mask copied to user_mask_ptr on success. An 4904 * error code otherwise. 4905 */ 4906 SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len, 4907 unsigned long __user *, user_mask_ptr) 4908 { 4909 int ret; 4910 cpumask_var_t mask; 4911 4912 if ((len * BITS_PER_BYTE) < nr_cpu_ids) 4913 return -EINVAL; 4914 if (len & (sizeof(unsigned long)-1)) 4915 return -EINVAL; 4916 4917 if (!alloc_cpumask_var(&mask, GFP_KERNEL)) 4918 return -ENOMEM; 4919 4920 ret = sched_getaffinity(pid, mask); 4921 if (ret == 0) { 4922 unsigned int retlen = min(len, cpumask_size()); 4923 4924 if (copy_to_user(user_mask_ptr, mask, retlen)) 4925 ret = -EFAULT; 4926 else 4927 ret = retlen; 4928 } 4929 free_cpumask_var(mask); 4930 4931 return ret; 4932 } 4933 4934 /** 4935 * sys_sched_yield - yield the current processor to other threads. 4936 * 4937 * This function yields the current CPU to other tasks. If there are no 4938 * other threads running on this CPU then this function will return. 4939 * 4940 * Return: 0. 4941 */ 4942 static void do_sched_yield(void) 4943 { 4944 struct rq_flags rf; 4945 struct rq *rq; 4946 4947 rq = this_rq_lock_irq(&rf); 4948 4949 schedstat_inc(rq->yld_count); 4950 current->sched_class->yield_task(rq); 4951 4952 /* 4953 * Since we are going to call schedule() anyway, there's 4954 * no need to preempt or enable interrupts: 4955 */ 4956 preempt_disable(); 4957 rq_unlock(rq, &rf); 4958 sched_preempt_enable_no_resched(); 4959 4960 schedule(); 4961 } 4962 4963 SYSCALL_DEFINE0(sched_yield) 4964 { 4965 do_sched_yield(); 4966 return 0; 4967 } 4968 4969 #ifndef CONFIG_PREEMPT 4970 int __sched _cond_resched(void) 4971 { 4972 if (should_resched(0)) { 4973 preempt_schedule_common(); 4974 return 1; 4975 } 4976 rcu_all_qs(); 4977 return 0; 4978 } 4979 EXPORT_SYMBOL(_cond_resched); 4980 #endif 4981 4982 /* 4983 * __cond_resched_lock() - if a reschedule is pending, drop the given lock, 4984 * call schedule, and on return reacquire the lock. 4985 * 4986 * This works OK both with and without CONFIG_PREEMPT. We do strange low-level 4987 * operations here to prevent schedule() from being called twice (once via 4988 * spin_unlock(), once by hand). 4989 */ 4990 int __cond_resched_lock(spinlock_t *lock) 4991 { 4992 int resched = should_resched(PREEMPT_LOCK_OFFSET); 4993 int ret = 0; 4994 4995 lockdep_assert_held(lock); 4996 4997 if (spin_needbreak(lock) || resched) { 4998 spin_unlock(lock); 4999 if (resched) 5000 preempt_schedule_common(); 5001 else 5002 cpu_relax(); 5003 ret = 1; 5004 spin_lock(lock); 5005 } 5006 return ret; 5007 } 5008 EXPORT_SYMBOL(__cond_resched_lock); 5009 5010 /** 5011 * yield - yield the current processor to other threads. 5012 * 5013 * Do not ever use this function, there's a 99% chance you're doing it wrong. 5014 * 5015 * The scheduler is at all times free to pick the calling task as the most 5016 * eligible task to run, if removing the yield() call from your code breaks 5017 * it, its already broken. 5018 * 5019 * Typical broken usage is: 5020 * 5021 * while (!event) 5022 * yield(); 5023 * 5024 * where one assumes that yield() will let 'the other' process run that will 5025 * make event true. If the current task is a SCHED_FIFO task that will never 5026 * happen. Never use yield() as a progress guarantee!! 5027 * 5028 * If you want to use yield() to wait for something, use wait_event(). 5029 * If you want to use yield() to be 'nice' for others, use cond_resched(). 5030 * If you still want to use yield(), do not! 5031 */ 5032 void __sched yield(void) 5033 { 5034 set_current_state(TASK_RUNNING); 5035 do_sched_yield(); 5036 } 5037 EXPORT_SYMBOL(yield); 5038 5039 /** 5040 * yield_to - yield the current processor to another thread in 5041 * your thread group, or accelerate that thread toward the 5042 * processor it's on. 5043 * @p: target task 5044 * @preempt: whether task preemption is allowed or not 5045 * 5046 * It's the caller's job to ensure that the target task struct 5047 * can't go away on us before we can do any checks. 5048 * 5049 * Return: 5050 * true (>0) if we indeed boosted the target task. 5051 * false (0) if we failed to boost the target. 5052 * -ESRCH if there's no task to yield to. 5053 */ 5054 int __sched yield_to(struct task_struct *p, bool preempt) 5055 { 5056 struct task_struct *curr = current; 5057 struct rq *rq, *p_rq; 5058 unsigned long flags; 5059 int yielded = 0; 5060 5061 local_irq_save(flags); 5062 rq = this_rq(); 5063 5064 again: 5065 p_rq = task_rq(p); 5066 /* 5067 * If we're the only runnable task on the rq and target rq also 5068 * has only one task, there's absolutely no point in yielding. 5069 */ 5070 if (rq->nr_running == 1 && p_rq->nr_running == 1) { 5071 yielded = -ESRCH; 5072 goto out_irq; 5073 } 5074 5075 double_rq_lock(rq, p_rq); 5076 if (task_rq(p) != p_rq) { 5077 double_rq_unlock(rq, p_rq); 5078 goto again; 5079 } 5080 5081 if (!curr->sched_class->yield_to_task) 5082 goto out_unlock; 5083 5084 if (curr->sched_class != p->sched_class) 5085 goto out_unlock; 5086 5087 if (task_running(p_rq, p) || p->state) 5088 goto out_unlock; 5089 5090 yielded = curr->sched_class->yield_to_task(rq, p, preempt); 5091 if (yielded) { 5092 schedstat_inc(rq->yld_count); 5093 /* 5094 * Make p's CPU reschedule; pick_next_entity takes care of 5095 * fairness. 5096 */ 5097 if (preempt && rq != p_rq) 5098 resched_curr(p_rq); 5099 } 5100 5101 out_unlock: 5102 double_rq_unlock(rq, p_rq); 5103 out_irq: 5104 local_irq_restore(flags); 5105 5106 if (yielded > 0) 5107 schedule(); 5108 5109 return yielded; 5110 } 5111 EXPORT_SYMBOL_GPL(yield_to); 5112 5113 int io_schedule_prepare(void) 5114 { 5115 int old_iowait = current->in_iowait; 5116 5117 current->in_iowait = 1; 5118 blk_schedule_flush_plug(current); 5119 5120 return old_iowait; 5121 } 5122 5123 void io_schedule_finish(int token) 5124 { 5125 current->in_iowait = token; 5126 } 5127 5128 /* 5129 * This task is about to go to sleep on IO. Increment rq->nr_iowait so 5130 * that process accounting knows that this is a task in IO wait state. 5131 */ 5132 long __sched io_schedule_timeout(long timeout) 5133 { 5134 int token; 5135 long ret; 5136 5137 token = io_schedule_prepare(); 5138 ret = schedule_timeout(timeout); 5139 io_schedule_finish(token); 5140 5141 return ret; 5142 } 5143 EXPORT_SYMBOL(io_schedule_timeout); 5144 5145 void io_schedule(void) 5146 { 5147 int token; 5148 5149 token = io_schedule_prepare(); 5150 schedule(); 5151 io_schedule_finish(token); 5152 } 5153 EXPORT_SYMBOL(io_schedule); 5154 5155 /** 5156 * sys_sched_get_priority_max - return maximum RT priority. 5157 * @policy: scheduling class. 5158 * 5159 * Return: On success, this syscall returns the maximum 5160 * rt_priority that can be used by a given scheduling class. 5161 * On failure, a negative error code is returned. 5162 */ 5163 SYSCALL_DEFINE1(sched_get_priority_max, int, policy) 5164 { 5165 int ret = -EINVAL; 5166 5167 switch (policy) { 5168 case SCHED_FIFO: 5169 case SCHED_RR: 5170 ret = MAX_USER_RT_PRIO-1; 5171 break; 5172 case SCHED_DEADLINE: 5173 case SCHED_NORMAL: 5174 case SCHED_BATCH: 5175 case SCHED_IDLE: 5176 ret = 0; 5177 break; 5178 } 5179 return ret; 5180 } 5181 5182 /** 5183 * sys_sched_get_priority_min - return minimum RT priority. 5184 * @policy: scheduling class. 5185 * 5186 * Return: On success, this syscall returns the minimum 5187 * rt_priority that can be used by a given scheduling class. 5188 * On failure, a negative error code is returned. 5189 */ 5190 SYSCALL_DEFINE1(sched_get_priority_min, int, policy) 5191 { 5192 int ret = -EINVAL; 5193 5194 switch (policy) { 5195 case SCHED_FIFO: 5196 case SCHED_RR: 5197 ret = 1; 5198 break; 5199 case SCHED_DEADLINE: 5200 case SCHED_NORMAL: 5201 case SCHED_BATCH: 5202 case SCHED_IDLE: 5203 ret = 0; 5204 } 5205 return ret; 5206 } 5207 5208 static int sched_rr_get_interval(pid_t pid, struct timespec64 *t) 5209 { 5210 struct task_struct *p; 5211 unsigned int time_slice; 5212 struct rq_flags rf; 5213 struct rq *rq; 5214 int retval; 5215 5216 if (pid < 0) 5217 return -EINVAL; 5218 5219 retval = -ESRCH; 5220 rcu_read_lock(); 5221 p = find_process_by_pid(pid); 5222 if (!p) 5223 goto out_unlock; 5224 5225 retval = security_task_getscheduler(p); 5226 if (retval) 5227 goto out_unlock; 5228 5229 rq = task_rq_lock(p, &rf); 5230 time_slice = 0; 5231 if (p->sched_class->get_rr_interval) 5232 time_slice = p->sched_class->get_rr_interval(rq, p); 5233 task_rq_unlock(rq, p, &rf); 5234 5235 rcu_read_unlock(); 5236 jiffies_to_timespec64(time_slice, t); 5237 return 0; 5238 5239 out_unlock: 5240 rcu_read_unlock(); 5241 return retval; 5242 } 5243 5244 /** 5245 * sys_sched_rr_get_interval - return the default timeslice of a process. 5246 * @pid: pid of the process. 5247 * @interval: userspace pointer to the timeslice value. 5248 * 5249 * this syscall writes the default timeslice value of a given process 5250 * into the user-space timespec buffer. A value of '0' means infinity. 5251 * 5252 * Return: On success, 0 and the timeslice is in @interval. Otherwise, 5253 * an error code. 5254 */ 5255 SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid, 5256 struct __kernel_timespec __user *, interval) 5257 { 5258 struct timespec64 t; 5259 int retval = sched_rr_get_interval(pid, &t); 5260 5261 if (retval == 0) 5262 retval = put_timespec64(&t, interval); 5263 5264 return retval; 5265 } 5266 5267 #ifdef CONFIG_COMPAT_32BIT_TIME 5268 COMPAT_SYSCALL_DEFINE2(sched_rr_get_interval, 5269 compat_pid_t, pid, 5270 struct old_timespec32 __user *, interval) 5271 { 5272 struct timespec64 t; 5273 int retval = sched_rr_get_interval(pid, &t); 5274 5275 if (retval == 0) 5276 retval = put_old_timespec32(&t, interval); 5277 return retval; 5278 } 5279 #endif 5280 5281 void sched_show_task(struct task_struct *p) 5282 { 5283 unsigned long free = 0; 5284 int ppid; 5285 5286 if (!try_get_task_stack(p)) 5287 return; 5288 5289 printk(KERN_INFO "%-15.15s %c", p->comm, task_state_to_char(p)); 5290 5291 if (p->state == TASK_RUNNING) 5292 printk(KERN_CONT " running task "); 5293 #ifdef CONFIG_DEBUG_STACK_USAGE 5294 free = stack_not_used(p); 5295 #endif 5296 ppid = 0; 5297 rcu_read_lock(); 5298 if (pid_alive(p)) 5299 ppid = task_pid_nr(rcu_dereference(p->real_parent)); 5300 rcu_read_unlock(); 5301 printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free, 5302 task_pid_nr(p), ppid, 5303 (unsigned long)task_thread_info(p)->flags); 5304 5305 print_worker_info(KERN_INFO, p); 5306 show_stack(p, NULL); 5307 put_task_stack(p); 5308 } 5309 EXPORT_SYMBOL_GPL(sched_show_task); 5310 5311 static inline bool 5312 state_filter_match(unsigned long state_filter, struct task_struct *p) 5313 { 5314 /* no filter, everything matches */ 5315 if (!state_filter) 5316 return true; 5317 5318 /* filter, but doesn't match */ 5319 if (!(p->state & state_filter)) 5320 return false; 5321 5322 /* 5323 * When looking for TASK_UNINTERRUPTIBLE skip TASK_IDLE (allows 5324 * TASK_KILLABLE). 5325 */ 5326 if (state_filter == TASK_UNINTERRUPTIBLE && p->state == TASK_IDLE) 5327 return false; 5328 5329 return true; 5330 } 5331 5332 5333 void show_state_filter(unsigned long state_filter) 5334 { 5335 struct task_struct *g, *p; 5336 5337 #if BITS_PER_LONG == 32 5338 printk(KERN_INFO 5339 " task PC stack pid father\n"); 5340 #else 5341 printk(KERN_INFO 5342 " task PC stack pid father\n"); 5343 #endif 5344 rcu_read_lock(); 5345 for_each_process_thread(g, p) { 5346 /* 5347 * reset the NMI-timeout, listing all files on a slow 5348 * console might take a lot of time: 5349 * Also, reset softlockup watchdogs on all CPUs, because 5350 * another CPU might be blocked waiting for us to process 5351 * an IPI. 5352 */ 5353 touch_nmi_watchdog(); 5354 touch_all_softlockup_watchdogs(); 5355 if (state_filter_match(state_filter, p)) 5356 sched_show_task(p); 5357 } 5358 5359 #ifdef CONFIG_SCHED_DEBUG 5360 if (!state_filter) 5361 sysrq_sched_debug_show(); 5362 #endif 5363 rcu_read_unlock(); 5364 /* 5365 * Only show locks if all tasks are dumped: 5366 */ 5367 if (!state_filter) 5368 debug_show_all_locks(); 5369 } 5370 5371 /** 5372 * init_idle - set up an idle thread for a given CPU 5373 * @idle: task in question 5374 * @cpu: CPU the idle task belongs to 5375 * 5376 * NOTE: this function does not set the idle thread's NEED_RESCHED 5377 * flag, to make booting more robust. 5378 */ 5379 void init_idle(struct task_struct *idle, int cpu) 5380 { 5381 struct rq *rq = cpu_rq(cpu); 5382 unsigned long flags; 5383 5384 raw_spin_lock_irqsave(&idle->pi_lock, flags); 5385 raw_spin_lock(&rq->lock); 5386 5387 __sched_fork(0, idle); 5388 idle->state = TASK_RUNNING; 5389 idle->se.exec_start = sched_clock(); 5390 idle->flags |= PF_IDLE; 5391 5392 kasan_unpoison_task_stack(idle); 5393 5394 #ifdef CONFIG_SMP 5395 /* 5396 * Its possible that init_idle() gets called multiple times on a task, 5397 * in that case do_set_cpus_allowed() will not do the right thing. 5398 * 5399 * And since this is boot we can forgo the serialization. 5400 */ 5401 set_cpus_allowed_common(idle, cpumask_of(cpu)); 5402 #endif 5403 /* 5404 * We're having a chicken and egg problem, even though we are 5405 * holding rq->lock, the CPU isn't yet set to this CPU so the 5406 * lockdep check in task_group() will fail. 5407 * 5408 * Similar case to sched_fork(). / Alternatively we could 5409 * use task_rq_lock() here and obtain the other rq->lock. 5410 * 5411 * Silence PROVE_RCU 5412 */ 5413 rcu_read_lock(); 5414 __set_task_cpu(idle, cpu); 5415 rcu_read_unlock(); 5416 5417 rq->curr = rq->idle = idle; 5418 idle->on_rq = TASK_ON_RQ_QUEUED; 5419 #ifdef CONFIG_SMP 5420 idle->on_cpu = 1; 5421 #endif 5422 raw_spin_unlock(&rq->lock); 5423 raw_spin_unlock_irqrestore(&idle->pi_lock, flags); 5424 5425 /* Set the preempt count _outside_ the spinlocks! */ 5426 init_idle_preempt_count(idle, cpu); 5427 5428 /* 5429 * The idle tasks have their own, simple scheduling class: 5430 */ 5431 idle->sched_class = &idle_sched_class; 5432 ftrace_graph_init_idle_task(idle, cpu); 5433 vtime_init_idle(idle, cpu); 5434 #ifdef CONFIG_SMP 5435 sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu); 5436 #endif 5437 } 5438 5439 #ifdef CONFIG_SMP 5440 5441 int cpuset_cpumask_can_shrink(const struct cpumask *cur, 5442 const struct cpumask *trial) 5443 { 5444 int ret = 1; 5445 5446 if (!cpumask_weight(cur)) 5447 return ret; 5448 5449 ret = dl_cpuset_cpumask_can_shrink(cur, trial); 5450 5451 return ret; 5452 } 5453 5454 int task_can_attach(struct task_struct *p, 5455 const struct cpumask *cs_cpus_allowed) 5456 { 5457 int ret = 0; 5458 5459 /* 5460 * Kthreads which disallow setaffinity shouldn't be moved 5461 * to a new cpuset; we don't want to change their CPU 5462 * affinity and isolating such threads by their set of 5463 * allowed nodes is unnecessary. Thus, cpusets are not 5464 * applicable for such threads. This prevents checking for 5465 * success of set_cpus_allowed_ptr() on all attached tasks 5466 * before cpus_allowed may be changed. 5467 */ 5468 if (p->flags & PF_NO_SETAFFINITY) { 5469 ret = -EINVAL; 5470 goto out; 5471 } 5472 5473 if (dl_task(p) && !cpumask_intersects(task_rq(p)->rd->span, 5474 cs_cpus_allowed)) 5475 ret = dl_task_can_attach(p, cs_cpus_allowed); 5476 5477 out: 5478 return ret; 5479 } 5480 5481 bool sched_smp_initialized __read_mostly; 5482 5483 #ifdef CONFIG_NUMA_BALANCING 5484 /* Migrate current task p to target_cpu */ 5485 int migrate_task_to(struct task_struct *p, int target_cpu) 5486 { 5487 struct migration_arg arg = { p, target_cpu }; 5488 int curr_cpu = task_cpu(p); 5489 5490 if (curr_cpu == target_cpu) 5491 return 0; 5492 5493 if (!cpumask_test_cpu(target_cpu, &p->cpus_allowed)) 5494 return -EINVAL; 5495 5496 /* TODO: This is not properly updating schedstats */ 5497 5498 trace_sched_move_numa(p, curr_cpu, target_cpu); 5499 return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg); 5500 } 5501 5502 /* 5503 * Requeue a task on a given node and accurately track the number of NUMA 5504 * tasks on the runqueues 5505 */ 5506 void sched_setnuma(struct task_struct *p, int nid) 5507 { 5508 bool queued, running; 5509 struct rq_flags rf; 5510 struct rq *rq; 5511 5512 rq = task_rq_lock(p, &rf); 5513 queued = task_on_rq_queued(p); 5514 running = task_current(rq, p); 5515 5516 if (queued) 5517 dequeue_task(rq, p, DEQUEUE_SAVE); 5518 if (running) 5519 put_prev_task(rq, p); 5520 5521 p->numa_preferred_nid = nid; 5522 5523 if (queued) 5524 enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK); 5525 if (running) 5526 set_curr_task(rq, p); 5527 task_rq_unlock(rq, p, &rf); 5528 } 5529 #endif /* CONFIG_NUMA_BALANCING */ 5530 5531 #ifdef CONFIG_HOTPLUG_CPU 5532 /* 5533 * Ensure that the idle task is using init_mm right before its CPU goes 5534 * offline. 5535 */ 5536 void idle_task_exit(void) 5537 { 5538 struct mm_struct *mm = current->active_mm; 5539 5540 BUG_ON(cpu_online(smp_processor_id())); 5541 5542 if (mm != &init_mm) { 5543 switch_mm(mm, &init_mm, current); 5544 current->active_mm = &init_mm; 5545 finish_arch_post_lock_switch(); 5546 } 5547 mmdrop(mm); 5548 } 5549 5550 /* 5551 * Since this CPU is going 'away' for a while, fold any nr_active delta 5552 * we might have. Assumes we're called after migrate_tasks() so that the 5553 * nr_active count is stable. We need to take the teardown thread which 5554 * is calling this into account, so we hand in adjust = 1 to the load 5555 * calculation. 5556 * 5557 * Also see the comment "Global load-average calculations". 5558 */ 5559 static void calc_load_migrate(struct rq *rq) 5560 { 5561 long delta = calc_load_fold_active(rq, 1); 5562 if (delta) 5563 atomic_long_add(delta, &calc_load_tasks); 5564 } 5565 5566 static void put_prev_task_fake(struct rq *rq, struct task_struct *prev) 5567 { 5568 } 5569 5570 static const struct sched_class fake_sched_class = { 5571 .put_prev_task = put_prev_task_fake, 5572 }; 5573 5574 static struct task_struct fake_task = { 5575 /* 5576 * Avoid pull_{rt,dl}_task() 5577 */ 5578 .prio = MAX_PRIO + 1, 5579 .sched_class = &fake_sched_class, 5580 }; 5581 5582 /* 5583 * Migrate all tasks from the rq, sleeping tasks will be migrated by 5584 * try_to_wake_up()->select_task_rq(). 5585 * 5586 * Called with rq->lock held even though we'er in stop_machine() and 5587 * there's no concurrency possible, we hold the required locks anyway 5588 * because of lock validation efforts. 5589 */ 5590 static void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf) 5591 { 5592 struct rq *rq = dead_rq; 5593 struct task_struct *next, *stop = rq->stop; 5594 struct rq_flags orf = *rf; 5595 int dest_cpu; 5596 5597 /* 5598 * Fudge the rq selection such that the below task selection loop 5599 * doesn't get stuck on the currently eligible stop task. 5600 * 5601 * We're currently inside stop_machine() and the rq is either stuck 5602 * in the stop_machine_cpu_stop() loop, or we're executing this code, 5603 * either way we should never end up calling schedule() until we're 5604 * done here. 5605 */ 5606 rq->stop = NULL; 5607 5608 /* 5609 * put_prev_task() and pick_next_task() sched 5610 * class method both need to have an up-to-date 5611 * value of rq->clock[_task] 5612 */ 5613 update_rq_clock(rq); 5614 5615 for (;;) { 5616 /* 5617 * There's this thread running, bail when that's the only 5618 * remaining thread: 5619 */ 5620 if (rq->nr_running == 1) 5621 break; 5622 5623 /* 5624 * pick_next_task() assumes pinned rq->lock: 5625 */ 5626 next = pick_next_task(rq, &fake_task, rf); 5627 BUG_ON(!next); 5628 put_prev_task(rq, next); 5629 5630 /* 5631 * Rules for changing task_struct::cpus_allowed are holding 5632 * both pi_lock and rq->lock, such that holding either 5633 * stabilizes the mask. 5634 * 5635 * Drop rq->lock is not quite as disastrous as it usually is 5636 * because !cpu_active at this point, which means load-balance 5637 * will not interfere. Also, stop-machine. 5638 */ 5639 rq_unlock(rq, rf); 5640 raw_spin_lock(&next->pi_lock); 5641 rq_relock(rq, rf); 5642 5643 /* 5644 * Since we're inside stop-machine, _nothing_ should have 5645 * changed the task, WARN if weird stuff happened, because in 5646 * that case the above rq->lock drop is a fail too. 5647 */ 5648 if (WARN_ON(task_rq(next) != rq || !task_on_rq_queued(next))) { 5649 raw_spin_unlock(&next->pi_lock); 5650 continue; 5651 } 5652 5653 /* Find suitable destination for @next, with force if needed. */ 5654 dest_cpu = select_fallback_rq(dead_rq->cpu, next); 5655 rq = __migrate_task(rq, rf, next, dest_cpu); 5656 if (rq != dead_rq) { 5657 rq_unlock(rq, rf); 5658 rq = dead_rq; 5659 *rf = orf; 5660 rq_relock(rq, rf); 5661 } 5662 raw_spin_unlock(&next->pi_lock); 5663 } 5664 5665 rq->stop = stop; 5666 } 5667 #endif /* CONFIG_HOTPLUG_CPU */ 5668 5669 void set_rq_online(struct rq *rq) 5670 { 5671 if (!rq->online) { 5672 const struct sched_class *class; 5673 5674 cpumask_set_cpu(rq->cpu, rq->rd->online); 5675 rq->online = 1; 5676 5677 for_each_class(class) { 5678 if (class->rq_online) 5679 class->rq_online(rq); 5680 } 5681 } 5682 } 5683 5684 void set_rq_offline(struct rq *rq) 5685 { 5686 if (rq->online) { 5687 const struct sched_class *class; 5688 5689 for_each_class(class) { 5690 if (class->rq_offline) 5691 class->rq_offline(rq); 5692 } 5693 5694 cpumask_clear_cpu(rq->cpu, rq->rd->online); 5695 rq->online = 0; 5696 } 5697 } 5698 5699 /* 5700 * used to mark begin/end of suspend/resume: 5701 */ 5702 static int num_cpus_frozen; 5703 5704 /* 5705 * Update cpusets according to cpu_active mask. If cpusets are 5706 * disabled, cpuset_update_active_cpus() becomes a simple wrapper 5707 * around partition_sched_domains(). 5708 * 5709 * If we come here as part of a suspend/resume, don't touch cpusets because we 5710 * want to restore it back to its original state upon resume anyway. 5711 */ 5712 static void cpuset_cpu_active(void) 5713 { 5714 if (cpuhp_tasks_frozen) { 5715 /* 5716 * num_cpus_frozen tracks how many CPUs are involved in suspend 5717 * resume sequence. As long as this is not the last online 5718 * operation in the resume sequence, just build a single sched 5719 * domain, ignoring cpusets. 5720 */ 5721 partition_sched_domains(1, NULL, NULL); 5722 if (--num_cpus_frozen) 5723 return; 5724 /* 5725 * This is the last CPU online operation. So fall through and 5726 * restore the original sched domains by considering the 5727 * cpuset configurations. 5728 */ 5729 cpuset_force_rebuild(); 5730 } 5731 cpuset_update_active_cpus(); 5732 } 5733 5734 static int cpuset_cpu_inactive(unsigned int cpu) 5735 { 5736 if (!cpuhp_tasks_frozen) { 5737 if (dl_cpu_busy(cpu)) 5738 return -EBUSY; 5739 cpuset_update_active_cpus(); 5740 } else { 5741 num_cpus_frozen++; 5742 partition_sched_domains(1, NULL, NULL); 5743 } 5744 return 0; 5745 } 5746 5747 int sched_cpu_activate(unsigned int cpu) 5748 { 5749 struct rq *rq = cpu_rq(cpu); 5750 struct rq_flags rf; 5751 5752 #ifdef CONFIG_SCHED_SMT 5753 /* 5754 * When going up, increment the number of cores with SMT present. 5755 */ 5756 if (cpumask_weight(cpu_smt_mask(cpu)) == 2) 5757 static_branch_inc_cpuslocked(&sched_smt_present); 5758 #endif 5759 set_cpu_active(cpu, true); 5760 5761 if (sched_smp_initialized) { 5762 sched_domains_numa_masks_set(cpu); 5763 cpuset_cpu_active(); 5764 } 5765 5766 /* 5767 * Put the rq online, if not already. This happens: 5768 * 5769 * 1) In the early boot process, because we build the real domains 5770 * after all CPUs have been brought up. 5771 * 5772 * 2) At runtime, if cpuset_cpu_active() fails to rebuild the 5773 * domains. 5774 */ 5775 rq_lock_irqsave(rq, &rf); 5776 if (rq->rd) { 5777 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); 5778 set_rq_online(rq); 5779 } 5780 rq_unlock_irqrestore(rq, &rf); 5781 5782 update_max_interval(); 5783 5784 return 0; 5785 } 5786 5787 int sched_cpu_deactivate(unsigned int cpu) 5788 { 5789 int ret; 5790 5791 set_cpu_active(cpu, false); 5792 /* 5793 * We've cleared cpu_active_mask, wait for all preempt-disabled and RCU 5794 * users of this state to go away such that all new such users will 5795 * observe it. 5796 * 5797 * Do sync before park smpboot threads to take care the rcu boost case. 5798 */ 5799 synchronize_rcu(); 5800 5801 #ifdef CONFIG_SCHED_SMT 5802 /* 5803 * When going down, decrement the number of cores with SMT present. 5804 */ 5805 if (cpumask_weight(cpu_smt_mask(cpu)) == 2) 5806 static_branch_dec_cpuslocked(&sched_smt_present); 5807 #endif 5808 5809 if (!sched_smp_initialized) 5810 return 0; 5811 5812 ret = cpuset_cpu_inactive(cpu); 5813 if (ret) { 5814 set_cpu_active(cpu, true); 5815 return ret; 5816 } 5817 sched_domains_numa_masks_clear(cpu); 5818 return 0; 5819 } 5820 5821 static void sched_rq_cpu_starting(unsigned int cpu) 5822 { 5823 struct rq *rq = cpu_rq(cpu); 5824 5825 rq->calc_load_update = calc_load_update; 5826 update_max_interval(); 5827 } 5828 5829 int sched_cpu_starting(unsigned int cpu) 5830 { 5831 sched_rq_cpu_starting(cpu); 5832 sched_tick_start(cpu); 5833 return 0; 5834 } 5835 5836 #ifdef CONFIG_HOTPLUG_CPU 5837 int sched_cpu_dying(unsigned int cpu) 5838 { 5839 struct rq *rq = cpu_rq(cpu); 5840 struct rq_flags rf; 5841 5842 /* Handle pending wakeups and then migrate everything off */ 5843 sched_ttwu_pending(); 5844 sched_tick_stop(cpu); 5845 5846 rq_lock_irqsave(rq, &rf); 5847 if (rq->rd) { 5848 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); 5849 set_rq_offline(rq); 5850 } 5851 migrate_tasks(rq, &rf); 5852 BUG_ON(rq->nr_running != 1); 5853 rq_unlock_irqrestore(rq, &rf); 5854 5855 calc_load_migrate(rq); 5856 update_max_interval(); 5857 nohz_balance_exit_idle(rq); 5858 hrtick_clear(rq); 5859 return 0; 5860 } 5861 #endif 5862 5863 void __init sched_init_smp(void) 5864 { 5865 sched_init_numa(); 5866 5867 /* 5868 * There's no userspace yet to cause hotplug operations; hence all the 5869 * CPU masks are stable and all blatant races in the below code cannot 5870 * happen. The hotplug lock is nevertheless taken to satisfy lockdep, 5871 * but there won't be any contention on it. 5872 */ 5873 cpus_read_lock(); 5874 mutex_lock(&sched_domains_mutex); 5875 sched_init_domains(cpu_active_mask); 5876 mutex_unlock(&sched_domains_mutex); 5877 cpus_read_unlock(); 5878 5879 /* Move init over to a non-isolated CPU */ 5880 if (set_cpus_allowed_ptr(current, housekeeping_cpumask(HK_FLAG_DOMAIN)) < 0) 5881 BUG(); 5882 sched_init_granularity(); 5883 5884 init_sched_rt_class(); 5885 init_sched_dl_class(); 5886 5887 sched_smp_initialized = true; 5888 } 5889 5890 static int __init migration_init(void) 5891 { 5892 sched_rq_cpu_starting(smp_processor_id()); 5893 return 0; 5894 } 5895 early_initcall(migration_init); 5896 5897 #else 5898 void __init sched_init_smp(void) 5899 { 5900 sched_init_granularity(); 5901 } 5902 #endif /* CONFIG_SMP */ 5903 5904 int in_sched_functions(unsigned long addr) 5905 { 5906 return in_lock_functions(addr) || 5907 (addr >= (unsigned long)__sched_text_start 5908 && addr < (unsigned long)__sched_text_end); 5909 } 5910 5911 #ifdef CONFIG_CGROUP_SCHED 5912 /* 5913 * Default task group. 5914 * Every task in system belongs to this group at bootup. 5915 */ 5916 struct task_group root_task_group; 5917 LIST_HEAD(task_groups); 5918 5919 /* Cacheline aligned slab cache for task_group */ 5920 static struct kmem_cache *task_group_cache __read_mostly; 5921 #endif 5922 5923 DECLARE_PER_CPU(cpumask_var_t, load_balance_mask); 5924 DECLARE_PER_CPU(cpumask_var_t, select_idle_mask); 5925 5926 void __init sched_init(void) 5927 { 5928 int i, j; 5929 unsigned long alloc_size = 0, ptr; 5930 5931 wait_bit_init(); 5932 5933 #ifdef CONFIG_FAIR_GROUP_SCHED 5934 alloc_size += 2 * nr_cpu_ids * sizeof(void **); 5935 #endif 5936 #ifdef CONFIG_RT_GROUP_SCHED 5937 alloc_size += 2 * nr_cpu_ids * sizeof(void **); 5938 #endif 5939 if (alloc_size) { 5940 ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT); 5941 5942 #ifdef CONFIG_FAIR_GROUP_SCHED 5943 root_task_group.se = (struct sched_entity **)ptr; 5944 ptr += nr_cpu_ids * sizeof(void **); 5945 5946 root_task_group.cfs_rq = (struct cfs_rq **)ptr; 5947 ptr += nr_cpu_ids * sizeof(void **); 5948 5949 #endif /* CONFIG_FAIR_GROUP_SCHED */ 5950 #ifdef CONFIG_RT_GROUP_SCHED 5951 root_task_group.rt_se = (struct sched_rt_entity **)ptr; 5952 ptr += nr_cpu_ids * sizeof(void **); 5953 5954 root_task_group.rt_rq = (struct rt_rq **)ptr; 5955 ptr += nr_cpu_ids * sizeof(void **); 5956 5957 #endif /* CONFIG_RT_GROUP_SCHED */ 5958 } 5959 #ifdef CONFIG_CPUMASK_OFFSTACK 5960 for_each_possible_cpu(i) { 5961 per_cpu(load_balance_mask, i) = (cpumask_var_t)kzalloc_node( 5962 cpumask_size(), GFP_KERNEL, cpu_to_node(i)); 5963 per_cpu(select_idle_mask, i) = (cpumask_var_t)kzalloc_node( 5964 cpumask_size(), GFP_KERNEL, cpu_to_node(i)); 5965 } 5966 #endif /* CONFIG_CPUMASK_OFFSTACK */ 5967 5968 init_rt_bandwidth(&def_rt_bandwidth, global_rt_period(), global_rt_runtime()); 5969 init_dl_bandwidth(&def_dl_bandwidth, global_rt_period(), global_rt_runtime()); 5970 5971 #ifdef CONFIG_SMP 5972 init_defrootdomain(); 5973 #endif 5974 5975 #ifdef CONFIG_RT_GROUP_SCHED 5976 init_rt_bandwidth(&root_task_group.rt_bandwidth, 5977 global_rt_period(), global_rt_runtime()); 5978 #endif /* CONFIG_RT_GROUP_SCHED */ 5979 5980 #ifdef CONFIG_CGROUP_SCHED 5981 task_group_cache = KMEM_CACHE(task_group, 0); 5982 5983 list_add(&root_task_group.list, &task_groups); 5984 INIT_LIST_HEAD(&root_task_group.children); 5985 INIT_LIST_HEAD(&root_task_group.siblings); 5986 autogroup_init(&init_task); 5987 #endif /* CONFIG_CGROUP_SCHED */ 5988 5989 for_each_possible_cpu(i) { 5990 struct rq *rq; 5991 5992 rq = cpu_rq(i); 5993 raw_spin_lock_init(&rq->lock); 5994 rq->nr_running = 0; 5995 rq->calc_load_active = 0; 5996 rq->calc_load_update = jiffies + LOAD_FREQ; 5997 init_cfs_rq(&rq->cfs); 5998 init_rt_rq(&rq->rt); 5999 init_dl_rq(&rq->dl); 6000 #ifdef CONFIG_FAIR_GROUP_SCHED 6001 root_task_group.shares = ROOT_TASK_GROUP_LOAD; 6002 INIT_LIST_HEAD(&rq->leaf_cfs_rq_list); 6003 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 6004 /* 6005 * How much CPU bandwidth does root_task_group get? 6006 * 6007 * In case of task-groups formed thr' the cgroup filesystem, it 6008 * gets 100% of the CPU resources in the system. This overall 6009 * system CPU resource is divided among the tasks of 6010 * root_task_group and its child task-groups in a fair manner, 6011 * based on each entity's (task or task-group's) weight 6012 * (se->load.weight). 6013 * 6014 * In other words, if root_task_group has 10 tasks of weight 6015 * 1024) and two child groups A0 and A1 (of weight 1024 each), 6016 * then A0's share of the CPU resource is: 6017 * 6018 * A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33% 6019 * 6020 * We achieve this by letting root_task_group's tasks sit 6021 * directly in rq->cfs (i.e root_task_group->se[] = NULL). 6022 */ 6023 init_cfs_bandwidth(&root_task_group.cfs_bandwidth); 6024 init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL); 6025 #endif /* CONFIG_FAIR_GROUP_SCHED */ 6026 6027 rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime; 6028 #ifdef CONFIG_RT_GROUP_SCHED 6029 init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL); 6030 #endif 6031 6032 for (j = 0; j < CPU_LOAD_IDX_MAX; j++) 6033 rq->cpu_load[j] = 0; 6034 6035 #ifdef CONFIG_SMP 6036 rq->sd = NULL; 6037 rq->rd = NULL; 6038 rq->cpu_capacity = rq->cpu_capacity_orig = SCHED_CAPACITY_SCALE; 6039 rq->balance_callback = NULL; 6040 rq->active_balance = 0; 6041 rq->next_balance = jiffies; 6042 rq->push_cpu = 0; 6043 rq->cpu = i; 6044 rq->online = 0; 6045 rq->idle_stamp = 0; 6046 rq->avg_idle = 2*sysctl_sched_migration_cost; 6047 rq->max_idle_balance_cost = sysctl_sched_migration_cost; 6048 6049 INIT_LIST_HEAD(&rq->cfs_tasks); 6050 6051 rq_attach_root(rq, &def_root_domain); 6052 #ifdef CONFIG_NO_HZ_COMMON 6053 rq->last_load_update_tick = jiffies; 6054 rq->last_blocked_load_update_tick = jiffies; 6055 atomic_set(&rq->nohz_flags, 0); 6056 #endif 6057 #endif /* CONFIG_SMP */ 6058 hrtick_rq_init(rq); 6059 atomic_set(&rq->nr_iowait, 0); 6060 } 6061 6062 set_load_weight(&init_task, false); 6063 6064 /* 6065 * The boot idle thread does lazy MMU switching as well: 6066 */ 6067 mmgrab(&init_mm); 6068 enter_lazy_tlb(&init_mm, current); 6069 6070 /* 6071 * Make us the idle thread. Technically, schedule() should not be 6072 * called from this thread, however somewhere below it might be, 6073 * but because we are the idle thread, we just pick up running again 6074 * when this runqueue becomes "idle". 6075 */ 6076 init_idle(current, smp_processor_id()); 6077 6078 calc_load_update = jiffies + LOAD_FREQ; 6079 6080 #ifdef CONFIG_SMP 6081 idle_thread_set_boot_cpu(); 6082 #endif 6083 init_sched_fair_class(); 6084 6085 init_schedstats(); 6086 6087 psi_init(); 6088 6089 scheduler_running = 1; 6090 } 6091 6092 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP 6093 static inline int preempt_count_equals(int preempt_offset) 6094 { 6095 int nested = preempt_count() + rcu_preempt_depth(); 6096 6097 return (nested == preempt_offset); 6098 } 6099 6100 void __might_sleep(const char *file, int line, int preempt_offset) 6101 { 6102 /* 6103 * Blocking primitives will set (and therefore destroy) current->state, 6104 * since we will exit with TASK_RUNNING make sure we enter with it, 6105 * otherwise we will destroy state. 6106 */ 6107 WARN_ONCE(current->state != TASK_RUNNING && current->task_state_change, 6108 "do not call blocking ops when !TASK_RUNNING; " 6109 "state=%lx set at [<%p>] %pS\n", 6110 current->state, 6111 (void *)current->task_state_change, 6112 (void *)current->task_state_change); 6113 6114 ___might_sleep(file, line, preempt_offset); 6115 } 6116 EXPORT_SYMBOL(__might_sleep); 6117 6118 void ___might_sleep(const char *file, int line, int preempt_offset) 6119 { 6120 /* Ratelimiting timestamp: */ 6121 static unsigned long prev_jiffy; 6122 6123 unsigned long preempt_disable_ip; 6124 6125 /* WARN_ON_ONCE() by default, no rate limit required: */ 6126 rcu_sleep_check(); 6127 6128 if ((preempt_count_equals(preempt_offset) && !irqs_disabled() && 6129 !is_idle_task(current)) || 6130 system_state == SYSTEM_BOOTING || system_state > SYSTEM_RUNNING || 6131 oops_in_progress) 6132 return; 6133 6134 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) 6135 return; 6136 prev_jiffy = jiffies; 6137 6138 /* Save this before calling printk(), since that will clobber it: */ 6139 preempt_disable_ip = get_preempt_disable_ip(current); 6140 6141 printk(KERN_ERR 6142 "BUG: sleeping function called from invalid context at %s:%d\n", 6143 file, line); 6144 printk(KERN_ERR 6145 "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n", 6146 in_atomic(), irqs_disabled(), 6147 current->pid, current->comm); 6148 6149 if (task_stack_end_corrupted(current)) 6150 printk(KERN_EMERG "Thread overran stack, or stack corrupted\n"); 6151 6152 debug_show_held_locks(current); 6153 if (irqs_disabled()) 6154 print_irqtrace_events(current); 6155 if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) 6156 && !preempt_count_equals(preempt_offset)) { 6157 pr_err("Preemption disabled at:"); 6158 print_ip_sym(preempt_disable_ip); 6159 pr_cont("\n"); 6160 } 6161 dump_stack(); 6162 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 6163 } 6164 EXPORT_SYMBOL(___might_sleep); 6165 #endif 6166 6167 #ifdef CONFIG_MAGIC_SYSRQ 6168 void normalize_rt_tasks(void) 6169 { 6170 struct task_struct *g, *p; 6171 struct sched_attr attr = { 6172 .sched_policy = SCHED_NORMAL, 6173 }; 6174 6175 read_lock(&tasklist_lock); 6176 for_each_process_thread(g, p) { 6177 /* 6178 * Only normalize user tasks: 6179 */ 6180 if (p->flags & PF_KTHREAD) 6181 continue; 6182 6183 p->se.exec_start = 0; 6184 schedstat_set(p->se.statistics.wait_start, 0); 6185 schedstat_set(p->se.statistics.sleep_start, 0); 6186 schedstat_set(p->se.statistics.block_start, 0); 6187 6188 if (!dl_task(p) && !rt_task(p)) { 6189 /* 6190 * Renice negative nice level userspace 6191 * tasks back to 0: 6192 */ 6193 if (task_nice(p) < 0) 6194 set_user_nice(p, 0); 6195 continue; 6196 } 6197 6198 __sched_setscheduler(p, &attr, false, false); 6199 } 6200 read_unlock(&tasklist_lock); 6201 } 6202 6203 #endif /* CONFIG_MAGIC_SYSRQ */ 6204 6205 #if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) 6206 /* 6207 * These functions are only useful for the IA64 MCA handling, or kdb. 6208 * 6209 * They can only be called when the whole system has been 6210 * stopped - every CPU needs to be quiescent, and no scheduling 6211 * activity can take place. Using them for anything else would 6212 * be a serious bug, and as a result, they aren't even visible 6213 * under any other configuration. 6214 */ 6215 6216 /** 6217 * curr_task - return the current task for a given CPU. 6218 * @cpu: the processor in question. 6219 * 6220 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! 6221 * 6222 * Return: The current task for @cpu. 6223 */ 6224 struct task_struct *curr_task(int cpu) 6225 { 6226 return cpu_curr(cpu); 6227 } 6228 6229 #endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */ 6230 6231 #ifdef CONFIG_IA64 6232 /** 6233 * set_curr_task - set the current task for a given CPU. 6234 * @cpu: the processor in question. 6235 * @p: the task pointer to set. 6236 * 6237 * Description: This function must only be used when non-maskable interrupts 6238 * are serviced on a separate stack. It allows the architecture to switch the 6239 * notion of the current task on a CPU in a non-blocking manner. This function 6240 * must be called with all CPU's synchronized, and interrupts disabled, the 6241 * and caller must save the original value of the current task (see 6242 * curr_task() above) and restore that value before reenabling interrupts and 6243 * re-starting the system. 6244 * 6245 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! 6246 */ 6247 void ia64_set_curr_task(int cpu, struct task_struct *p) 6248 { 6249 cpu_curr(cpu) = p; 6250 } 6251 6252 #endif 6253 6254 #ifdef CONFIG_CGROUP_SCHED 6255 /* task_group_lock serializes the addition/removal of task groups */ 6256 static DEFINE_SPINLOCK(task_group_lock); 6257 6258 static void sched_free_group(struct task_group *tg) 6259 { 6260 free_fair_sched_group(tg); 6261 free_rt_sched_group(tg); 6262 autogroup_free(tg); 6263 kmem_cache_free(task_group_cache, tg); 6264 } 6265 6266 /* allocate runqueue etc for a new task group */ 6267 struct task_group *sched_create_group(struct task_group *parent) 6268 { 6269 struct task_group *tg; 6270 6271 tg = kmem_cache_alloc(task_group_cache, GFP_KERNEL | __GFP_ZERO); 6272 if (!tg) 6273 return ERR_PTR(-ENOMEM); 6274 6275 if (!alloc_fair_sched_group(tg, parent)) 6276 goto err; 6277 6278 if (!alloc_rt_sched_group(tg, parent)) 6279 goto err; 6280 6281 return tg; 6282 6283 err: 6284 sched_free_group(tg); 6285 return ERR_PTR(-ENOMEM); 6286 } 6287 6288 void sched_online_group(struct task_group *tg, struct task_group *parent) 6289 { 6290 unsigned long flags; 6291 6292 spin_lock_irqsave(&task_group_lock, flags); 6293 list_add_rcu(&tg->list, &task_groups); 6294 6295 /* Root should already exist: */ 6296 WARN_ON(!parent); 6297 6298 tg->parent = parent; 6299 INIT_LIST_HEAD(&tg->children); 6300 list_add_rcu(&tg->siblings, &parent->children); 6301 spin_unlock_irqrestore(&task_group_lock, flags); 6302 6303 online_fair_sched_group(tg); 6304 } 6305 6306 /* rcu callback to free various structures associated with a task group */ 6307 static void sched_free_group_rcu(struct rcu_head *rhp) 6308 { 6309 /* Now it should be safe to free those cfs_rqs: */ 6310 sched_free_group(container_of(rhp, struct task_group, rcu)); 6311 } 6312 6313 void sched_destroy_group(struct task_group *tg) 6314 { 6315 /* Wait for possible concurrent references to cfs_rqs complete: */ 6316 call_rcu(&tg->rcu, sched_free_group_rcu); 6317 } 6318 6319 void sched_offline_group(struct task_group *tg) 6320 { 6321 unsigned long flags; 6322 6323 /* End participation in shares distribution: */ 6324 unregister_fair_sched_group(tg); 6325 6326 spin_lock_irqsave(&task_group_lock, flags); 6327 list_del_rcu(&tg->list); 6328 list_del_rcu(&tg->siblings); 6329 spin_unlock_irqrestore(&task_group_lock, flags); 6330 } 6331 6332 static void sched_change_group(struct task_struct *tsk, int type) 6333 { 6334 struct task_group *tg; 6335 6336 /* 6337 * All callers are synchronized by task_rq_lock(); we do not use RCU 6338 * which is pointless here. Thus, we pass "true" to task_css_check() 6339 * to prevent lockdep warnings. 6340 */ 6341 tg = container_of(task_css_check(tsk, cpu_cgrp_id, true), 6342 struct task_group, css); 6343 tg = autogroup_task_group(tsk, tg); 6344 tsk->sched_task_group = tg; 6345 6346 #ifdef CONFIG_FAIR_GROUP_SCHED 6347 if (tsk->sched_class->task_change_group) 6348 tsk->sched_class->task_change_group(tsk, type); 6349 else 6350 #endif 6351 set_task_rq(tsk, task_cpu(tsk)); 6352 } 6353 6354 /* 6355 * Change task's runqueue when it moves between groups. 6356 * 6357 * The caller of this function should have put the task in its new group by 6358 * now. This function just updates tsk->se.cfs_rq and tsk->se.parent to reflect 6359 * its new group. 6360 */ 6361 void sched_move_task(struct task_struct *tsk) 6362 { 6363 int queued, running, queue_flags = 6364 DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; 6365 struct rq_flags rf; 6366 struct rq *rq; 6367 6368 rq = task_rq_lock(tsk, &rf); 6369 update_rq_clock(rq); 6370 6371 running = task_current(rq, tsk); 6372 queued = task_on_rq_queued(tsk); 6373 6374 if (queued) 6375 dequeue_task(rq, tsk, queue_flags); 6376 if (running) 6377 put_prev_task(rq, tsk); 6378 6379 sched_change_group(tsk, TASK_MOVE_GROUP); 6380 6381 if (queued) 6382 enqueue_task(rq, tsk, queue_flags); 6383 if (running) 6384 set_curr_task(rq, tsk); 6385 6386 task_rq_unlock(rq, tsk, &rf); 6387 } 6388 6389 static inline struct task_group *css_tg(struct cgroup_subsys_state *css) 6390 { 6391 return css ? container_of(css, struct task_group, css) : NULL; 6392 } 6393 6394 static struct cgroup_subsys_state * 6395 cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) 6396 { 6397 struct task_group *parent = css_tg(parent_css); 6398 struct task_group *tg; 6399 6400 if (!parent) { 6401 /* This is early initialization for the top cgroup */ 6402 return &root_task_group.css; 6403 } 6404 6405 tg = sched_create_group(parent); 6406 if (IS_ERR(tg)) 6407 return ERR_PTR(-ENOMEM); 6408 6409 return &tg->css; 6410 } 6411 6412 /* Expose task group only after completing cgroup initialization */ 6413 static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) 6414 { 6415 struct task_group *tg = css_tg(css); 6416 struct task_group *parent = css_tg(css->parent); 6417 6418 if (parent) 6419 sched_online_group(tg, parent); 6420 return 0; 6421 } 6422 6423 static void cpu_cgroup_css_released(struct cgroup_subsys_state *css) 6424 { 6425 struct task_group *tg = css_tg(css); 6426 6427 sched_offline_group(tg); 6428 } 6429 6430 static void cpu_cgroup_css_free(struct cgroup_subsys_state *css) 6431 { 6432 struct task_group *tg = css_tg(css); 6433 6434 /* 6435 * Relies on the RCU grace period between css_released() and this. 6436 */ 6437 sched_free_group(tg); 6438 } 6439 6440 /* 6441 * This is called before wake_up_new_task(), therefore we really only 6442 * have to set its group bits, all the other stuff does not apply. 6443 */ 6444 static void cpu_cgroup_fork(struct task_struct *task) 6445 { 6446 struct rq_flags rf; 6447 struct rq *rq; 6448 6449 rq = task_rq_lock(task, &rf); 6450 6451 update_rq_clock(rq); 6452 sched_change_group(task, TASK_SET_GROUP); 6453 6454 task_rq_unlock(rq, task, &rf); 6455 } 6456 6457 static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) 6458 { 6459 struct task_struct *task; 6460 struct cgroup_subsys_state *css; 6461 int ret = 0; 6462 6463 cgroup_taskset_for_each(task, css, tset) { 6464 #ifdef CONFIG_RT_GROUP_SCHED 6465 if (!sched_rt_can_attach(css_tg(css), task)) 6466 return -EINVAL; 6467 #else 6468 /* We don't support RT-tasks being in separate groups */ 6469 if (task->sched_class != &fair_sched_class) 6470 return -EINVAL; 6471 #endif 6472 /* 6473 * Serialize against wake_up_new_task() such that if its 6474 * running, we're sure to observe its full state. 6475 */ 6476 raw_spin_lock_irq(&task->pi_lock); 6477 /* 6478 * Avoid calling sched_move_task() before wake_up_new_task() 6479 * has happened. This would lead to problems with PELT, due to 6480 * move wanting to detach+attach while we're not attached yet. 6481 */ 6482 if (task->state == TASK_NEW) 6483 ret = -EINVAL; 6484 raw_spin_unlock_irq(&task->pi_lock); 6485 6486 if (ret) 6487 break; 6488 } 6489 return ret; 6490 } 6491 6492 static void cpu_cgroup_attach(struct cgroup_taskset *tset) 6493 { 6494 struct task_struct *task; 6495 struct cgroup_subsys_state *css; 6496 6497 cgroup_taskset_for_each(task, css, tset) 6498 sched_move_task(task); 6499 } 6500 6501 #ifdef CONFIG_FAIR_GROUP_SCHED 6502 static int cpu_shares_write_u64(struct cgroup_subsys_state *css, 6503 struct cftype *cftype, u64 shareval) 6504 { 6505 return sched_group_set_shares(css_tg(css), scale_load(shareval)); 6506 } 6507 6508 static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css, 6509 struct cftype *cft) 6510 { 6511 struct task_group *tg = css_tg(css); 6512 6513 return (u64) scale_load_down(tg->shares); 6514 } 6515 6516 #ifdef CONFIG_CFS_BANDWIDTH 6517 static DEFINE_MUTEX(cfs_constraints_mutex); 6518 6519 const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */ 6520 const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */ 6521 6522 static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime); 6523 6524 static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota) 6525 { 6526 int i, ret = 0, runtime_enabled, runtime_was_enabled; 6527 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 6528 6529 if (tg == &root_task_group) 6530 return -EINVAL; 6531 6532 /* 6533 * Ensure we have at some amount of bandwidth every period. This is 6534 * to prevent reaching a state of large arrears when throttled via 6535 * entity_tick() resulting in prolonged exit starvation. 6536 */ 6537 if (quota < min_cfs_quota_period || period < min_cfs_quota_period) 6538 return -EINVAL; 6539 6540 /* 6541 * Likewise, bound things on the otherside by preventing insane quota 6542 * periods. This also allows us to normalize in computing quota 6543 * feasibility. 6544 */ 6545 if (period > max_cfs_quota_period) 6546 return -EINVAL; 6547 6548 /* 6549 * Prevent race between setting of cfs_rq->runtime_enabled and 6550 * unthrottle_offline_cfs_rqs(). 6551 */ 6552 get_online_cpus(); 6553 mutex_lock(&cfs_constraints_mutex); 6554 ret = __cfs_schedulable(tg, period, quota); 6555 if (ret) 6556 goto out_unlock; 6557 6558 runtime_enabled = quota != RUNTIME_INF; 6559 runtime_was_enabled = cfs_b->quota != RUNTIME_INF; 6560 /* 6561 * If we need to toggle cfs_bandwidth_used, off->on must occur 6562 * before making related changes, and on->off must occur afterwards 6563 */ 6564 if (runtime_enabled && !runtime_was_enabled) 6565 cfs_bandwidth_usage_inc(); 6566 raw_spin_lock_irq(&cfs_b->lock); 6567 cfs_b->period = ns_to_ktime(period); 6568 cfs_b->quota = quota; 6569 6570 __refill_cfs_bandwidth_runtime(cfs_b); 6571 6572 /* Restart the period timer (if active) to handle new period expiry: */ 6573 if (runtime_enabled) 6574 start_cfs_bandwidth(cfs_b); 6575 6576 raw_spin_unlock_irq(&cfs_b->lock); 6577 6578 for_each_online_cpu(i) { 6579 struct cfs_rq *cfs_rq = tg->cfs_rq[i]; 6580 struct rq *rq = cfs_rq->rq; 6581 struct rq_flags rf; 6582 6583 rq_lock_irq(rq, &rf); 6584 cfs_rq->runtime_enabled = runtime_enabled; 6585 cfs_rq->runtime_remaining = 0; 6586 6587 if (cfs_rq->throttled) 6588 unthrottle_cfs_rq(cfs_rq); 6589 rq_unlock_irq(rq, &rf); 6590 } 6591 if (runtime_was_enabled && !runtime_enabled) 6592 cfs_bandwidth_usage_dec(); 6593 out_unlock: 6594 mutex_unlock(&cfs_constraints_mutex); 6595 put_online_cpus(); 6596 6597 return ret; 6598 } 6599 6600 int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us) 6601 { 6602 u64 quota, period; 6603 6604 period = ktime_to_ns(tg->cfs_bandwidth.period); 6605 if (cfs_quota_us < 0) 6606 quota = RUNTIME_INF; 6607 else 6608 quota = (u64)cfs_quota_us * NSEC_PER_USEC; 6609 6610 return tg_set_cfs_bandwidth(tg, period, quota); 6611 } 6612 6613 long tg_get_cfs_quota(struct task_group *tg) 6614 { 6615 u64 quota_us; 6616 6617 if (tg->cfs_bandwidth.quota == RUNTIME_INF) 6618 return -1; 6619 6620 quota_us = tg->cfs_bandwidth.quota; 6621 do_div(quota_us, NSEC_PER_USEC); 6622 6623 return quota_us; 6624 } 6625 6626 int tg_set_cfs_period(struct task_group *tg, long cfs_period_us) 6627 { 6628 u64 quota, period; 6629 6630 period = (u64)cfs_period_us * NSEC_PER_USEC; 6631 quota = tg->cfs_bandwidth.quota; 6632 6633 return tg_set_cfs_bandwidth(tg, period, quota); 6634 } 6635 6636 long tg_get_cfs_period(struct task_group *tg) 6637 { 6638 u64 cfs_period_us; 6639 6640 cfs_period_us = ktime_to_ns(tg->cfs_bandwidth.period); 6641 do_div(cfs_period_us, NSEC_PER_USEC); 6642 6643 return cfs_period_us; 6644 } 6645 6646 static s64 cpu_cfs_quota_read_s64(struct cgroup_subsys_state *css, 6647 struct cftype *cft) 6648 { 6649 return tg_get_cfs_quota(css_tg(css)); 6650 } 6651 6652 static int cpu_cfs_quota_write_s64(struct cgroup_subsys_state *css, 6653 struct cftype *cftype, s64 cfs_quota_us) 6654 { 6655 return tg_set_cfs_quota(css_tg(css), cfs_quota_us); 6656 } 6657 6658 static u64 cpu_cfs_period_read_u64(struct cgroup_subsys_state *css, 6659 struct cftype *cft) 6660 { 6661 return tg_get_cfs_period(css_tg(css)); 6662 } 6663 6664 static int cpu_cfs_period_write_u64(struct cgroup_subsys_state *css, 6665 struct cftype *cftype, u64 cfs_period_us) 6666 { 6667 return tg_set_cfs_period(css_tg(css), cfs_period_us); 6668 } 6669 6670 struct cfs_schedulable_data { 6671 struct task_group *tg; 6672 u64 period, quota; 6673 }; 6674 6675 /* 6676 * normalize group quota/period to be quota/max_period 6677 * note: units are usecs 6678 */ 6679 static u64 normalize_cfs_quota(struct task_group *tg, 6680 struct cfs_schedulable_data *d) 6681 { 6682 u64 quota, period; 6683 6684 if (tg == d->tg) { 6685 period = d->period; 6686 quota = d->quota; 6687 } else { 6688 period = tg_get_cfs_period(tg); 6689 quota = tg_get_cfs_quota(tg); 6690 } 6691 6692 /* note: these should typically be equivalent */ 6693 if (quota == RUNTIME_INF || quota == -1) 6694 return RUNTIME_INF; 6695 6696 return to_ratio(period, quota); 6697 } 6698 6699 static int tg_cfs_schedulable_down(struct task_group *tg, void *data) 6700 { 6701 struct cfs_schedulable_data *d = data; 6702 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 6703 s64 quota = 0, parent_quota = -1; 6704 6705 if (!tg->parent) { 6706 quota = RUNTIME_INF; 6707 } else { 6708 struct cfs_bandwidth *parent_b = &tg->parent->cfs_bandwidth; 6709 6710 quota = normalize_cfs_quota(tg, d); 6711 parent_quota = parent_b->hierarchical_quota; 6712 6713 /* 6714 * Ensure max(child_quota) <= parent_quota. On cgroup2, 6715 * always take the min. On cgroup1, only inherit when no 6716 * limit is set: 6717 */ 6718 if (cgroup_subsys_on_dfl(cpu_cgrp_subsys)) { 6719 quota = min(quota, parent_quota); 6720 } else { 6721 if (quota == RUNTIME_INF) 6722 quota = parent_quota; 6723 else if (parent_quota != RUNTIME_INF && quota > parent_quota) 6724 return -EINVAL; 6725 } 6726 } 6727 cfs_b->hierarchical_quota = quota; 6728 6729 return 0; 6730 } 6731 6732 static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota) 6733 { 6734 int ret; 6735 struct cfs_schedulable_data data = { 6736 .tg = tg, 6737 .period = period, 6738 .quota = quota, 6739 }; 6740 6741 if (quota != RUNTIME_INF) { 6742 do_div(data.period, NSEC_PER_USEC); 6743 do_div(data.quota, NSEC_PER_USEC); 6744 } 6745 6746 rcu_read_lock(); 6747 ret = walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data); 6748 rcu_read_unlock(); 6749 6750 return ret; 6751 } 6752 6753 static int cpu_cfs_stat_show(struct seq_file *sf, void *v) 6754 { 6755 struct task_group *tg = css_tg(seq_css(sf)); 6756 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 6757 6758 seq_printf(sf, "nr_periods %d\n", cfs_b->nr_periods); 6759 seq_printf(sf, "nr_throttled %d\n", cfs_b->nr_throttled); 6760 seq_printf(sf, "throttled_time %llu\n", cfs_b->throttled_time); 6761 6762 if (schedstat_enabled() && tg != &root_task_group) { 6763 u64 ws = 0; 6764 int i; 6765 6766 for_each_possible_cpu(i) 6767 ws += schedstat_val(tg->se[i]->statistics.wait_sum); 6768 6769 seq_printf(sf, "wait_sum %llu\n", ws); 6770 } 6771 6772 return 0; 6773 } 6774 #endif /* CONFIG_CFS_BANDWIDTH */ 6775 #endif /* CONFIG_FAIR_GROUP_SCHED */ 6776 6777 #ifdef CONFIG_RT_GROUP_SCHED 6778 static int cpu_rt_runtime_write(struct cgroup_subsys_state *css, 6779 struct cftype *cft, s64 val) 6780 { 6781 return sched_group_set_rt_runtime(css_tg(css), val); 6782 } 6783 6784 static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css, 6785 struct cftype *cft) 6786 { 6787 return sched_group_rt_runtime(css_tg(css)); 6788 } 6789 6790 static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css, 6791 struct cftype *cftype, u64 rt_period_us) 6792 { 6793 return sched_group_set_rt_period(css_tg(css), rt_period_us); 6794 } 6795 6796 static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css, 6797 struct cftype *cft) 6798 { 6799 return sched_group_rt_period(css_tg(css)); 6800 } 6801 #endif /* CONFIG_RT_GROUP_SCHED */ 6802 6803 static struct cftype cpu_legacy_files[] = { 6804 #ifdef CONFIG_FAIR_GROUP_SCHED 6805 { 6806 .name = "shares", 6807 .read_u64 = cpu_shares_read_u64, 6808 .write_u64 = cpu_shares_write_u64, 6809 }, 6810 #endif 6811 #ifdef CONFIG_CFS_BANDWIDTH 6812 { 6813 .name = "cfs_quota_us", 6814 .read_s64 = cpu_cfs_quota_read_s64, 6815 .write_s64 = cpu_cfs_quota_write_s64, 6816 }, 6817 { 6818 .name = "cfs_period_us", 6819 .read_u64 = cpu_cfs_period_read_u64, 6820 .write_u64 = cpu_cfs_period_write_u64, 6821 }, 6822 { 6823 .name = "stat", 6824 .seq_show = cpu_cfs_stat_show, 6825 }, 6826 #endif 6827 #ifdef CONFIG_RT_GROUP_SCHED 6828 { 6829 .name = "rt_runtime_us", 6830 .read_s64 = cpu_rt_runtime_read, 6831 .write_s64 = cpu_rt_runtime_write, 6832 }, 6833 { 6834 .name = "rt_period_us", 6835 .read_u64 = cpu_rt_period_read_uint, 6836 .write_u64 = cpu_rt_period_write_uint, 6837 }, 6838 #endif 6839 { } /* Terminate */ 6840 }; 6841 6842 static int cpu_extra_stat_show(struct seq_file *sf, 6843 struct cgroup_subsys_state *css) 6844 { 6845 #ifdef CONFIG_CFS_BANDWIDTH 6846 { 6847 struct task_group *tg = css_tg(css); 6848 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 6849 u64 throttled_usec; 6850 6851 throttled_usec = cfs_b->throttled_time; 6852 do_div(throttled_usec, NSEC_PER_USEC); 6853 6854 seq_printf(sf, "nr_periods %d\n" 6855 "nr_throttled %d\n" 6856 "throttled_usec %llu\n", 6857 cfs_b->nr_periods, cfs_b->nr_throttled, 6858 throttled_usec); 6859 } 6860 #endif 6861 return 0; 6862 } 6863 6864 #ifdef CONFIG_FAIR_GROUP_SCHED 6865 static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css, 6866 struct cftype *cft) 6867 { 6868 struct task_group *tg = css_tg(css); 6869 u64 weight = scale_load_down(tg->shares); 6870 6871 return DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024); 6872 } 6873 6874 static int cpu_weight_write_u64(struct cgroup_subsys_state *css, 6875 struct cftype *cft, u64 weight) 6876 { 6877 /* 6878 * cgroup weight knobs should use the common MIN, DFL and MAX 6879 * values which are 1, 100 and 10000 respectively. While it loses 6880 * a bit of range on both ends, it maps pretty well onto the shares 6881 * value used by scheduler and the round-trip conversions preserve 6882 * the original value over the entire range. 6883 */ 6884 if (weight < CGROUP_WEIGHT_MIN || weight > CGROUP_WEIGHT_MAX) 6885 return -ERANGE; 6886 6887 weight = DIV_ROUND_CLOSEST_ULL(weight * 1024, CGROUP_WEIGHT_DFL); 6888 6889 return sched_group_set_shares(css_tg(css), scale_load(weight)); 6890 } 6891 6892 static s64 cpu_weight_nice_read_s64(struct cgroup_subsys_state *css, 6893 struct cftype *cft) 6894 { 6895 unsigned long weight = scale_load_down(css_tg(css)->shares); 6896 int last_delta = INT_MAX; 6897 int prio, delta; 6898 6899 /* find the closest nice value to the current weight */ 6900 for (prio = 0; prio < ARRAY_SIZE(sched_prio_to_weight); prio++) { 6901 delta = abs(sched_prio_to_weight[prio] - weight); 6902 if (delta >= last_delta) 6903 break; 6904 last_delta = delta; 6905 } 6906 6907 return PRIO_TO_NICE(prio - 1 + MAX_RT_PRIO); 6908 } 6909 6910 static int cpu_weight_nice_write_s64(struct cgroup_subsys_state *css, 6911 struct cftype *cft, s64 nice) 6912 { 6913 unsigned long weight; 6914 int idx; 6915 6916 if (nice < MIN_NICE || nice > MAX_NICE) 6917 return -ERANGE; 6918 6919 idx = NICE_TO_PRIO(nice) - MAX_RT_PRIO; 6920 idx = array_index_nospec(idx, 40); 6921 weight = sched_prio_to_weight[idx]; 6922 6923 return sched_group_set_shares(css_tg(css), scale_load(weight)); 6924 } 6925 #endif 6926 6927 static void __maybe_unused cpu_period_quota_print(struct seq_file *sf, 6928 long period, long quota) 6929 { 6930 if (quota < 0) 6931 seq_puts(sf, "max"); 6932 else 6933 seq_printf(sf, "%ld", quota); 6934 6935 seq_printf(sf, " %ld\n", period); 6936 } 6937 6938 /* caller should put the current value in *@periodp before calling */ 6939 static int __maybe_unused cpu_period_quota_parse(char *buf, 6940 u64 *periodp, u64 *quotap) 6941 { 6942 char tok[21]; /* U64_MAX */ 6943 6944 if (!sscanf(buf, "%s %llu", tok, periodp)) 6945 return -EINVAL; 6946 6947 *periodp *= NSEC_PER_USEC; 6948 6949 if (sscanf(tok, "%llu", quotap)) 6950 *quotap *= NSEC_PER_USEC; 6951 else if (!strcmp(tok, "max")) 6952 *quotap = RUNTIME_INF; 6953 else 6954 return -EINVAL; 6955 6956 return 0; 6957 } 6958 6959 #ifdef CONFIG_CFS_BANDWIDTH 6960 static int cpu_max_show(struct seq_file *sf, void *v) 6961 { 6962 struct task_group *tg = css_tg(seq_css(sf)); 6963 6964 cpu_period_quota_print(sf, tg_get_cfs_period(tg), tg_get_cfs_quota(tg)); 6965 return 0; 6966 } 6967 6968 static ssize_t cpu_max_write(struct kernfs_open_file *of, 6969 char *buf, size_t nbytes, loff_t off) 6970 { 6971 struct task_group *tg = css_tg(of_css(of)); 6972 u64 period = tg_get_cfs_period(tg); 6973 u64 quota; 6974 int ret; 6975 6976 ret = cpu_period_quota_parse(buf, &period, "a); 6977 if (!ret) 6978 ret = tg_set_cfs_bandwidth(tg, period, quota); 6979 return ret ?: nbytes; 6980 } 6981 #endif 6982 6983 static struct cftype cpu_files[] = { 6984 #ifdef CONFIG_FAIR_GROUP_SCHED 6985 { 6986 .name = "weight", 6987 .flags = CFTYPE_NOT_ON_ROOT, 6988 .read_u64 = cpu_weight_read_u64, 6989 .write_u64 = cpu_weight_write_u64, 6990 }, 6991 { 6992 .name = "weight.nice", 6993 .flags = CFTYPE_NOT_ON_ROOT, 6994 .read_s64 = cpu_weight_nice_read_s64, 6995 .write_s64 = cpu_weight_nice_write_s64, 6996 }, 6997 #endif 6998 #ifdef CONFIG_CFS_BANDWIDTH 6999 { 7000 .name = "max", 7001 .flags = CFTYPE_NOT_ON_ROOT, 7002 .seq_show = cpu_max_show, 7003 .write = cpu_max_write, 7004 }, 7005 #endif 7006 { } /* terminate */ 7007 }; 7008 7009 struct cgroup_subsys cpu_cgrp_subsys = { 7010 .css_alloc = cpu_cgroup_css_alloc, 7011 .css_online = cpu_cgroup_css_online, 7012 .css_released = cpu_cgroup_css_released, 7013 .css_free = cpu_cgroup_css_free, 7014 .css_extra_stat_show = cpu_extra_stat_show, 7015 .fork = cpu_cgroup_fork, 7016 .can_attach = cpu_cgroup_can_attach, 7017 .attach = cpu_cgroup_attach, 7018 .legacy_cftypes = cpu_legacy_files, 7019 .dfl_cftypes = cpu_files, 7020 .early_init = true, 7021 .threaded = true, 7022 }; 7023 7024 #endif /* CONFIG_CGROUP_SCHED */ 7025 7026 void dump_cpu_task(int cpu) 7027 { 7028 pr_info("Task dump for CPU %d:\n", cpu); 7029 sched_show_task(cpu_curr(cpu)); 7030 } 7031 7032 /* 7033 * Nice levels are multiplicative, with a gentle 10% change for every 7034 * nice level changed. I.e. when a CPU-bound task goes from nice 0 to 7035 * nice 1, it will get ~10% less CPU time than another CPU-bound task 7036 * that remained on nice 0. 7037 * 7038 * The "10% effect" is relative and cumulative: from _any_ nice level, 7039 * if you go up 1 level, it's -10% CPU usage, if you go down 1 level 7040 * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25. 7041 * If a task goes up by ~10% and another task goes down by ~10% then 7042 * the relative distance between them is ~25%.) 7043 */ 7044 const int sched_prio_to_weight[40] = { 7045 /* -20 */ 88761, 71755, 56483, 46273, 36291, 7046 /* -15 */ 29154, 23254, 18705, 14949, 11916, 7047 /* -10 */ 9548, 7620, 6100, 4904, 3906, 7048 /* -5 */ 3121, 2501, 1991, 1586, 1277, 7049 /* 0 */ 1024, 820, 655, 526, 423, 7050 /* 5 */ 335, 272, 215, 172, 137, 7051 /* 10 */ 110, 87, 70, 56, 45, 7052 /* 15 */ 36, 29, 23, 18, 15, 7053 }; 7054 7055 /* 7056 * Inverse (2^32/x) values of the sched_prio_to_weight[] array, precalculated. 7057 * 7058 * In cases where the weight does not change often, we can use the 7059 * precalculated inverse to speed up arithmetics by turning divisions 7060 * into multiplications: 7061 */ 7062 const u32 sched_prio_to_wmult[40] = { 7063 /* -20 */ 48388, 59856, 76040, 92818, 118348, 7064 /* -15 */ 147320, 184698, 229616, 287308, 360437, 7065 /* -10 */ 449829, 563644, 704093, 875809, 1099582, 7066 /* -5 */ 1376151, 1717300, 2157191, 2708050, 3363326, 7067 /* 0 */ 4194304, 5237765, 6557202, 8165337, 10153587, 7068 /* 5 */ 12820798, 15790321, 19976592, 24970740, 31350126, 7069 /* 10 */ 39045157, 49367440, 61356676, 76695844, 95443717, 7070 /* 15 */ 119304647, 148102320, 186737708, 238609294, 286331153, 7071 }; 7072 7073 #undef CREATE_TRACE_POINTS 7074