xref: /openbmc/linux/kernel/sched/core.c (revision fa7f32422ea1ac276b45b96a540ed5981caaa61f)
1 /*
2  *  kernel/sched/core.c
3  *
4  *  Kernel scheduler and related syscalls
5  *
6  *  Copyright (C) 1991-2002  Linus Torvalds
7  *
8  *  1996-12-23  Modified by Dave Grothe to fix bugs in semaphores and
9  *		make semaphores SMP safe
10  *  1998-11-19	Implemented schedule_timeout() and related stuff
11  *		by Andrea Arcangeli
12  *  2002-01-04	New ultra-scalable O(1) scheduler by Ingo Molnar:
13  *		hybrid priority-list and round-robin design with
14  *		an array-switch method of distributing timeslices
15  *		and per-CPU runqueues.  Cleanups and useful suggestions
16  *		by Davide Libenzi, preemptible kernel bits by Robert Love.
17  *  2003-09-03	Interactivity tuning by Con Kolivas.
18  *  2004-04-02	Scheduler domains code by Nick Piggin
19  *  2007-04-15  Work begun on replacing all interactivity tuning with a
20  *              fair scheduling design by Con Kolivas.
21  *  2007-05-05  Load balancing (smp-nice) and other improvements
22  *              by Peter Williams
23  *  2007-05-06  Interactivity improvements to CFS by Mike Galbraith
24  *  2007-07-01  Group scheduling enhancements by Srivatsa Vaddagiri
25  *  2007-11-29  RT balancing improvements by Steven Rostedt, Gregory Haskins,
26  *              Thomas Gleixner, Mike Kravetz
27  */
28 
29 #include <linux/kasan.h>
30 #include <linux/mm.h>
31 #include <linux/module.h>
32 #include <linux/nmi.h>
33 #include <linux/init.h>
34 #include <linux/uaccess.h>
35 #include <linux/highmem.h>
36 #include <linux/mmu_context.h>
37 #include <linux/interrupt.h>
38 #include <linux/capability.h>
39 #include <linux/completion.h>
40 #include <linux/kernel_stat.h>
41 #include <linux/debug_locks.h>
42 #include <linux/perf_event.h>
43 #include <linux/security.h>
44 #include <linux/notifier.h>
45 #include <linux/profile.h>
46 #include <linux/freezer.h>
47 #include <linux/vmalloc.h>
48 #include <linux/blkdev.h>
49 #include <linux/delay.h>
50 #include <linux/pid_namespace.h>
51 #include <linux/smp.h>
52 #include <linux/threads.h>
53 #include <linux/timer.h>
54 #include <linux/rcupdate.h>
55 #include <linux/cpu.h>
56 #include <linux/cpuset.h>
57 #include <linux/percpu.h>
58 #include <linux/proc_fs.h>
59 #include <linux/seq_file.h>
60 #include <linux/sysctl.h>
61 #include <linux/syscalls.h>
62 #include <linux/times.h>
63 #include <linux/tsacct_kern.h>
64 #include <linux/kprobes.h>
65 #include <linux/delayacct.h>
66 #include <linux/unistd.h>
67 #include <linux/pagemap.h>
68 #include <linux/hrtimer.h>
69 #include <linux/tick.h>
70 #include <linux/ctype.h>
71 #include <linux/ftrace.h>
72 #include <linux/slab.h>
73 #include <linux/init_task.h>
74 #include <linux/context_tracking.h>
75 #include <linux/compiler.h>
76 #include <linux/frame.h>
77 #include <linux/prefetch.h>
78 
79 #include <asm/switch_to.h>
80 #include <asm/tlb.h>
81 #include <asm/irq_regs.h>
82 #include <asm/mutex.h>
83 #ifdef CONFIG_PARAVIRT
84 #include <asm/paravirt.h>
85 #endif
86 
87 #include "sched.h"
88 #include "../workqueue_internal.h"
89 #include "../smpboot.h"
90 
91 #define CREATE_TRACE_POINTS
92 #include <trace/events/sched.h>
93 
94 DEFINE_MUTEX(sched_domains_mutex);
95 DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
96 
97 static void update_rq_clock_task(struct rq *rq, s64 delta);
98 
99 void update_rq_clock(struct rq *rq)
100 {
101 	s64 delta;
102 
103 	lockdep_assert_held(&rq->lock);
104 
105 	if (rq->clock_skip_update & RQCF_ACT_SKIP)
106 		return;
107 
108 	delta = sched_clock_cpu(cpu_of(rq)) - rq->clock;
109 	if (delta < 0)
110 		return;
111 	rq->clock += delta;
112 	update_rq_clock_task(rq, delta);
113 }
114 
115 /*
116  * Debugging: various feature bits
117  */
118 
119 #define SCHED_FEAT(name, enabled)	\
120 	(1UL << __SCHED_FEAT_##name) * enabled |
121 
122 const_debug unsigned int sysctl_sched_features =
123 #include "features.h"
124 	0;
125 
126 #undef SCHED_FEAT
127 
128 /*
129  * Number of tasks to iterate in a single balance run.
130  * Limited because this is done with IRQs disabled.
131  */
132 const_debug unsigned int sysctl_sched_nr_migrate = 32;
133 
134 /*
135  * period over which we average the RT time consumption, measured
136  * in ms.
137  *
138  * default: 1s
139  */
140 const_debug unsigned int sysctl_sched_time_avg = MSEC_PER_SEC;
141 
142 /*
143  * period over which we measure -rt task cpu usage in us.
144  * default: 1s
145  */
146 unsigned int sysctl_sched_rt_period = 1000000;
147 
148 __read_mostly int scheduler_running;
149 
150 /*
151  * part of the period that we allow rt tasks to run in us.
152  * default: 0.95s
153  */
154 int sysctl_sched_rt_runtime = 950000;
155 
156 /* cpus with isolated domains */
157 cpumask_var_t cpu_isolated_map;
158 
159 /*
160  * this_rq_lock - lock this runqueue and disable interrupts.
161  */
162 static struct rq *this_rq_lock(void)
163 	__acquires(rq->lock)
164 {
165 	struct rq *rq;
166 
167 	local_irq_disable();
168 	rq = this_rq();
169 	raw_spin_lock(&rq->lock);
170 
171 	return rq;
172 }
173 
174 /*
175  * __task_rq_lock - lock the rq @p resides on.
176  */
177 struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf)
178 	__acquires(rq->lock)
179 {
180 	struct rq *rq;
181 
182 	lockdep_assert_held(&p->pi_lock);
183 
184 	for (;;) {
185 		rq = task_rq(p);
186 		raw_spin_lock(&rq->lock);
187 		if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {
188 			rf->cookie = lockdep_pin_lock(&rq->lock);
189 			return rq;
190 		}
191 		raw_spin_unlock(&rq->lock);
192 
193 		while (unlikely(task_on_rq_migrating(p)))
194 			cpu_relax();
195 	}
196 }
197 
198 /*
199  * task_rq_lock - lock p->pi_lock and lock the rq @p resides on.
200  */
201 struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf)
202 	__acquires(p->pi_lock)
203 	__acquires(rq->lock)
204 {
205 	struct rq *rq;
206 
207 	for (;;) {
208 		raw_spin_lock_irqsave(&p->pi_lock, rf->flags);
209 		rq = task_rq(p);
210 		raw_spin_lock(&rq->lock);
211 		/*
212 		 *	move_queued_task()		task_rq_lock()
213 		 *
214 		 *	ACQUIRE (rq->lock)
215 		 *	[S] ->on_rq = MIGRATING		[L] rq = task_rq()
216 		 *	WMB (__set_task_cpu())		ACQUIRE (rq->lock);
217 		 *	[S] ->cpu = new_cpu		[L] task_rq()
218 		 *					[L] ->on_rq
219 		 *	RELEASE (rq->lock)
220 		 *
221 		 * If we observe the old cpu in task_rq_lock, the acquire of
222 		 * the old rq->lock will fully serialize against the stores.
223 		 *
224 		 * If we observe the new cpu in task_rq_lock, the acquire will
225 		 * pair with the WMB to ensure we must then also see migrating.
226 		 */
227 		if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {
228 			rf->cookie = lockdep_pin_lock(&rq->lock);
229 			return rq;
230 		}
231 		raw_spin_unlock(&rq->lock);
232 		raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags);
233 
234 		while (unlikely(task_on_rq_migrating(p)))
235 			cpu_relax();
236 	}
237 }
238 
239 #ifdef CONFIG_SCHED_HRTICK
240 /*
241  * Use HR-timers to deliver accurate preemption points.
242  */
243 
244 static void hrtick_clear(struct rq *rq)
245 {
246 	if (hrtimer_active(&rq->hrtick_timer))
247 		hrtimer_cancel(&rq->hrtick_timer);
248 }
249 
250 /*
251  * High-resolution timer tick.
252  * Runs from hardirq context with interrupts disabled.
253  */
254 static enum hrtimer_restart hrtick(struct hrtimer *timer)
255 {
256 	struct rq *rq = container_of(timer, struct rq, hrtick_timer);
257 
258 	WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
259 
260 	raw_spin_lock(&rq->lock);
261 	update_rq_clock(rq);
262 	rq->curr->sched_class->task_tick(rq, rq->curr, 1);
263 	raw_spin_unlock(&rq->lock);
264 
265 	return HRTIMER_NORESTART;
266 }
267 
268 #ifdef CONFIG_SMP
269 
270 static void __hrtick_restart(struct rq *rq)
271 {
272 	struct hrtimer *timer = &rq->hrtick_timer;
273 
274 	hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED);
275 }
276 
277 /*
278  * called from hardirq (IPI) context
279  */
280 static void __hrtick_start(void *arg)
281 {
282 	struct rq *rq = arg;
283 
284 	raw_spin_lock(&rq->lock);
285 	__hrtick_restart(rq);
286 	rq->hrtick_csd_pending = 0;
287 	raw_spin_unlock(&rq->lock);
288 }
289 
290 /*
291  * Called to set the hrtick timer state.
292  *
293  * called with rq->lock held and irqs disabled
294  */
295 void hrtick_start(struct rq *rq, u64 delay)
296 {
297 	struct hrtimer *timer = &rq->hrtick_timer;
298 	ktime_t time;
299 	s64 delta;
300 
301 	/*
302 	 * Don't schedule slices shorter than 10000ns, that just
303 	 * doesn't make sense and can cause timer DoS.
304 	 */
305 	delta = max_t(s64, delay, 10000LL);
306 	time = ktime_add_ns(timer->base->get_time(), delta);
307 
308 	hrtimer_set_expires(timer, time);
309 
310 	if (rq == this_rq()) {
311 		__hrtick_restart(rq);
312 	} else if (!rq->hrtick_csd_pending) {
313 		smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd);
314 		rq->hrtick_csd_pending = 1;
315 	}
316 }
317 
318 #else
319 /*
320  * Called to set the hrtick timer state.
321  *
322  * called with rq->lock held and irqs disabled
323  */
324 void hrtick_start(struct rq *rq, u64 delay)
325 {
326 	/*
327 	 * Don't schedule slices shorter than 10000ns, that just
328 	 * doesn't make sense. Rely on vruntime for fairness.
329 	 */
330 	delay = max_t(u64, delay, 10000LL);
331 	hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay),
332 		      HRTIMER_MODE_REL_PINNED);
333 }
334 #endif /* CONFIG_SMP */
335 
336 static void init_rq_hrtick(struct rq *rq)
337 {
338 #ifdef CONFIG_SMP
339 	rq->hrtick_csd_pending = 0;
340 
341 	rq->hrtick_csd.flags = 0;
342 	rq->hrtick_csd.func = __hrtick_start;
343 	rq->hrtick_csd.info = rq;
344 #endif
345 
346 	hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
347 	rq->hrtick_timer.function = hrtick;
348 }
349 #else	/* CONFIG_SCHED_HRTICK */
350 static inline void hrtick_clear(struct rq *rq)
351 {
352 }
353 
354 static inline void init_rq_hrtick(struct rq *rq)
355 {
356 }
357 #endif	/* CONFIG_SCHED_HRTICK */
358 
359 /*
360  * cmpxchg based fetch_or, macro so it works for different integer types
361  */
362 #define fetch_or(ptr, mask)						\
363 	({								\
364 		typeof(ptr) _ptr = (ptr);				\
365 		typeof(mask) _mask = (mask);				\
366 		typeof(*_ptr) _old, _val = *_ptr;			\
367 									\
368 		for (;;) {						\
369 			_old = cmpxchg(_ptr, _val, _val | _mask);	\
370 			if (_old == _val)				\
371 				break;					\
372 			_val = _old;					\
373 		}							\
374 	_old;								\
375 })
376 
377 #if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG)
378 /*
379  * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG,
380  * this avoids any races wrt polling state changes and thereby avoids
381  * spurious IPIs.
382  */
383 static bool set_nr_and_not_polling(struct task_struct *p)
384 {
385 	struct thread_info *ti = task_thread_info(p);
386 	return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG);
387 }
388 
389 /*
390  * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set.
391  *
392  * If this returns true, then the idle task promises to call
393  * sched_ttwu_pending() and reschedule soon.
394  */
395 static bool set_nr_if_polling(struct task_struct *p)
396 {
397 	struct thread_info *ti = task_thread_info(p);
398 	typeof(ti->flags) old, val = READ_ONCE(ti->flags);
399 
400 	for (;;) {
401 		if (!(val & _TIF_POLLING_NRFLAG))
402 			return false;
403 		if (val & _TIF_NEED_RESCHED)
404 			return true;
405 		old = cmpxchg(&ti->flags, val, val | _TIF_NEED_RESCHED);
406 		if (old == val)
407 			break;
408 		val = old;
409 	}
410 	return true;
411 }
412 
413 #else
414 static bool set_nr_and_not_polling(struct task_struct *p)
415 {
416 	set_tsk_need_resched(p);
417 	return true;
418 }
419 
420 #ifdef CONFIG_SMP
421 static bool set_nr_if_polling(struct task_struct *p)
422 {
423 	return false;
424 }
425 #endif
426 #endif
427 
428 void wake_q_add(struct wake_q_head *head, struct task_struct *task)
429 {
430 	struct wake_q_node *node = &task->wake_q;
431 
432 	/*
433 	 * Atomically grab the task, if ->wake_q is !nil already it means
434 	 * its already queued (either by us or someone else) and will get the
435 	 * wakeup due to that.
436 	 *
437 	 * This cmpxchg() implies a full barrier, which pairs with the write
438 	 * barrier implied by the wakeup in wake_up_q().
439 	 */
440 	if (cmpxchg(&node->next, NULL, WAKE_Q_TAIL))
441 		return;
442 
443 	get_task_struct(task);
444 
445 	/*
446 	 * The head is context local, there can be no concurrency.
447 	 */
448 	*head->lastp = node;
449 	head->lastp = &node->next;
450 }
451 
452 void wake_up_q(struct wake_q_head *head)
453 {
454 	struct wake_q_node *node = head->first;
455 
456 	while (node != WAKE_Q_TAIL) {
457 		struct task_struct *task;
458 
459 		task = container_of(node, struct task_struct, wake_q);
460 		BUG_ON(!task);
461 		/* task can safely be re-inserted now */
462 		node = node->next;
463 		task->wake_q.next = NULL;
464 
465 		/*
466 		 * wake_up_process() implies a wmb() to pair with the queueing
467 		 * in wake_q_add() so as not to miss wakeups.
468 		 */
469 		wake_up_process(task);
470 		put_task_struct(task);
471 	}
472 }
473 
474 /*
475  * resched_curr - mark rq's current task 'to be rescheduled now'.
476  *
477  * On UP this means the setting of the need_resched flag, on SMP it
478  * might also involve a cross-CPU call to trigger the scheduler on
479  * the target CPU.
480  */
481 void resched_curr(struct rq *rq)
482 {
483 	struct task_struct *curr = rq->curr;
484 	int cpu;
485 
486 	lockdep_assert_held(&rq->lock);
487 
488 	if (test_tsk_need_resched(curr))
489 		return;
490 
491 	cpu = cpu_of(rq);
492 
493 	if (cpu == smp_processor_id()) {
494 		set_tsk_need_resched(curr);
495 		set_preempt_need_resched();
496 		return;
497 	}
498 
499 	if (set_nr_and_not_polling(curr))
500 		smp_send_reschedule(cpu);
501 	else
502 		trace_sched_wake_idle_without_ipi(cpu);
503 }
504 
505 void resched_cpu(int cpu)
506 {
507 	struct rq *rq = cpu_rq(cpu);
508 	unsigned long flags;
509 
510 	if (!raw_spin_trylock_irqsave(&rq->lock, flags))
511 		return;
512 	resched_curr(rq);
513 	raw_spin_unlock_irqrestore(&rq->lock, flags);
514 }
515 
516 #ifdef CONFIG_SMP
517 #ifdef CONFIG_NO_HZ_COMMON
518 /*
519  * In the semi idle case, use the nearest busy cpu for migrating timers
520  * from an idle cpu.  This is good for power-savings.
521  *
522  * We don't do similar optimization for completely idle system, as
523  * selecting an idle cpu will add more delays to the timers than intended
524  * (as that cpu's timer base may not be uptodate wrt jiffies etc).
525  */
526 int get_nohz_timer_target(void)
527 {
528 	int i, cpu = smp_processor_id();
529 	struct sched_domain *sd;
530 
531 	if (!idle_cpu(cpu) && is_housekeeping_cpu(cpu))
532 		return cpu;
533 
534 	rcu_read_lock();
535 	for_each_domain(cpu, sd) {
536 		for_each_cpu(i, sched_domain_span(sd)) {
537 			if (cpu == i)
538 				continue;
539 
540 			if (!idle_cpu(i) && is_housekeeping_cpu(i)) {
541 				cpu = i;
542 				goto unlock;
543 			}
544 		}
545 	}
546 
547 	if (!is_housekeeping_cpu(cpu))
548 		cpu = housekeeping_any_cpu();
549 unlock:
550 	rcu_read_unlock();
551 	return cpu;
552 }
553 /*
554  * When add_timer_on() enqueues a timer into the timer wheel of an
555  * idle CPU then this timer might expire before the next timer event
556  * which is scheduled to wake up that CPU. In case of a completely
557  * idle system the next event might even be infinite time into the
558  * future. wake_up_idle_cpu() ensures that the CPU is woken up and
559  * leaves the inner idle loop so the newly added timer is taken into
560  * account when the CPU goes back to idle and evaluates the timer
561  * wheel for the next timer event.
562  */
563 static void wake_up_idle_cpu(int cpu)
564 {
565 	struct rq *rq = cpu_rq(cpu);
566 
567 	if (cpu == smp_processor_id())
568 		return;
569 
570 	if (set_nr_and_not_polling(rq->idle))
571 		smp_send_reschedule(cpu);
572 	else
573 		trace_sched_wake_idle_without_ipi(cpu);
574 }
575 
576 static bool wake_up_full_nohz_cpu(int cpu)
577 {
578 	/*
579 	 * We just need the target to call irq_exit() and re-evaluate
580 	 * the next tick. The nohz full kick at least implies that.
581 	 * If needed we can still optimize that later with an
582 	 * empty IRQ.
583 	 */
584 	if (cpu_is_offline(cpu))
585 		return true;  /* Don't try to wake offline CPUs. */
586 	if (tick_nohz_full_cpu(cpu)) {
587 		if (cpu != smp_processor_id() ||
588 		    tick_nohz_tick_stopped())
589 			tick_nohz_full_kick_cpu(cpu);
590 		return true;
591 	}
592 
593 	return false;
594 }
595 
596 /*
597  * Wake up the specified CPU.  If the CPU is going offline, it is the
598  * caller's responsibility to deal with the lost wakeup, for example,
599  * by hooking into the CPU_DEAD notifier like timers and hrtimers do.
600  */
601 void wake_up_nohz_cpu(int cpu)
602 {
603 	if (!wake_up_full_nohz_cpu(cpu))
604 		wake_up_idle_cpu(cpu);
605 }
606 
607 static inline bool got_nohz_idle_kick(void)
608 {
609 	int cpu = smp_processor_id();
610 
611 	if (!test_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu)))
612 		return false;
613 
614 	if (idle_cpu(cpu) && !need_resched())
615 		return true;
616 
617 	/*
618 	 * We can't run Idle Load Balance on this CPU for this time so we
619 	 * cancel it and clear NOHZ_BALANCE_KICK
620 	 */
621 	clear_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu));
622 	return false;
623 }
624 
625 #else /* CONFIG_NO_HZ_COMMON */
626 
627 static inline bool got_nohz_idle_kick(void)
628 {
629 	return false;
630 }
631 
632 #endif /* CONFIG_NO_HZ_COMMON */
633 
634 #ifdef CONFIG_NO_HZ_FULL
635 bool sched_can_stop_tick(struct rq *rq)
636 {
637 	int fifo_nr_running;
638 
639 	/* Deadline tasks, even if single, need the tick */
640 	if (rq->dl.dl_nr_running)
641 		return false;
642 
643 	/*
644 	 * If there are more than one RR tasks, we need the tick to effect the
645 	 * actual RR behaviour.
646 	 */
647 	if (rq->rt.rr_nr_running) {
648 		if (rq->rt.rr_nr_running == 1)
649 			return true;
650 		else
651 			return false;
652 	}
653 
654 	/*
655 	 * If there's no RR tasks, but FIFO tasks, we can skip the tick, no
656 	 * forced preemption between FIFO tasks.
657 	 */
658 	fifo_nr_running = rq->rt.rt_nr_running - rq->rt.rr_nr_running;
659 	if (fifo_nr_running)
660 		return true;
661 
662 	/*
663 	 * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left;
664 	 * if there's more than one we need the tick for involuntary
665 	 * preemption.
666 	 */
667 	if (rq->nr_running > 1)
668 		return false;
669 
670 	return true;
671 }
672 #endif /* CONFIG_NO_HZ_FULL */
673 
674 void sched_avg_update(struct rq *rq)
675 {
676 	s64 period = sched_avg_period();
677 
678 	while ((s64)(rq_clock(rq) - rq->age_stamp) > period) {
679 		/*
680 		 * Inline assembly required to prevent the compiler
681 		 * optimising this loop into a divmod call.
682 		 * See __iter_div_u64_rem() for another example of this.
683 		 */
684 		asm("" : "+rm" (rq->age_stamp));
685 		rq->age_stamp += period;
686 		rq->rt_avg /= 2;
687 	}
688 }
689 
690 #endif /* CONFIG_SMP */
691 
692 #if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \
693 			(defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH)))
694 /*
695  * Iterate task_group tree rooted at *from, calling @down when first entering a
696  * node and @up when leaving it for the final time.
697  *
698  * Caller must hold rcu_lock or sufficient equivalent.
699  */
700 int walk_tg_tree_from(struct task_group *from,
701 			     tg_visitor down, tg_visitor up, void *data)
702 {
703 	struct task_group *parent, *child;
704 	int ret;
705 
706 	parent = from;
707 
708 down:
709 	ret = (*down)(parent, data);
710 	if (ret)
711 		goto out;
712 	list_for_each_entry_rcu(child, &parent->children, siblings) {
713 		parent = child;
714 		goto down;
715 
716 up:
717 		continue;
718 	}
719 	ret = (*up)(parent, data);
720 	if (ret || parent == from)
721 		goto out;
722 
723 	child = parent;
724 	parent = parent->parent;
725 	if (parent)
726 		goto up;
727 out:
728 	return ret;
729 }
730 
731 int tg_nop(struct task_group *tg, void *data)
732 {
733 	return 0;
734 }
735 #endif
736 
737 static void set_load_weight(struct task_struct *p)
738 {
739 	int prio = p->static_prio - MAX_RT_PRIO;
740 	struct load_weight *load = &p->se.load;
741 
742 	/*
743 	 * SCHED_IDLE tasks get minimal weight:
744 	 */
745 	if (idle_policy(p->policy)) {
746 		load->weight = scale_load(WEIGHT_IDLEPRIO);
747 		load->inv_weight = WMULT_IDLEPRIO;
748 		return;
749 	}
750 
751 	load->weight = scale_load(sched_prio_to_weight[prio]);
752 	load->inv_weight = sched_prio_to_wmult[prio];
753 }
754 
755 static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
756 {
757 	update_rq_clock(rq);
758 	if (!(flags & ENQUEUE_RESTORE))
759 		sched_info_queued(rq, p);
760 	p->sched_class->enqueue_task(rq, p, flags);
761 }
762 
763 static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
764 {
765 	update_rq_clock(rq);
766 	if (!(flags & DEQUEUE_SAVE))
767 		sched_info_dequeued(rq, p);
768 	p->sched_class->dequeue_task(rq, p, flags);
769 }
770 
771 void activate_task(struct rq *rq, struct task_struct *p, int flags)
772 {
773 	if (task_contributes_to_load(p))
774 		rq->nr_uninterruptible--;
775 
776 	enqueue_task(rq, p, flags);
777 }
778 
779 void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
780 {
781 	if (task_contributes_to_load(p))
782 		rq->nr_uninterruptible++;
783 
784 	dequeue_task(rq, p, flags);
785 }
786 
787 static void update_rq_clock_task(struct rq *rq, s64 delta)
788 {
789 /*
790  * In theory, the compile should just see 0 here, and optimize out the call
791  * to sched_rt_avg_update. But I don't trust it...
792  */
793 #if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING)
794 	s64 steal = 0, irq_delta = 0;
795 #endif
796 #ifdef CONFIG_IRQ_TIME_ACCOUNTING
797 	irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time;
798 
799 	/*
800 	 * Since irq_time is only updated on {soft,}irq_exit, we might run into
801 	 * this case when a previous update_rq_clock() happened inside a
802 	 * {soft,}irq region.
803 	 *
804 	 * When this happens, we stop ->clock_task and only update the
805 	 * prev_irq_time stamp to account for the part that fit, so that a next
806 	 * update will consume the rest. This ensures ->clock_task is
807 	 * monotonic.
808 	 *
809 	 * It does however cause some slight miss-attribution of {soft,}irq
810 	 * time, a more accurate solution would be to update the irq_time using
811 	 * the current rq->clock timestamp, except that would require using
812 	 * atomic ops.
813 	 */
814 	if (irq_delta > delta)
815 		irq_delta = delta;
816 
817 	rq->prev_irq_time += irq_delta;
818 	delta -= irq_delta;
819 #endif
820 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
821 	if (static_key_false((&paravirt_steal_rq_enabled))) {
822 		steal = paravirt_steal_clock(cpu_of(rq));
823 		steal -= rq->prev_steal_time_rq;
824 
825 		if (unlikely(steal > delta))
826 			steal = delta;
827 
828 		rq->prev_steal_time_rq += steal;
829 		delta -= steal;
830 	}
831 #endif
832 
833 	rq->clock_task += delta;
834 
835 #if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING)
836 	if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY))
837 		sched_rt_avg_update(rq, irq_delta + steal);
838 #endif
839 }
840 
841 void sched_set_stop_task(int cpu, struct task_struct *stop)
842 {
843 	struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
844 	struct task_struct *old_stop = cpu_rq(cpu)->stop;
845 
846 	if (stop) {
847 		/*
848 		 * Make it appear like a SCHED_FIFO task, its something
849 		 * userspace knows about and won't get confused about.
850 		 *
851 		 * Also, it will make PI more or less work without too
852 		 * much confusion -- but then, stop work should not
853 		 * rely on PI working anyway.
854 		 */
855 		sched_setscheduler_nocheck(stop, SCHED_FIFO, &param);
856 
857 		stop->sched_class = &stop_sched_class;
858 	}
859 
860 	cpu_rq(cpu)->stop = stop;
861 
862 	if (old_stop) {
863 		/*
864 		 * Reset it back to a normal scheduling class so that
865 		 * it can die in pieces.
866 		 */
867 		old_stop->sched_class = &rt_sched_class;
868 	}
869 }
870 
871 /*
872  * __normal_prio - return the priority that is based on the static prio
873  */
874 static inline int __normal_prio(struct task_struct *p)
875 {
876 	return p->static_prio;
877 }
878 
879 /*
880  * Calculate the expected normal priority: i.e. priority
881  * without taking RT-inheritance into account. Might be
882  * boosted by interactivity modifiers. Changes upon fork,
883  * setprio syscalls, and whenever the interactivity
884  * estimator recalculates.
885  */
886 static inline int normal_prio(struct task_struct *p)
887 {
888 	int prio;
889 
890 	if (task_has_dl_policy(p))
891 		prio = MAX_DL_PRIO-1;
892 	else if (task_has_rt_policy(p))
893 		prio = MAX_RT_PRIO-1 - p->rt_priority;
894 	else
895 		prio = __normal_prio(p);
896 	return prio;
897 }
898 
899 /*
900  * Calculate the current priority, i.e. the priority
901  * taken into account by the scheduler. This value might
902  * be boosted by RT tasks, or might be boosted by
903  * interactivity modifiers. Will be RT if the task got
904  * RT-boosted. If not then it returns p->normal_prio.
905  */
906 static int effective_prio(struct task_struct *p)
907 {
908 	p->normal_prio = normal_prio(p);
909 	/*
910 	 * If we are RT tasks or we were boosted to RT priority,
911 	 * keep the priority unchanged. Otherwise, update priority
912 	 * to the normal priority:
913 	 */
914 	if (!rt_prio(p->prio))
915 		return p->normal_prio;
916 	return p->prio;
917 }
918 
919 /**
920  * task_curr - is this task currently executing on a CPU?
921  * @p: the task in question.
922  *
923  * Return: 1 if the task is currently executing. 0 otherwise.
924  */
925 inline int task_curr(const struct task_struct *p)
926 {
927 	return cpu_curr(task_cpu(p)) == p;
928 }
929 
930 /*
931  * switched_from, switched_to and prio_changed must _NOT_ drop rq->lock,
932  * use the balance_callback list if you want balancing.
933  *
934  * this means any call to check_class_changed() must be followed by a call to
935  * balance_callback().
936  */
937 static inline void check_class_changed(struct rq *rq, struct task_struct *p,
938 				       const struct sched_class *prev_class,
939 				       int oldprio)
940 {
941 	if (prev_class != p->sched_class) {
942 		if (prev_class->switched_from)
943 			prev_class->switched_from(rq, p);
944 
945 		p->sched_class->switched_to(rq, p);
946 	} else if (oldprio != p->prio || dl_task(p))
947 		p->sched_class->prio_changed(rq, p, oldprio);
948 }
949 
950 void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
951 {
952 	const struct sched_class *class;
953 
954 	if (p->sched_class == rq->curr->sched_class) {
955 		rq->curr->sched_class->check_preempt_curr(rq, p, flags);
956 	} else {
957 		for_each_class(class) {
958 			if (class == rq->curr->sched_class)
959 				break;
960 			if (class == p->sched_class) {
961 				resched_curr(rq);
962 				break;
963 			}
964 		}
965 	}
966 
967 	/*
968 	 * A queue event has occurred, and we're going to schedule.  In
969 	 * this case, we can save a useless back to back clock update.
970 	 */
971 	if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr))
972 		rq_clock_skip_update(rq, true);
973 }
974 
975 #ifdef CONFIG_SMP
976 /*
977  * This is how migration works:
978  *
979  * 1) we invoke migration_cpu_stop() on the target CPU using
980  *    stop_one_cpu().
981  * 2) stopper starts to run (implicitly forcing the migrated thread
982  *    off the CPU)
983  * 3) it checks whether the migrated task is still in the wrong runqueue.
984  * 4) if it's in the wrong runqueue then the migration thread removes
985  *    it and puts it into the right queue.
986  * 5) stopper completes and stop_one_cpu() returns and the migration
987  *    is done.
988  */
989 
990 /*
991  * move_queued_task - move a queued task to new rq.
992  *
993  * Returns (locked) new rq. Old rq's lock is released.
994  */
995 static struct rq *move_queued_task(struct rq *rq, struct task_struct *p, int new_cpu)
996 {
997 	lockdep_assert_held(&rq->lock);
998 
999 	p->on_rq = TASK_ON_RQ_MIGRATING;
1000 	dequeue_task(rq, p, 0);
1001 	set_task_cpu(p, new_cpu);
1002 	raw_spin_unlock(&rq->lock);
1003 
1004 	rq = cpu_rq(new_cpu);
1005 
1006 	raw_spin_lock(&rq->lock);
1007 	BUG_ON(task_cpu(p) != new_cpu);
1008 	enqueue_task(rq, p, 0);
1009 	p->on_rq = TASK_ON_RQ_QUEUED;
1010 	check_preempt_curr(rq, p, 0);
1011 
1012 	return rq;
1013 }
1014 
1015 struct migration_arg {
1016 	struct task_struct *task;
1017 	int dest_cpu;
1018 };
1019 
1020 /*
1021  * Move (not current) task off this cpu, onto dest cpu. We're doing
1022  * this because either it can't run here any more (set_cpus_allowed()
1023  * away from this CPU, or CPU going down), or because we're
1024  * attempting to rebalance this task on exec (sched_exec).
1025  *
1026  * So we race with normal scheduler movements, but that's OK, as long
1027  * as the task is no longer on this CPU.
1028  */
1029 static struct rq *__migrate_task(struct rq *rq, struct task_struct *p, int dest_cpu)
1030 {
1031 	if (unlikely(!cpu_active(dest_cpu)))
1032 		return rq;
1033 
1034 	/* Affinity changed (again). */
1035 	if (!cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p)))
1036 		return rq;
1037 
1038 	rq = move_queued_task(rq, p, dest_cpu);
1039 
1040 	return rq;
1041 }
1042 
1043 /*
1044  * migration_cpu_stop - this will be executed by a highprio stopper thread
1045  * and performs thread migration by bumping thread off CPU then
1046  * 'pushing' onto another runqueue.
1047  */
1048 static int migration_cpu_stop(void *data)
1049 {
1050 	struct migration_arg *arg = data;
1051 	struct task_struct *p = arg->task;
1052 	struct rq *rq = this_rq();
1053 
1054 	/*
1055 	 * The original target cpu might have gone down and we might
1056 	 * be on another cpu but it doesn't matter.
1057 	 */
1058 	local_irq_disable();
1059 	/*
1060 	 * We need to explicitly wake pending tasks before running
1061 	 * __migrate_task() such that we will not miss enforcing cpus_allowed
1062 	 * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test.
1063 	 */
1064 	sched_ttwu_pending();
1065 
1066 	raw_spin_lock(&p->pi_lock);
1067 	raw_spin_lock(&rq->lock);
1068 	/*
1069 	 * If task_rq(p) != rq, it cannot be migrated here, because we're
1070 	 * holding rq->lock, if p->on_rq == 0 it cannot get enqueued because
1071 	 * we're holding p->pi_lock.
1072 	 */
1073 	if (task_rq(p) == rq) {
1074 		if (task_on_rq_queued(p))
1075 			rq = __migrate_task(rq, p, arg->dest_cpu);
1076 		else
1077 			p->wake_cpu = arg->dest_cpu;
1078 	}
1079 	raw_spin_unlock(&rq->lock);
1080 	raw_spin_unlock(&p->pi_lock);
1081 
1082 	local_irq_enable();
1083 	return 0;
1084 }
1085 
1086 /*
1087  * sched_class::set_cpus_allowed must do the below, but is not required to
1088  * actually call this function.
1089  */
1090 void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask)
1091 {
1092 	cpumask_copy(&p->cpus_allowed, new_mask);
1093 	p->nr_cpus_allowed = cpumask_weight(new_mask);
1094 }
1095 
1096 void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
1097 {
1098 	struct rq *rq = task_rq(p);
1099 	bool queued, running;
1100 
1101 	lockdep_assert_held(&p->pi_lock);
1102 
1103 	queued = task_on_rq_queued(p);
1104 	running = task_current(rq, p);
1105 
1106 	if (queued) {
1107 		/*
1108 		 * Because __kthread_bind() calls this on blocked tasks without
1109 		 * holding rq->lock.
1110 		 */
1111 		lockdep_assert_held(&rq->lock);
1112 		dequeue_task(rq, p, DEQUEUE_SAVE);
1113 	}
1114 	if (running)
1115 		put_prev_task(rq, p);
1116 
1117 	p->sched_class->set_cpus_allowed(p, new_mask);
1118 
1119 	if (queued)
1120 		enqueue_task(rq, p, ENQUEUE_RESTORE);
1121 	if (running)
1122 		set_curr_task(rq, p);
1123 }
1124 
1125 /*
1126  * Change a given task's CPU affinity. Migrate the thread to a
1127  * proper CPU and schedule it away if the CPU it's executing on
1128  * is removed from the allowed bitmask.
1129  *
1130  * NOTE: the caller must have a valid reference to the task, the
1131  * task must not exit() & deallocate itself prematurely. The
1132  * call is not atomic; no spinlocks may be held.
1133  */
1134 static int __set_cpus_allowed_ptr(struct task_struct *p,
1135 				  const struct cpumask *new_mask, bool check)
1136 {
1137 	const struct cpumask *cpu_valid_mask = cpu_active_mask;
1138 	unsigned int dest_cpu;
1139 	struct rq_flags rf;
1140 	struct rq *rq;
1141 	int ret = 0;
1142 
1143 	rq = task_rq_lock(p, &rf);
1144 
1145 	if (p->flags & PF_KTHREAD) {
1146 		/*
1147 		 * Kernel threads are allowed on online && !active CPUs
1148 		 */
1149 		cpu_valid_mask = cpu_online_mask;
1150 	}
1151 
1152 	/*
1153 	 * Must re-check here, to close a race against __kthread_bind(),
1154 	 * sched_setaffinity() is not guaranteed to observe the flag.
1155 	 */
1156 	if (check && (p->flags & PF_NO_SETAFFINITY)) {
1157 		ret = -EINVAL;
1158 		goto out;
1159 	}
1160 
1161 	if (cpumask_equal(&p->cpus_allowed, new_mask))
1162 		goto out;
1163 
1164 	if (!cpumask_intersects(new_mask, cpu_valid_mask)) {
1165 		ret = -EINVAL;
1166 		goto out;
1167 	}
1168 
1169 	do_set_cpus_allowed(p, new_mask);
1170 
1171 	if (p->flags & PF_KTHREAD) {
1172 		/*
1173 		 * For kernel threads that do indeed end up on online &&
1174 		 * !active we want to ensure they are strict per-cpu threads.
1175 		 */
1176 		WARN_ON(cpumask_intersects(new_mask, cpu_online_mask) &&
1177 			!cpumask_intersects(new_mask, cpu_active_mask) &&
1178 			p->nr_cpus_allowed != 1);
1179 	}
1180 
1181 	/* Can the task run on the task's current CPU? If so, we're done */
1182 	if (cpumask_test_cpu(task_cpu(p), new_mask))
1183 		goto out;
1184 
1185 	dest_cpu = cpumask_any_and(cpu_valid_mask, new_mask);
1186 	if (task_running(rq, p) || p->state == TASK_WAKING) {
1187 		struct migration_arg arg = { p, dest_cpu };
1188 		/* Need help from migration thread: drop lock and wait. */
1189 		task_rq_unlock(rq, p, &rf);
1190 		stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
1191 		tlb_migrate_finish(p->mm);
1192 		return 0;
1193 	} else if (task_on_rq_queued(p)) {
1194 		/*
1195 		 * OK, since we're going to drop the lock immediately
1196 		 * afterwards anyway.
1197 		 */
1198 		lockdep_unpin_lock(&rq->lock, rf.cookie);
1199 		rq = move_queued_task(rq, p, dest_cpu);
1200 		lockdep_repin_lock(&rq->lock, rf.cookie);
1201 	}
1202 out:
1203 	task_rq_unlock(rq, p, &rf);
1204 
1205 	return ret;
1206 }
1207 
1208 int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
1209 {
1210 	return __set_cpus_allowed_ptr(p, new_mask, false);
1211 }
1212 EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
1213 
1214 void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
1215 {
1216 #ifdef CONFIG_SCHED_DEBUG
1217 	/*
1218 	 * We should never call set_task_cpu() on a blocked task,
1219 	 * ttwu() will sort out the placement.
1220 	 */
1221 	WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING &&
1222 			!p->on_rq);
1223 
1224 	/*
1225 	 * Migrating fair class task must have p->on_rq = TASK_ON_RQ_MIGRATING,
1226 	 * because schedstat_wait_{start,end} rebase migrating task's wait_start
1227 	 * time relying on p->on_rq.
1228 	 */
1229 	WARN_ON_ONCE(p->state == TASK_RUNNING &&
1230 		     p->sched_class == &fair_sched_class &&
1231 		     (p->on_rq && !task_on_rq_migrating(p)));
1232 
1233 #ifdef CONFIG_LOCKDEP
1234 	/*
1235 	 * The caller should hold either p->pi_lock or rq->lock, when changing
1236 	 * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks.
1237 	 *
1238 	 * sched_move_task() holds both and thus holding either pins the cgroup,
1239 	 * see task_group().
1240 	 *
1241 	 * Furthermore, all task_rq users should acquire both locks, see
1242 	 * task_rq_lock().
1243 	 */
1244 	WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) ||
1245 				      lockdep_is_held(&task_rq(p)->lock)));
1246 #endif
1247 #endif
1248 
1249 	trace_sched_migrate_task(p, new_cpu);
1250 
1251 	if (task_cpu(p) != new_cpu) {
1252 		if (p->sched_class->migrate_task_rq)
1253 			p->sched_class->migrate_task_rq(p);
1254 		p->se.nr_migrations++;
1255 		perf_event_task_migrate(p);
1256 	}
1257 
1258 	__set_task_cpu(p, new_cpu);
1259 }
1260 
1261 static void __migrate_swap_task(struct task_struct *p, int cpu)
1262 {
1263 	if (task_on_rq_queued(p)) {
1264 		struct rq *src_rq, *dst_rq;
1265 
1266 		src_rq = task_rq(p);
1267 		dst_rq = cpu_rq(cpu);
1268 
1269 		p->on_rq = TASK_ON_RQ_MIGRATING;
1270 		deactivate_task(src_rq, p, 0);
1271 		set_task_cpu(p, cpu);
1272 		activate_task(dst_rq, p, 0);
1273 		p->on_rq = TASK_ON_RQ_QUEUED;
1274 		check_preempt_curr(dst_rq, p, 0);
1275 	} else {
1276 		/*
1277 		 * Task isn't running anymore; make it appear like we migrated
1278 		 * it before it went to sleep. This means on wakeup we make the
1279 		 * previous cpu our target instead of where it really is.
1280 		 */
1281 		p->wake_cpu = cpu;
1282 	}
1283 }
1284 
1285 struct migration_swap_arg {
1286 	struct task_struct *src_task, *dst_task;
1287 	int src_cpu, dst_cpu;
1288 };
1289 
1290 static int migrate_swap_stop(void *data)
1291 {
1292 	struct migration_swap_arg *arg = data;
1293 	struct rq *src_rq, *dst_rq;
1294 	int ret = -EAGAIN;
1295 
1296 	if (!cpu_active(arg->src_cpu) || !cpu_active(arg->dst_cpu))
1297 		return -EAGAIN;
1298 
1299 	src_rq = cpu_rq(arg->src_cpu);
1300 	dst_rq = cpu_rq(arg->dst_cpu);
1301 
1302 	double_raw_lock(&arg->src_task->pi_lock,
1303 			&arg->dst_task->pi_lock);
1304 	double_rq_lock(src_rq, dst_rq);
1305 
1306 	if (task_cpu(arg->dst_task) != arg->dst_cpu)
1307 		goto unlock;
1308 
1309 	if (task_cpu(arg->src_task) != arg->src_cpu)
1310 		goto unlock;
1311 
1312 	if (!cpumask_test_cpu(arg->dst_cpu, tsk_cpus_allowed(arg->src_task)))
1313 		goto unlock;
1314 
1315 	if (!cpumask_test_cpu(arg->src_cpu, tsk_cpus_allowed(arg->dst_task)))
1316 		goto unlock;
1317 
1318 	__migrate_swap_task(arg->src_task, arg->dst_cpu);
1319 	__migrate_swap_task(arg->dst_task, arg->src_cpu);
1320 
1321 	ret = 0;
1322 
1323 unlock:
1324 	double_rq_unlock(src_rq, dst_rq);
1325 	raw_spin_unlock(&arg->dst_task->pi_lock);
1326 	raw_spin_unlock(&arg->src_task->pi_lock);
1327 
1328 	return ret;
1329 }
1330 
1331 /*
1332  * Cross migrate two tasks
1333  */
1334 int migrate_swap(struct task_struct *cur, struct task_struct *p)
1335 {
1336 	struct migration_swap_arg arg;
1337 	int ret = -EINVAL;
1338 
1339 	arg = (struct migration_swap_arg){
1340 		.src_task = cur,
1341 		.src_cpu = task_cpu(cur),
1342 		.dst_task = p,
1343 		.dst_cpu = task_cpu(p),
1344 	};
1345 
1346 	if (arg.src_cpu == arg.dst_cpu)
1347 		goto out;
1348 
1349 	/*
1350 	 * These three tests are all lockless; this is OK since all of them
1351 	 * will be re-checked with proper locks held further down the line.
1352 	 */
1353 	if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu))
1354 		goto out;
1355 
1356 	if (!cpumask_test_cpu(arg.dst_cpu, tsk_cpus_allowed(arg.src_task)))
1357 		goto out;
1358 
1359 	if (!cpumask_test_cpu(arg.src_cpu, tsk_cpus_allowed(arg.dst_task)))
1360 		goto out;
1361 
1362 	trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu);
1363 	ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg);
1364 
1365 out:
1366 	return ret;
1367 }
1368 
1369 /*
1370  * wait_task_inactive - wait for a thread to unschedule.
1371  *
1372  * If @match_state is nonzero, it's the @p->state value just checked and
1373  * not expected to change.  If it changes, i.e. @p might have woken up,
1374  * then return zero.  When we succeed in waiting for @p to be off its CPU,
1375  * we return a positive number (its total switch count).  If a second call
1376  * a short while later returns the same number, the caller can be sure that
1377  * @p has remained unscheduled the whole time.
1378  *
1379  * The caller must ensure that the task *will* unschedule sometime soon,
1380  * else this function might spin for a *long* time. This function can't
1381  * be called with interrupts off, or it may introduce deadlock with
1382  * smp_call_function() if an IPI is sent by the same process we are
1383  * waiting to become inactive.
1384  */
1385 unsigned long wait_task_inactive(struct task_struct *p, long match_state)
1386 {
1387 	int running, queued;
1388 	struct rq_flags rf;
1389 	unsigned long ncsw;
1390 	struct rq *rq;
1391 
1392 	for (;;) {
1393 		/*
1394 		 * We do the initial early heuristics without holding
1395 		 * any task-queue locks at all. We'll only try to get
1396 		 * the runqueue lock when things look like they will
1397 		 * work out!
1398 		 */
1399 		rq = task_rq(p);
1400 
1401 		/*
1402 		 * If the task is actively running on another CPU
1403 		 * still, just relax and busy-wait without holding
1404 		 * any locks.
1405 		 *
1406 		 * NOTE! Since we don't hold any locks, it's not
1407 		 * even sure that "rq" stays as the right runqueue!
1408 		 * But we don't care, since "task_running()" will
1409 		 * return false if the runqueue has changed and p
1410 		 * is actually now running somewhere else!
1411 		 */
1412 		while (task_running(rq, p)) {
1413 			if (match_state && unlikely(p->state != match_state))
1414 				return 0;
1415 			cpu_relax();
1416 		}
1417 
1418 		/*
1419 		 * Ok, time to look more closely! We need the rq
1420 		 * lock now, to be *sure*. If we're wrong, we'll
1421 		 * just go back and repeat.
1422 		 */
1423 		rq = task_rq_lock(p, &rf);
1424 		trace_sched_wait_task(p);
1425 		running = task_running(rq, p);
1426 		queued = task_on_rq_queued(p);
1427 		ncsw = 0;
1428 		if (!match_state || p->state == match_state)
1429 			ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
1430 		task_rq_unlock(rq, p, &rf);
1431 
1432 		/*
1433 		 * If it changed from the expected state, bail out now.
1434 		 */
1435 		if (unlikely(!ncsw))
1436 			break;
1437 
1438 		/*
1439 		 * Was it really running after all now that we
1440 		 * checked with the proper locks actually held?
1441 		 *
1442 		 * Oops. Go back and try again..
1443 		 */
1444 		if (unlikely(running)) {
1445 			cpu_relax();
1446 			continue;
1447 		}
1448 
1449 		/*
1450 		 * It's not enough that it's not actively running,
1451 		 * it must be off the runqueue _entirely_, and not
1452 		 * preempted!
1453 		 *
1454 		 * So if it was still runnable (but just not actively
1455 		 * running right now), it's preempted, and we should
1456 		 * yield - it could be a while.
1457 		 */
1458 		if (unlikely(queued)) {
1459 			ktime_t to = ktime_set(0, NSEC_PER_SEC/HZ);
1460 
1461 			set_current_state(TASK_UNINTERRUPTIBLE);
1462 			schedule_hrtimeout(&to, HRTIMER_MODE_REL);
1463 			continue;
1464 		}
1465 
1466 		/*
1467 		 * Ahh, all good. It wasn't running, and it wasn't
1468 		 * runnable, which means that it will never become
1469 		 * running in the future either. We're all done!
1470 		 */
1471 		break;
1472 	}
1473 
1474 	return ncsw;
1475 }
1476 
1477 /***
1478  * kick_process - kick a running thread to enter/exit the kernel
1479  * @p: the to-be-kicked thread
1480  *
1481  * Cause a process which is running on another CPU to enter
1482  * kernel-mode, without any delay. (to get signals handled.)
1483  *
1484  * NOTE: this function doesn't have to take the runqueue lock,
1485  * because all it wants to ensure is that the remote task enters
1486  * the kernel. If the IPI races and the task has been migrated
1487  * to another CPU then no harm is done and the purpose has been
1488  * achieved as well.
1489  */
1490 void kick_process(struct task_struct *p)
1491 {
1492 	int cpu;
1493 
1494 	preempt_disable();
1495 	cpu = task_cpu(p);
1496 	if ((cpu != smp_processor_id()) && task_curr(p))
1497 		smp_send_reschedule(cpu);
1498 	preempt_enable();
1499 }
1500 EXPORT_SYMBOL_GPL(kick_process);
1501 
1502 /*
1503  * ->cpus_allowed is protected by both rq->lock and p->pi_lock
1504  *
1505  * A few notes on cpu_active vs cpu_online:
1506  *
1507  *  - cpu_active must be a subset of cpu_online
1508  *
1509  *  - on cpu-up we allow per-cpu kthreads on the online && !active cpu,
1510  *    see __set_cpus_allowed_ptr(). At this point the newly online
1511  *    cpu isn't yet part of the sched domains, and balancing will not
1512  *    see it.
1513  *
1514  *  - on cpu-down we clear cpu_active() to mask the sched domains and
1515  *    avoid the load balancer to place new tasks on the to be removed
1516  *    cpu. Existing tasks will remain running there and will be taken
1517  *    off.
1518  *
1519  * This means that fallback selection must not select !active CPUs.
1520  * And can assume that any active CPU must be online. Conversely
1521  * select_task_rq() below may allow selection of !active CPUs in order
1522  * to satisfy the above rules.
1523  */
1524 static int select_fallback_rq(int cpu, struct task_struct *p)
1525 {
1526 	int nid = cpu_to_node(cpu);
1527 	const struct cpumask *nodemask = NULL;
1528 	enum { cpuset, possible, fail } state = cpuset;
1529 	int dest_cpu;
1530 
1531 	/*
1532 	 * If the node that the cpu is on has been offlined, cpu_to_node()
1533 	 * will return -1. There is no cpu on the node, and we should
1534 	 * select the cpu on the other node.
1535 	 */
1536 	if (nid != -1) {
1537 		nodemask = cpumask_of_node(nid);
1538 
1539 		/* Look for allowed, online CPU in same node. */
1540 		for_each_cpu(dest_cpu, nodemask) {
1541 			if (!cpu_active(dest_cpu))
1542 				continue;
1543 			if (cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p)))
1544 				return dest_cpu;
1545 		}
1546 	}
1547 
1548 	for (;;) {
1549 		/* Any allowed, online CPU? */
1550 		for_each_cpu(dest_cpu, tsk_cpus_allowed(p)) {
1551 			if (!(p->flags & PF_KTHREAD) && !cpu_active(dest_cpu))
1552 				continue;
1553 			if (!cpu_online(dest_cpu))
1554 				continue;
1555 			goto out;
1556 		}
1557 
1558 		/* No more Mr. Nice Guy. */
1559 		switch (state) {
1560 		case cpuset:
1561 			if (IS_ENABLED(CONFIG_CPUSETS)) {
1562 				cpuset_cpus_allowed_fallback(p);
1563 				state = possible;
1564 				break;
1565 			}
1566 			/* fall-through */
1567 		case possible:
1568 			do_set_cpus_allowed(p, cpu_possible_mask);
1569 			state = fail;
1570 			break;
1571 
1572 		case fail:
1573 			BUG();
1574 			break;
1575 		}
1576 	}
1577 
1578 out:
1579 	if (state != cpuset) {
1580 		/*
1581 		 * Don't tell them about moving exiting tasks or
1582 		 * kernel threads (both mm NULL), since they never
1583 		 * leave kernel.
1584 		 */
1585 		if (p->mm && printk_ratelimit()) {
1586 			printk_deferred("process %d (%s) no longer affine to cpu%d\n",
1587 					task_pid_nr(p), p->comm, cpu);
1588 		}
1589 	}
1590 
1591 	return dest_cpu;
1592 }
1593 
1594 /*
1595  * The caller (fork, wakeup) owns p->pi_lock, ->cpus_allowed is stable.
1596  */
1597 static inline
1598 int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags)
1599 {
1600 	lockdep_assert_held(&p->pi_lock);
1601 
1602 	if (tsk_nr_cpus_allowed(p) > 1)
1603 		cpu = p->sched_class->select_task_rq(p, cpu, sd_flags, wake_flags);
1604 	else
1605 		cpu = cpumask_any(tsk_cpus_allowed(p));
1606 
1607 	/*
1608 	 * In order not to call set_task_cpu() on a blocking task we need
1609 	 * to rely on ttwu() to place the task on a valid ->cpus_allowed
1610 	 * cpu.
1611 	 *
1612 	 * Since this is common to all placement strategies, this lives here.
1613 	 *
1614 	 * [ this allows ->select_task() to simply return task_cpu(p) and
1615 	 *   not worry about this generic constraint ]
1616 	 */
1617 	if (unlikely(!cpumask_test_cpu(cpu, tsk_cpus_allowed(p)) ||
1618 		     !cpu_online(cpu)))
1619 		cpu = select_fallback_rq(task_cpu(p), p);
1620 
1621 	return cpu;
1622 }
1623 
1624 static void update_avg(u64 *avg, u64 sample)
1625 {
1626 	s64 diff = sample - *avg;
1627 	*avg += diff >> 3;
1628 }
1629 
1630 #else
1631 
1632 static inline int __set_cpus_allowed_ptr(struct task_struct *p,
1633 					 const struct cpumask *new_mask, bool check)
1634 {
1635 	return set_cpus_allowed_ptr(p, new_mask);
1636 }
1637 
1638 #endif /* CONFIG_SMP */
1639 
1640 static void
1641 ttwu_stat(struct task_struct *p, int cpu, int wake_flags)
1642 {
1643 	struct rq *rq;
1644 
1645 	if (!schedstat_enabled())
1646 		return;
1647 
1648 	rq = this_rq();
1649 
1650 #ifdef CONFIG_SMP
1651 	if (cpu == rq->cpu) {
1652 		schedstat_inc(rq->ttwu_local);
1653 		schedstat_inc(p->se.statistics.nr_wakeups_local);
1654 	} else {
1655 		struct sched_domain *sd;
1656 
1657 		schedstat_inc(p->se.statistics.nr_wakeups_remote);
1658 		rcu_read_lock();
1659 		for_each_domain(rq->cpu, sd) {
1660 			if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
1661 				schedstat_inc(sd->ttwu_wake_remote);
1662 				break;
1663 			}
1664 		}
1665 		rcu_read_unlock();
1666 	}
1667 
1668 	if (wake_flags & WF_MIGRATED)
1669 		schedstat_inc(p->se.statistics.nr_wakeups_migrate);
1670 #endif /* CONFIG_SMP */
1671 
1672 	schedstat_inc(rq->ttwu_count);
1673 	schedstat_inc(p->se.statistics.nr_wakeups);
1674 
1675 	if (wake_flags & WF_SYNC)
1676 		schedstat_inc(p->se.statistics.nr_wakeups_sync);
1677 }
1678 
1679 static inline void ttwu_activate(struct rq *rq, struct task_struct *p, int en_flags)
1680 {
1681 	activate_task(rq, p, en_flags);
1682 	p->on_rq = TASK_ON_RQ_QUEUED;
1683 
1684 	/* if a worker is waking up, notify workqueue */
1685 	if (p->flags & PF_WQ_WORKER)
1686 		wq_worker_waking_up(p, cpu_of(rq));
1687 }
1688 
1689 /*
1690  * Mark the task runnable and perform wakeup-preemption.
1691  */
1692 static void ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags,
1693 			   struct pin_cookie cookie)
1694 {
1695 	check_preempt_curr(rq, p, wake_flags);
1696 	p->state = TASK_RUNNING;
1697 	trace_sched_wakeup(p);
1698 
1699 #ifdef CONFIG_SMP
1700 	if (p->sched_class->task_woken) {
1701 		/*
1702 		 * Our task @p is fully woken up and running; so its safe to
1703 		 * drop the rq->lock, hereafter rq is only used for statistics.
1704 		 */
1705 		lockdep_unpin_lock(&rq->lock, cookie);
1706 		p->sched_class->task_woken(rq, p);
1707 		lockdep_repin_lock(&rq->lock, cookie);
1708 	}
1709 
1710 	if (rq->idle_stamp) {
1711 		u64 delta = rq_clock(rq) - rq->idle_stamp;
1712 		u64 max = 2*rq->max_idle_balance_cost;
1713 
1714 		update_avg(&rq->avg_idle, delta);
1715 
1716 		if (rq->avg_idle > max)
1717 			rq->avg_idle = max;
1718 
1719 		rq->idle_stamp = 0;
1720 	}
1721 #endif
1722 }
1723 
1724 static void
1725 ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags,
1726 		 struct pin_cookie cookie)
1727 {
1728 	int en_flags = ENQUEUE_WAKEUP;
1729 
1730 	lockdep_assert_held(&rq->lock);
1731 
1732 #ifdef CONFIG_SMP
1733 	if (p->sched_contributes_to_load)
1734 		rq->nr_uninterruptible--;
1735 
1736 	if (wake_flags & WF_MIGRATED)
1737 		en_flags |= ENQUEUE_MIGRATED;
1738 #endif
1739 
1740 	ttwu_activate(rq, p, en_flags);
1741 	ttwu_do_wakeup(rq, p, wake_flags, cookie);
1742 }
1743 
1744 /*
1745  * Called in case the task @p isn't fully descheduled from its runqueue,
1746  * in this case we must do a remote wakeup. Its a 'light' wakeup though,
1747  * since all we need to do is flip p->state to TASK_RUNNING, since
1748  * the task is still ->on_rq.
1749  */
1750 static int ttwu_remote(struct task_struct *p, int wake_flags)
1751 {
1752 	struct rq_flags rf;
1753 	struct rq *rq;
1754 	int ret = 0;
1755 
1756 	rq = __task_rq_lock(p, &rf);
1757 	if (task_on_rq_queued(p)) {
1758 		/* check_preempt_curr() may use rq clock */
1759 		update_rq_clock(rq);
1760 		ttwu_do_wakeup(rq, p, wake_flags, rf.cookie);
1761 		ret = 1;
1762 	}
1763 	__task_rq_unlock(rq, &rf);
1764 
1765 	return ret;
1766 }
1767 
1768 #ifdef CONFIG_SMP
1769 void sched_ttwu_pending(void)
1770 {
1771 	struct rq *rq = this_rq();
1772 	struct llist_node *llist = llist_del_all(&rq->wake_list);
1773 	struct pin_cookie cookie;
1774 	struct task_struct *p;
1775 	unsigned long flags;
1776 
1777 	if (!llist)
1778 		return;
1779 
1780 	raw_spin_lock_irqsave(&rq->lock, flags);
1781 	cookie = lockdep_pin_lock(&rq->lock);
1782 
1783 	while (llist) {
1784 		int wake_flags = 0;
1785 
1786 		p = llist_entry(llist, struct task_struct, wake_entry);
1787 		llist = llist_next(llist);
1788 
1789 		if (p->sched_remote_wakeup)
1790 			wake_flags = WF_MIGRATED;
1791 
1792 		ttwu_do_activate(rq, p, wake_flags, cookie);
1793 	}
1794 
1795 	lockdep_unpin_lock(&rq->lock, cookie);
1796 	raw_spin_unlock_irqrestore(&rq->lock, flags);
1797 }
1798 
1799 void scheduler_ipi(void)
1800 {
1801 	/*
1802 	 * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting
1803 	 * TIF_NEED_RESCHED remotely (for the first time) will also send
1804 	 * this IPI.
1805 	 */
1806 	preempt_fold_need_resched();
1807 
1808 	if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick())
1809 		return;
1810 
1811 	/*
1812 	 * Not all reschedule IPI handlers call irq_enter/irq_exit, since
1813 	 * traditionally all their work was done from the interrupt return
1814 	 * path. Now that we actually do some work, we need to make sure
1815 	 * we do call them.
1816 	 *
1817 	 * Some archs already do call them, luckily irq_enter/exit nest
1818 	 * properly.
1819 	 *
1820 	 * Arguably we should visit all archs and update all handlers,
1821 	 * however a fair share of IPIs are still resched only so this would
1822 	 * somewhat pessimize the simple resched case.
1823 	 */
1824 	irq_enter();
1825 	sched_ttwu_pending();
1826 
1827 	/*
1828 	 * Check if someone kicked us for doing the nohz idle load balance.
1829 	 */
1830 	if (unlikely(got_nohz_idle_kick())) {
1831 		this_rq()->idle_balance = 1;
1832 		raise_softirq_irqoff(SCHED_SOFTIRQ);
1833 	}
1834 	irq_exit();
1835 }
1836 
1837 static void ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags)
1838 {
1839 	struct rq *rq = cpu_rq(cpu);
1840 
1841 	p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED);
1842 
1843 	if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) {
1844 		if (!set_nr_if_polling(rq->idle))
1845 			smp_send_reschedule(cpu);
1846 		else
1847 			trace_sched_wake_idle_without_ipi(cpu);
1848 	}
1849 }
1850 
1851 void wake_up_if_idle(int cpu)
1852 {
1853 	struct rq *rq = cpu_rq(cpu);
1854 	unsigned long flags;
1855 
1856 	rcu_read_lock();
1857 
1858 	if (!is_idle_task(rcu_dereference(rq->curr)))
1859 		goto out;
1860 
1861 	if (set_nr_if_polling(rq->idle)) {
1862 		trace_sched_wake_idle_without_ipi(cpu);
1863 	} else {
1864 		raw_spin_lock_irqsave(&rq->lock, flags);
1865 		if (is_idle_task(rq->curr))
1866 			smp_send_reschedule(cpu);
1867 		/* Else cpu is not in idle, do nothing here */
1868 		raw_spin_unlock_irqrestore(&rq->lock, flags);
1869 	}
1870 
1871 out:
1872 	rcu_read_unlock();
1873 }
1874 
1875 bool cpus_share_cache(int this_cpu, int that_cpu)
1876 {
1877 	return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
1878 }
1879 #endif /* CONFIG_SMP */
1880 
1881 static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags)
1882 {
1883 	struct rq *rq = cpu_rq(cpu);
1884 	struct pin_cookie cookie;
1885 
1886 #if defined(CONFIG_SMP)
1887 	if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) {
1888 		sched_clock_cpu(cpu); /* sync clocks x-cpu */
1889 		ttwu_queue_remote(p, cpu, wake_flags);
1890 		return;
1891 	}
1892 #endif
1893 
1894 	raw_spin_lock(&rq->lock);
1895 	cookie = lockdep_pin_lock(&rq->lock);
1896 	ttwu_do_activate(rq, p, wake_flags, cookie);
1897 	lockdep_unpin_lock(&rq->lock, cookie);
1898 	raw_spin_unlock(&rq->lock);
1899 }
1900 
1901 /*
1902  * Notes on Program-Order guarantees on SMP systems.
1903  *
1904  *  MIGRATION
1905  *
1906  * The basic program-order guarantee on SMP systems is that when a task [t]
1907  * migrates, all its activity on its old cpu [c0] happens-before any subsequent
1908  * execution on its new cpu [c1].
1909  *
1910  * For migration (of runnable tasks) this is provided by the following means:
1911  *
1912  *  A) UNLOCK of the rq(c0)->lock scheduling out task t
1913  *  B) migration for t is required to synchronize *both* rq(c0)->lock and
1914  *     rq(c1)->lock (if not at the same time, then in that order).
1915  *  C) LOCK of the rq(c1)->lock scheduling in task
1916  *
1917  * Transitivity guarantees that B happens after A and C after B.
1918  * Note: we only require RCpc transitivity.
1919  * Note: the cpu doing B need not be c0 or c1
1920  *
1921  * Example:
1922  *
1923  *   CPU0            CPU1            CPU2
1924  *
1925  *   LOCK rq(0)->lock
1926  *   sched-out X
1927  *   sched-in Y
1928  *   UNLOCK rq(0)->lock
1929  *
1930  *                                   LOCK rq(0)->lock // orders against CPU0
1931  *                                   dequeue X
1932  *                                   UNLOCK rq(0)->lock
1933  *
1934  *                                   LOCK rq(1)->lock
1935  *                                   enqueue X
1936  *                                   UNLOCK rq(1)->lock
1937  *
1938  *                   LOCK rq(1)->lock // orders against CPU2
1939  *                   sched-out Z
1940  *                   sched-in X
1941  *                   UNLOCK rq(1)->lock
1942  *
1943  *
1944  *  BLOCKING -- aka. SLEEP + WAKEUP
1945  *
1946  * For blocking we (obviously) need to provide the same guarantee as for
1947  * migration. However the means are completely different as there is no lock
1948  * chain to provide order. Instead we do:
1949  *
1950  *   1) smp_store_release(X->on_cpu, 0)
1951  *   2) smp_cond_load_acquire(!X->on_cpu)
1952  *
1953  * Example:
1954  *
1955  *   CPU0 (schedule)  CPU1 (try_to_wake_up) CPU2 (schedule)
1956  *
1957  *   LOCK rq(0)->lock LOCK X->pi_lock
1958  *   dequeue X
1959  *   sched-out X
1960  *   smp_store_release(X->on_cpu, 0);
1961  *
1962  *                    smp_cond_load_acquire(&X->on_cpu, !VAL);
1963  *                    X->state = WAKING
1964  *                    set_task_cpu(X,2)
1965  *
1966  *                    LOCK rq(2)->lock
1967  *                    enqueue X
1968  *                    X->state = RUNNING
1969  *                    UNLOCK rq(2)->lock
1970  *
1971  *                                          LOCK rq(2)->lock // orders against CPU1
1972  *                                          sched-out Z
1973  *                                          sched-in X
1974  *                                          UNLOCK rq(2)->lock
1975  *
1976  *                    UNLOCK X->pi_lock
1977  *   UNLOCK rq(0)->lock
1978  *
1979  *
1980  * However; for wakeups there is a second guarantee we must provide, namely we
1981  * must observe the state that lead to our wakeup. That is, not only must our
1982  * task observe its own prior state, it must also observe the stores prior to
1983  * its wakeup.
1984  *
1985  * This means that any means of doing remote wakeups must order the CPU doing
1986  * the wakeup against the CPU the task is going to end up running on. This,
1987  * however, is already required for the regular Program-Order guarantee above,
1988  * since the waking CPU is the one issueing the ACQUIRE (smp_cond_load_acquire).
1989  *
1990  */
1991 
1992 /**
1993  * try_to_wake_up - wake up a thread
1994  * @p: the thread to be awakened
1995  * @state: the mask of task states that can be woken
1996  * @wake_flags: wake modifier flags (WF_*)
1997  *
1998  * Put it on the run-queue if it's not already there. The "current"
1999  * thread is always on the run-queue (except when the actual
2000  * re-schedule is in progress), and as such you're allowed to do
2001  * the simpler "current->state = TASK_RUNNING" to mark yourself
2002  * runnable without the overhead of this.
2003  *
2004  * Return: %true if @p was woken up, %false if it was already running.
2005  * or @state didn't match @p's state.
2006  */
2007 static int
2008 try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
2009 {
2010 	unsigned long flags;
2011 	int cpu, success = 0;
2012 
2013 	/*
2014 	 * If we are going to wake up a thread waiting for CONDITION we
2015 	 * need to ensure that CONDITION=1 done by the caller can not be
2016 	 * reordered with p->state check below. This pairs with mb() in
2017 	 * set_current_state() the waiting thread does.
2018 	 */
2019 	smp_mb__before_spinlock();
2020 	raw_spin_lock_irqsave(&p->pi_lock, flags);
2021 	if (!(p->state & state))
2022 		goto out;
2023 
2024 	trace_sched_waking(p);
2025 
2026 	success = 1; /* we're going to change ->state */
2027 	cpu = task_cpu(p);
2028 
2029 	/*
2030 	 * Ensure we load p->on_rq _after_ p->state, otherwise it would
2031 	 * be possible to, falsely, observe p->on_rq == 0 and get stuck
2032 	 * in smp_cond_load_acquire() below.
2033 	 *
2034 	 * sched_ttwu_pending()                 try_to_wake_up()
2035 	 *   [S] p->on_rq = 1;                  [L] P->state
2036 	 *       UNLOCK rq->lock  -----.
2037 	 *                              \
2038 	 *				 +---   RMB
2039 	 * schedule()                   /
2040 	 *       LOCK rq->lock    -----'
2041 	 *       UNLOCK rq->lock
2042 	 *
2043 	 * [task p]
2044 	 *   [S] p->state = UNINTERRUPTIBLE     [L] p->on_rq
2045 	 *
2046 	 * Pairs with the UNLOCK+LOCK on rq->lock from the
2047 	 * last wakeup of our task and the schedule that got our task
2048 	 * current.
2049 	 */
2050 	smp_rmb();
2051 	if (p->on_rq && ttwu_remote(p, wake_flags))
2052 		goto stat;
2053 
2054 #ifdef CONFIG_SMP
2055 	/*
2056 	 * Ensure we load p->on_cpu _after_ p->on_rq, otherwise it would be
2057 	 * possible to, falsely, observe p->on_cpu == 0.
2058 	 *
2059 	 * One must be running (->on_cpu == 1) in order to remove oneself
2060 	 * from the runqueue.
2061 	 *
2062 	 *  [S] ->on_cpu = 1;	[L] ->on_rq
2063 	 *      UNLOCK rq->lock
2064 	 *			RMB
2065 	 *      LOCK   rq->lock
2066 	 *  [S] ->on_rq = 0;    [L] ->on_cpu
2067 	 *
2068 	 * Pairs with the full barrier implied in the UNLOCK+LOCK on rq->lock
2069 	 * from the consecutive calls to schedule(); the first switching to our
2070 	 * task, the second putting it to sleep.
2071 	 */
2072 	smp_rmb();
2073 
2074 	/*
2075 	 * If the owning (remote) cpu is still in the middle of schedule() with
2076 	 * this task as prev, wait until its done referencing the task.
2077 	 *
2078 	 * Pairs with the smp_store_release() in finish_lock_switch().
2079 	 *
2080 	 * This ensures that tasks getting woken will be fully ordered against
2081 	 * their previous state and preserve Program Order.
2082 	 */
2083 	smp_cond_load_acquire(&p->on_cpu, !VAL);
2084 
2085 	p->sched_contributes_to_load = !!task_contributes_to_load(p);
2086 	p->state = TASK_WAKING;
2087 
2088 	cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags);
2089 	if (task_cpu(p) != cpu) {
2090 		wake_flags |= WF_MIGRATED;
2091 		set_task_cpu(p, cpu);
2092 	}
2093 #endif /* CONFIG_SMP */
2094 
2095 	ttwu_queue(p, cpu, wake_flags);
2096 stat:
2097 	ttwu_stat(p, cpu, wake_flags);
2098 out:
2099 	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
2100 
2101 	return success;
2102 }
2103 
2104 /**
2105  * try_to_wake_up_local - try to wake up a local task with rq lock held
2106  * @p: the thread to be awakened
2107  * @cookie: context's cookie for pinning
2108  *
2109  * Put @p on the run-queue if it's not already there. The caller must
2110  * ensure that this_rq() is locked, @p is bound to this_rq() and not
2111  * the current task.
2112  */
2113 static void try_to_wake_up_local(struct task_struct *p, struct pin_cookie cookie)
2114 {
2115 	struct rq *rq = task_rq(p);
2116 
2117 	if (WARN_ON_ONCE(rq != this_rq()) ||
2118 	    WARN_ON_ONCE(p == current))
2119 		return;
2120 
2121 	lockdep_assert_held(&rq->lock);
2122 
2123 	if (!raw_spin_trylock(&p->pi_lock)) {
2124 		/*
2125 		 * This is OK, because current is on_cpu, which avoids it being
2126 		 * picked for load-balance and preemption/IRQs are still
2127 		 * disabled avoiding further scheduler activity on it and we've
2128 		 * not yet picked a replacement task.
2129 		 */
2130 		lockdep_unpin_lock(&rq->lock, cookie);
2131 		raw_spin_unlock(&rq->lock);
2132 		raw_spin_lock(&p->pi_lock);
2133 		raw_spin_lock(&rq->lock);
2134 		lockdep_repin_lock(&rq->lock, cookie);
2135 	}
2136 
2137 	if (!(p->state & TASK_NORMAL))
2138 		goto out;
2139 
2140 	trace_sched_waking(p);
2141 
2142 	if (!task_on_rq_queued(p))
2143 		ttwu_activate(rq, p, ENQUEUE_WAKEUP);
2144 
2145 	ttwu_do_wakeup(rq, p, 0, cookie);
2146 	ttwu_stat(p, smp_processor_id(), 0);
2147 out:
2148 	raw_spin_unlock(&p->pi_lock);
2149 }
2150 
2151 /**
2152  * wake_up_process - Wake up a specific process
2153  * @p: The process to be woken up.
2154  *
2155  * Attempt to wake up the nominated process and move it to the set of runnable
2156  * processes.
2157  *
2158  * Return: 1 if the process was woken up, 0 if it was already running.
2159  *
2160  * It may be assumed that this function implies a write memory barrier before
2161  * changing the task state if and only if any tasks are woken up.
2162  */
2163 int wake_up_process(struct task_struct *p)
2164 {
2165 	return try_to_wake_up(p, TASK_NORMAL, 0);
2166 }
2167 EXPORT_SYMBOL(wake_up_process);
2168 
2169 int wake_up_state(struct task_struct *p, unsigned int state)
2170 {
2171 	return try_to_wake_up(p, state, 0);
2172 }
2173 
2174 /*
2175  * This function clears the sched_dl_entity static params.
2176  */
2177 void __dl_clear_params(struct task_struct *p)
2178 {
2179 	struct sched_dl_entity *dl_se = &p->dl;
2180 
2181 	dl_se->dl_runtime = 0;
2182 	dl_se->dl_deadline = 0;
2183 	dl_se->dl_period = 0;
2184 	dl_se->flags = 0;
2185 	dl_se->dl_bw = 0;
2186 
2187 	dl_se->dl_throttled = 0;
2188 	dl_se->dl_yielded = 0;
2189 }
2190 
2191 /*
2192  * Perform scheduler related setup for a newly forked process p.
2193  * p is forked by current.
2194  *
2195  * __sched_fork() is basic setup used by init_idle() too:
2196  */
2197 static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
2198 {
2199 	p->on_rq			= 0;
2200 
2201 	p->se.on_rq			= 0;
2202 	p->se.exec_start		= 0;
2203 	p->se.sum_exec_runtime		= 0;
2204 	p->se.prev_sum_exec_runtime	= 0;
2205 	p->se.nr_migrations		= 0;
2206 	p->se.vruntime			= 0;
2207 	INIT_LIST_HEAD(&p->se.group_node);
2208 
2209 #ifdef CONFIG_FAIR_GROUP_SCHED
2210 	p->se.cfs_rq			= NULL;
2211 #endif
2212 
2213 #ifdef CONFIG_SCHEDSTATS
2214 	/* Even if schedstat is disabled, there should not be garbage */
2215 	memset(&p->se.statistics, 0, sizeof(p->se.statistics));
2216 #endif
2217 
2218 	RB_CLEAR_NODE(&p->dl.rb_node);
2219 	init_dl_task_timer(&p->dl);
2220 	__dl_clear_params(p);
2221 
2222 	INIT_LIST_HEAD(&p->rt.run_list);
2223 	p->rt.timeout		= 0;
2224 	p->rt.time_slice	= sched_rr_timeslice;
2225 	p->rt.on_rq		= 0;
2226 	p->rt.on_list		= 0;
2227 
2228 #ifdef CONFIG_PREEMPT_NOTIFIERS
2229 	INIT_HLIST_HEAD(&p->preempt_notifiers);
2230 #endif
2231 
2232 #ifdef CONFIG_NUMA_BALANCING
2233 	if (p->mm && atomic_read(&p->mm->mm_users) == 1) {
2234 		p->mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay);
2235 		p->mm->numa_scan_seq = 0;
2236 	}
2237 
2238 	if (clone_flags & CLONE_VM)
2239 		p->numa_preferred_nid = current->numa_preferred_nid;
2240 	else
2241 		p->numa_preferred_nid = -1;
2242 
2243 	p->node_stamp = 0ULL;
2244 	p->numa_scan_seq = p->mm ? p->mm->numa_scan_seq : 0;
2245 	p->numa_scan_period = sysctl_numa_balancing_scan_delay;
2246 	p->numa_work.next = &p->numa_work;
2247 	p->numa_faults = NULL;
2248 	p->last_task_numa_placement = 0;
2249 	p->last_sum_exec_runtime = 0;
2250 
2251 	p->numa_group = NULL;
2252 #endif /* CONFIG_NUMA_BALANCING */
2253 }
2254 
2255 DEFINE_STATIC_KEY_FALSE(sched_numa_balancing);
2256 
2257 #ifdef CONFIG_NUMA_BALANCING
2258 
2259 void set_numabalancing_state(bool enabled)
2260 {
2261 	if (enabled)
2262 		static_branch_enable(&sched_numa_balancing);
2263 	else
2264 		static_branch_disable(&sched_numa_balancing);
2265 }
2266 
2267 #ifdef CONFIG_PROC_SYSCTL
2268 int sysctl_numa_balancing(struct ctl_table *table, int write,
2269 			 void __user *buffer, size_t *lenp, loff_t *ppos)
2270 {
2271 	struct ctl_table t;
2272 	int err;
2273 	int state = static_branch_likely(&sched_numa_balancing);
2274 
2275 	if (write && !capable(CAP_SYS_ADMIN))
2276 		return -EPERM;
2277 
2278 	t = *table;
2279 	t.data = &state;
2280 	err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
2281 	if (err < 0)
2282 		return err;
2283 	if (write)
2284 		set_numabalancing_state(state);
2285 	return err;
2286 }
2287 #endif
2288 #endif
2289 
2290 #ifdef CONFIG_SCHEDSTATS
2291 
2292 DEFINE_STATIC_KEY_FALSE(sched_schedstats);
2293 static bool __initdata __sched_schedstats = false;
2294 
2295 static void set_schedstats(bool enabled)
2296 {
2297 	if (enabled)
2298 		static_branch_enable(&sched_schedstats);
2299 	else
2300 		static_branch_disable(&sched_schedstats);
2301 }
2302 
2303 void force_schedstat_enabled(void)
2304 {
2305 	if (!schedstat_enabled()) {
2306 		pr_info("kernel profiling enabled schedstats, disable via kernel.sched_schedstats.\n");
2307 		static_branch_enable(&sched_schedstats);
2308 	}
2309 }
2310 
2311 static int __init setup_schedstats(char *str)
2312 {
2313 	int ret = 0;
2314 	if (!str)
2315 		goto out;
2316 
2317 	/*
2318 	 * This code is called before jump labels have been set up, so we can't
2319 	 * change the static branch directly just yet.  Instead set a temporary
2320 	 * variable so init_schedstats() can do it later.
2321 	 */
2322 	if (!strcmp(str, "enable")) {
2323 		__sched_schedstats = true;
2324 		ret = 1;
2325 	} else if (!strcmp(str, "disable")) {
2326 		__sched_schedstats = false;
2327 		ret = 1;
2328 	}
2329 out:
2330 	if (!ret)
2331 		pr_warn("Unable to parse schedstats=\n");
2332 
2333 	return ret;
2334 }
2335 __setup("schedstats=", setup_schedstats);
2336 
2337 static void __init init_schedstats(void)
2338 {
2339 	set_schedstats(__sched_schedstats);
2340 }
2341 
2342 #ifdef CONFIG_PROC_SYSCTL
2343 int sysctl_schedstats(struct ctl_table *table, int write,
2344 			 void __user *buffer, size_t *lenp, loff_t *ppos)
2345 {
2346 	struct ctl_table t;
2347 	int err;
2348 	int state = static_branch_likely(&sched_schedstats);
2349 
2350 	if (write && !capable(CAP_SYS_ADMIN))
2351 		return -EPERM;
2352 
2353 	t = *table;
2354 	t.data = &state;
2355 	err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
2356 	if (err < 0)
2357 		return err;
2358 	if (write)
2359 		set_schedstats(state);
2360 	return err;
2361 }
2362 #endif /* CONFIG_PROC_SYSCTL */
2363 #else  /* !CONFIG_SCHEDSTATS */
2364 static inline void init_schedstats(void) {}
2365 #endif /* CONFIG_SCHEDSTATS */
2366 
2367 /*
2368  * fork()/clone()-time setup:
2369  */
2370 int sched_fork(unsigned long clone_flags, struct task_struct *p)
2371 {
2372 	unsigned long flags;
2373 	int cpu = get_cpu();
2374 
2375 	__sched_fork(clone_flags, p);
2376 	/*
2377 	 * We mark the process as NEW here. This guarantees that
2378 	 * nobody will actually run it, and a signal or other external
2379 	 * event cannot wake it up and insert it on the runqueue either.
2380 	 */
2381 	p->state = TASK_NEW;
2382 
2383 	/*
2384 	 * Make sure we do not leak PI boosting priority to the child.
2385 	 */
2386 	p->prio = current->normal_prio;
2387 
2388 	/*
2389 	 * Revert to default priority/policy on fork if requested.
2390 	 */
2391 	if (unlikely(p->sched_reset_on_fork)) {
2392 		if (task_has_dl_policy(p) || task_has_rt_policy(p)) {
2393 			p->policy = SCHED_NORMAL;
2394 			p->static_prio = NICE_TO_PRIO(0);
2395 			p->rt_priority = 0;
2396 		} else if (PRIO_TO_NICE(p->static_prio) < 0)
2397 			p->static_prio = NICE_TO_PRIO(0);
2398 
2399 		p->prio = p->normal_prio = __normal_prio(p);
2400 		set_load_weight(p);
2401 
2402 		/*
2403 		 * We don't need the reset flag anymore after the fork. It has
2404 		 * fulfilled its duty:
2405 		 */
2406 		p->sched_reset_on_fork = 0;
2407 	}
2408 
2409 	if (dl_prio(p->prio)) {
2410 		put_cpu();
2411 		return -EAGAIN;
2412 	} else if (rt_prio(p->prio)) {
2413 		p->sched_class = &rt_sched_class;
2414 	} else {
2415 		p->sched_class = &fair_sched_class;
2416 	}
2417 
2418 	init_entity_runnable_average(&p->se);
2419 
2420 	/*
2421 	 * The child is not yet in the pid-hash so no cgroup attach races,
2422 	 * and the cgroup is pinned to this child due to cgroup_fork()
2423 	 * is ran before sched_fork().
2424 	 *
2425 	 * Silence PROVE_RCU.
2426 	 */
2427 	raw_spin_lock_irqsave(&p->pi_lock, flags);
2428 	/*
2429 	 * We're setting the cpu for the first time, we don't migrate,
2430 	 * so use __set_task_cpu().
2431 	 */
2432 	__set_task_cpu(p, cpu);
2433 	if (p->sched_class->task_fork)
2434 		p->sched_class->task_fork(p);
2435 	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
2436 
2437 #ifdef CONFIG_SCHED_INFO
2438 	if (likely(sched_info_on()))
2439 		memset(&p->sched_info, 0, sizeof(p->sched_info));
2440 #endif
2441 #if defined(CONFIG_SMP)
2442 	p->on_cpu = 0;
2443 #endif
2444 	init_task_preempt_count(p);
2445 #ifdef CONFIG_SMP
2446 	plist_node_init(&p->pushable_tasks, MAX_PRIO);
2447 	RB_CLEAR_NODE(&p->pushable_dl_tasks);
2448 #endif
2449 
2450 	put_cpu();
2451 	return 0;
2452 }
2453 
2454 unsigned long to_ratio(u64 period, u64 runtime)
2455 {
2456 	if (runtime == RUNTIME_INF)
2457 		return 1ULL << 20;
2458 
2459 	/*
2460 	 * Doing this here saves a lot of checks in all
2461 	 * the calling paths, and returning zero seems
2462 	 * safe for them anyway.
2463 	 */
2464 	if (period == 0)
2465 		return 0;
2466 
2467 	return div64_u64(runtime << 20, period);
2468 }
2469 
2470 #ifdef CONFIG_SMP
2471 inline struct dl_bw *dl_bw_of(int i)
2472 {
2473 	RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held(),
2474 			 "sched RCU must be held");
2475 	return &cpu_rq(i)->rd->dl_bw;
2476 }
2477 
2478 static inline int dl_bw_cpus(int i)
2479 {
2480 	struct root_domain *rd = cpu_rq(i)->rd;
2481 	int cpus = 0;
2482 
2483 	RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held(),
2484 			 "sched RCU must be held");
2485 	for_each_cpu_and(i, rd->span, cpu_active_mask)
2486 		cpus++;
2487 
2488 	return cpus;
2489 }
2490 #else
2491 inline struct dl_bw *dl_bw_of(int i)
2492 {
2493 	return &cpu_rq(i)->dl.dl_bw;
2494 }
2495 
2496 static inline int dl_bw_cpus(int i)
2497 {
2498 	return 1;
2499 }
2500 #endif
2501 
2502 /*
2503  * We must be sure that accepting a new task (or allowing changing the
2504  * parameters of an existing one) is consistent with the bandwidth
2505  * constraints. If yes, this function also accordingly updates the currently
2506  * allocated bandwidth to reflect the new situation.
2507  *
2508  * This function is called while holding p's rq->lock.
2509  *
2510  * XXX we should delay bw change until the task's 0-lag point, see
2511  * __setparam_dl().
2512  */
2513 static int dl_overflow(struct task_struct *p, int policy,
2514 		       const struct sched_attr *attr)
2515 {
2516 
2517 	struct dl_bw *dl_b = dl_bw_of(task_cpu(p));
2518 	u64 period = attr->sched_period ?: attr->sched_deadline;
2519 	u64 runtime = attr->sched_runtime;
2520 	u64 new_bw = dl_policy(policy) ? to_ratio(period, runtime) : 0;
2521 	int cpus, err = -1;
2522 
2523 	/* !deadline task may carry old deadline bandwidth */
2524 	if (new_bw == p->dl.dl_bw && task_has_dl_policy(p))
2525 		return 0;
2526 
2527 	/*
2528 	 * Either if a task, enters, leave, or stays -deadline but changes
2529 	 * its parameters, we may need to update accordingly the total
2530 	 * allocated bandwidth of the container.
2531 	 */
2532 	raw_spin_lock(&dl_b->lock);
2533 	cpus = dl_bw_cpus(task_cpu(p));
2534 	if (dl_policy(policy) && !task_has_dl_policy(p) &&
2535 	    !__dl_overflow(dl_b, cpus, 0, new_bw)) {
2536 		__dl_add(dl_b, new_bw);
2537 		err = 0;
2538 	} else if (dl_policy(policy) && task_has_dl_policy(p) &&
2539 		   !__dl_overflow(dl_b, cpus, p->dl.dl_bw, new_bw)) {
2540 		__dl_clear(dl_b, p->dl.dl_bw);
2541 		__dl_add(dl_b, new_bw);
2542 		err = 0;
2543 	} else if (!dl_policy(policy) && task_has_dl_policy(p)) {
2544 		__dl_clear(dl_b, p->dl.dl_bw);
2545 		err = 0;
2546 	}
2547 	raw_spin_unlock(&dl_b->lock);
2548 
2549 	return err;
2550 }
2551 
2552 extern void init_dl_bw(struct dl_bw *dl_b);
2553 
2554 /*
2555  * wake_up_new_task - wake up a newly created task for the first time.
2556  *
2557  * This function will do some initial scheduler statistics housekeeping
2558  * that must be done for every newly created context, then puts the task
2559  * on the runqueue and wakes it.
2560  */
2561 void wake_up_new_task(struct task_struct *p)
2562 {
2563 	struct rq_flags rf;
2564 	struct rq *rq;
2565 
2566 	raw_spin_lock_irqsave(&p->pi_lock, rf.flags);
2567 	p->state = TASK_RUNNING;
2568 #ifdef CONFIG_SMP
2569 	/*
2570 	 * Fork balancing, do it here and not earlier because:
2571 	 *  - cpus_allowed can change in the fork path
2572 	 *  - any previously selected cpu might disappear through hotplug
2573 	 *
2574 	 * Use __set_task_cpu() to avoid calling sched_class::migrate_task_rq,
2575 	 * as we're not fully set-up yet.
2576 	 */
2577 	__set_task_cpu(p, select_task_rq(p, task_cpu(p), SD_BALANCE_FORK, 0));
2578 #endif
2579 	rq = __task_rq_lock(p, &rf);
2580 	post_init_entity_util_avg(&p->se);
2581 
2582 	activate_task(rq, p, 0);
2583 	p->on_rq = TASK_ON_RQ_QUEUED;
2584 	trace_sched_wakeup_new(p);
2585 	check_preempt_curr(rq, p, WF_FORK);
2586 #ifdef CONFIG_SMP
2587 	if (p->sched_class->task_woken) {
2588 		/*
2589 		 * Nothing relies on rq->lock after this, so its fine to
2590 		 * drop it.
2591 		 */
2592 		lockdep_unpin_lock(&rq->lock, rf.cookie);
2593 		p->sched_class->task_woken(rq, p);
2594 		lockdep_repin_lock(&rq->lock, rf.cookie);
2595 	}
2596 #endif
2597 	task_rq_unlock(rq, p, &rf);
2598 }
2599 
2600 #ifdef CONFIG_PREEMPT_NOTIFIERS
2601 
2602 static struct static_key preempt_notifier_key = STATIC_KEY_INIT_FALSE;
2603 
2604 void preempt_notifier_inc(void)
2605 {
2606 	static_key_slow_inc(&preempt_notifier_key);
2607 }
2608 EXPORT_SYMBOL_GPL(preempt_notifier_inc);
2609 
2610 void preempt_notifier_dec(void)
2611 {
2612 	static_key_slow_dec(&preempt_notifier_key);
2613 }
2614 EXPORT_SYMBOL_GPL(preempt_notifier_dec);
2615 
2616 /**
2617  * preempt_notifier_register - tell me when current is being preempted & rescheduled
2618  * @notifier: notifier struct to register
2619  */
2620 void preempt_notifier_register(struct preempt_notifier *notifier)
2621 {
2622 	if (!static_key_false(&preempt_notifier_key))
2623 		WARN(1, "registering preempt_notifier while notifiers disabled\n");
2624 
2625 	hlist_add_head(&notifier->link, &current->preempt_notifiers);
2626 }
2627 EXPORT_SYMBOL_GPL(preempt_notifier_register);
2628 
2629 /**
2630  * preempt_notifier_unregister - no longer interested in preemption notifications
2631  * @notifier: notifier struct to unregister
2632  *
2633  * This is *not* safe to call from within a preemption notifier.
2634  */
2635 void preempt_notifier_unregister(struct preempt_notifier *notifier)
2636 {
2637 	hlist_del(&notifier->link);
2638 }
2639 EXPORT_SYMBOL_GPL(preempt_notifier_unregister);
2640 
2641 static void __fire_sched_in_preempt_notifiers(struct task_struct *curr)
2642 {
2643 	struct preempt_notifier *notifier;
2644 
2645 	hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
2646 		notifier->ops->sched_in(notifier, raw_smp_processor_id());
2647 }
2648 
2649 static __always_inline void fire_sched_in_preempt_notifiers(struct task_struct *curr)
2650 {
2651 	if (static_key_false(&preempt_notifier_key))
2652 		__fire_sched_in_preempt_notifiers(curr);
2653 }
2654 
2655 static void
2656 __fire_sched_out_preempt_notifiers(struct task_struct *curr,
2657 				   struct task_struct *next)
2658 {
2659 	struct preempt_notifier *notifier;
2660 
2661 	hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
2662 		notifier->ops->sched_out(notifier, next);
2663 }
2664 
2665 static __always_inline void
2666 fire_sched_out_preempt_notifiers(struct task_struct *curr,
2667 				 struct task_struct *next)
2668 {
2669 	if (static_key_false(&preempt_notifier_key))
2670 		__fire_sched_out_preempt_notifiers(curr, next);
2671 }
2672 
2673 #else /* !CONFIG_PREEMPT_NOTIFIERS */
2674 
2675 static inline void fire_sched_in_preempt_notifiers(struct task_struct *curr)
2676 {
2677 }
2678 
2679 static inline void
2680 fire_sched_out_preempt_notifiers(struct task_struct *curr,
2681 				 struct task_struct *next)
2682 {
2683 }
2684 
2685 #endif /* CONFIG_PREEMPT_NOTIFIERS */
2686 
2687 /**
2688  * prepare_task_switch - prepare to switch tasks
2689  * @rq: the runqueue preparing to switch
2690  * @prev: the current task that is being switched out
2691  * @next: the task we are going to switch to.
2692  *
2693  * This is called with the rq lock held and interrupts off. It must
2694  * be paired with a subsequent finish_task_switch after the context
2695  * switch.
2696  *
2697  * prepare_task_switch sets up locking and calls architecture specific
2698  * hooks.
2699  */
2700 static inline void
2701 prepare_task_switch(struct rq *rq, struct task_struct *prev,
2702 		    struct task_struct *next)
2703 {
2704 	sched_info_switch(rq, prev, next);
2705 	perf_event_task_sched_out(prev, next);
2706 	fire_sched_out_preempt_notifiers(prev, next);
2707 	prepare_lock_switch(rq, next);
2708 	prepare_arch_switch(next);
2709 }
2710 
2711 /**
2712  * finish_task_switch - clean up after a task-switch
2713  * @prev: the thread we just switched away from.
2714  *
2715  * finish_task_switch must be called after the context switch, paired
2716  * with a prepare_task_switch call before the context switch.
2717  * finish_task_switch will reconcile locking set up by prepare_task_switch,
2718  * and do any other architecture-specific cleanup actions.
2719  *
2720  * Note that we may have delayed dropping an mm in context_switch(). If
2721  * so, we finish that here outside of the runqueue lock. (Doing it
2722  * with the lock held can cause deadlocks; see schedule() for
2723  * details.)
2724  *
2725  * The context switch have flipped the stack from under us and restored the
2726  * local variables which were saved when this task called schedule() in the
2727  * past. prev == current is still correct but we need to recalculate this_rq
2728  * because prev may have moved to another CPU.
2729  */
2730 static struct rq *finish_task_switch(struct task_struct *prev)
2731 	__releases(rq->lock)
2732 {
2733 	struct rq *rq = this_rq();
2734 	struct mm_struct *mm = rq->prev_mm;
2735 	long prev_state;
2736 
2737 	/*
2738 	 * The previous task will have left us with a preempt_count of 2
2739 	 * because it left us after:
2740 	 *
2741 	 *	schedule()
2742 	 *	  preempt_disable();			// 1
2743 	 *	  __schedule()
2744 	 *	    raw_spin_lock_irq(&rq->lock)	// 2
2745 	 *
2746 	 * Also, see FORK_PREEMPT_COUNT.
2747 	 */
2748 	if (WARN_ONCE(preempt_count() != 2*PREEMPT_DISABLE_OFFSET,
2749 		      "corrupted preempt_count: %s/%d/0x%x\n",
2750 		      current->comm, current->pid, preempt_count()))
2751 		preempt_count_set(FORK_PREEMPT_COUNT);
2752 
2753 	rq->prev_mm = NULL;
2754 
2755 	/*
2756 	 * A task struct has one reference for the use as "current".
2757 	 * If a task dies, then it sets TASK_DEAD in tsk->state and calls
2758 	 * schedule one last time. The schedule call will never return, and
2759 	 * the scheduled task must drop that reference.
2760 	 *
2761 	 * We must observe prev->state before clearing prev->on_cpu (in
2762 	 * finish_lock_switch), otherwise a concurrent wakeup can get prev
2763 	 * running on another CPU and we could rave with its RUNNING -> DEAD
2764 	 * transition, resulting in a double drop.
2765 	 */
2766 	prev_state = prev->state;
2767 	vtime_task_switch(prev);
2768 	perf_event_task_sched_in(prev, current);
2769 	finish_lock_switch(rq, prev);
2770 	finish_arch_post_lock_switch();
2771 
2772 	fire_sched_in_preempt_notifiers(current);
2773 	if (mm)
2774 		mmdrop(mm);
2775 	if (unlikely(prev_state == TASK_DEAD)) {
2776 		if (prev->sched_class->task_dead)
2777 			prev->sched_class->task_dead(prev);
2778 
2779 		/*
2780 		 * Remove function-return probe instances associated with this
2781 		 * task and put them back on the free list.
2782 		 */
2783 		kprobe_flush_task(prev);
2784 
2785 		/* Task is done with its stack. */
2786 		put_task_stack(prev);
2787 
2788 		put_task_struct(prev);
2789 	}
2790 
2791 	tick_nohz_task_switch();
2792 	return rq;
2793 }
2794 
2795 #ifdef CONFIG_SMP
2796 
2797 /* rq->lock is NOT held, but preemption is disabled */
2798 static void __balance_callback(struct rq *rq)
2799 {
2800 	struct callback_head *head, *next;
2801 	void (*func)(struct rq *rq);
2802 	unsigned long flags;
2803 
2804 	raw_spin_lock_irqsave(&rq->lock, flags);
2805 	head = rq->balance_callback;
2806 	rq->balance_callback = NULL;
2807 	while (head) {
2808 		func = (void (*)(struct rq *))head->func;
2809 		next = head->next;
2810 		head->next = NULL;
2811 		head = next;
2812 
2813 		func(rq);
2814 	}
2815 	raw_spin_unlock_irqrestore(&rq->lock, flags);
2816 }
2817 
2818 static inline void balance_callback(struct rq *rq)
2819 {
2820 	if (unlikely(rq->balance_callback))
2821 		__balance_callback(rq);
2822 }
2823 
2824 #else
2825 
2826 static inline void balance_callback(struct rq *rq)
2827 {
2828 }
2829 
2830 #endif
2831 
2832 /**
2833  * schedule_tail - first thing a freshly forked thread must call.
2834  * @prev: the thread we just switched away from.
2835  */
2836 asmlinkage __visible void schedule_tail(struct task_struct *prev)
2837 	__releases(rq->lock)
2838 {
2839 	struct rq *rq;
2840 
2841 	/*
2842 	 * New tasks start with FORK_PREEMPT_COUNT, see there and
2843 	 * finish_task_switch() for details.
2844 	 *
2845 	 * finish_task_switch() will drop rq->lock() and lower preempt_count
2846 	 * and the preempt_enable() will end up enabling preemption (on
2847 	 * PREEMPT_COUNT kernels).
2848 	 */
2849 
2850 	rq = finish_task_switch(prev);
2851 	balance_callback(rq);
2852 	preempt_enable();
2853 
2854 	if (current->set_child_tid)
2855 		put_user(task_pid_vnr(current), current->set_child_tid);
2856 }
2857 
2858 /*
2859  * context_switch - switch to the new MM and the new thread's register state.
2860  */
2861 static __always_inline struct rq *
2862 context_switch(struct rq *rq, struct task_struct *prev,
2863 	       struct task_struct *next, struct pin_cookie cookie)
2864 {
2865 	struct mm_struct *mm, *oldmm;
2866 
2867 	prepare_task_switch(rq, prev, next);
2868 
2869 	mm = next->mm;
2870 	oldmm = prev->active_mm;
2871 	/*
2872 	 * For paravirt, this is coupled with an exit in switch_to to
2873 	 * combine the page table reload and the switch backend into
2874 	 * one hypercall.
2875 	 */
2876 	arch_start_context_switch(prev);
2877 
2878 	if (!mm) {
2879 		next->active_mm = oldmm;
2880 		atomic_inc(&oldmm->mm_count);
2881 		enter_lazy_tlb(oldmm, next);
2882 	} else
2883 		switch_mm_irqs_off(oldmm, mm, next);
2884 
2885 	if (!prev->mm) {
2886 		prev->active_mm = NULL;
2887 		rq->prev_mm = oldmm;
2888 	}
2889 	/*
2890 	 * Since the runqueue lock will be released by the next
2891 	 * task (which is an invalid locking op but in the case
2892 	 * of the scheduler it's an obvious special-case), so we
2893 	 * do an early lockdep release here:
2894 	 */
2895 	lockdep_unpin_lock(&rq->lock, cookie);
2896 	spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
2897 
2898 	/* Here we just switch the register state and the stack. */
2899 	switch_to(prev, next, prev);
2900 	barrier();
2901 
2902 	return finish_task_switch(prev);
2903 }
2904 
2905 /*
2906  * nr_running and nr_context_switches:
2907  *
2908  * externally visible scheduler statistics: current number of runnable
2909  * threads, total number of context switches performed since bootup.
2910  */
2911 unsigned long nr_running(void)
2912 {
2913 	unsigned long i, sum = 0;
2914 
2915 	for_each_online_cpu(i)
2916 		sum += cpu_rq(i)->nr_running;
2917 
2918 	return sum;
2919 }
2920 
2921 /*
2922  * Check if only the current task is running on the cpu.
2923  *
2924  * Caution: this function does not check that the caller has disabled
2925  * preemption, thus the result might have a time-of-check-to-time-of-use
2926  * race.  The caller is responsible to use it correctly, for example:
2927  *
2928  * - from a non-preemptable section (of course)
2929  *
2930  * - from a thread that is bound to a single CPU
2931  *
2932  * - in a loop with very short iterations (e.g. a polling loop)
2933  */
2934 bool single_task_running(void)
2935 {
2936 	return raw_rq()->nr_running == 1;
2937 }
2938 EXPORT_SYMBOL(single_task_running);
2939 
2940 unsigned long long nr_context_switches(void)
2941 {
2942 	int i;
2943 	unsigned long long sum = 0;
2944 
2945 	for_each_possible_cpu(i)
2946 		sum += cpu_rq(i)->nr_switches;
2947 
2948 	return sum;
2949 }
2950 
2951 unsigned long nr_iowait(void)
2952 {
2953 	unsigned long i, sum = 0;
2954 
2955 	for_each_possible_cpu(i)
2956 		sum += atomic_read(&cpu_rq(i)->nr_iowait);
2957 
2958 	return sum;
2959 }
2960 
2961 unsigned long nr_iowait_cpu(int cpu)
2962 {
2963 	struct rq *this = cpu_rq(cpu);
2964 	return atomic_read(&this->nr_iowait);
2965 }
2966 
2967 void get_iowait_load(unsigned long *nr_waiters, unsigned long *load)
2968 {
2969 	struct rq *rq = this_rq();
2970 	*nr_waiters = atomic_read(&rq->nr_iowait);
2971 	*load = rq->load.weight;
2972 }
2973 
2974 #ifdef CONFIG_SMP
2975 
2976 /*
2977  * sched_exec - execve() is a valuable balancing opportunity, because at
2978  * this point the task has the smallest effective memory and cache footprint.
2979  */
2980 void sched_exec(void)
2981 {
2982 	struct task_struct *p = current;
2983 	unsigned long flags;
2984 	int dest_cpu;
2985 
2986 	raw_spin_lock_irqsave(&p->pi_lock, flags);
2987 	dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), SD_BALANCE_EXEC, 0);
2988 	if (dest_cpu == smp_processor_id())
2989 		goto unlock;
2990 
2991 	if (likely(cpu_active(dest_cpu))) {
2992 		struct migration_arg arg = { p, dest_cpu };
2993 
2994 		raw_spin_unlock_irqrestore(&p->pi_lock, flags);
2995 		stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg);
2996 		return;
2997 	}
2998 unlock:
2999 	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
3000 }
3001 
3002 #endif
3003 
3004 DEFINE_PER_CPU(struct kernel_stat, kstat);
3005 DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat);
3006 
3007 EXPORT_PER_CPU_SYMBOL(kstat);
3008 EXPORT_PER_CPU_SYMBOL(kernel_cpustat);
3009 
3010 /*
3011  * The function fair_sched_class.update_curr accesses the struct curr
3012  * and its field curr->exec_start; when called from task_sched_runtime(),
3013  * we observe a high rate of cache misses in practice.
3014  * Prefetching this data results in improved performance.
3015  */
3016 static inline void prefetch_curr_exec_start(struct task_struct *p)
3017 {
3018 #ifdef CONFIG_FAIR_GROUP_SCHED
3019 	struct sched_entity *curr = (&p->se)->cfs_rq->curr;
3020 #else
3021 	struct sched_entity *curr = (&task_rq(p)->cfs)->curr;
3022 #endif
3023 	prefetch(curr);
3024 	prefetch(&curr->exec_start);
3025 }
3026 
3027 /*
3028  * Return accounted runtime for the task.
3029  * In case the task is currently running, return the runtime plus current's
3030  * pending runtime that have not been accounted yet.
3031  */
3032 unsigned long long task_sched_runtime(struct task_struct *p)
3033 {
3034 	struct rq_flags rf;
3035 	struct rq *rq;
3036 	u64 ns;
3037 
3038 #if defined(CONFIG_64BIT) && defined(CONFIG_SMP)
3039 	/*
3040 	 * 64-bit doesn't need locks to atomically read a 64bit value.
3041 	 * So we have a optimization chance when the task's delta_exec is 0.
3042 	 * Reading ->on_cpu is racy, but this is ok.
3043 	 *
3044 	 * If we race with it leaving cpu, we'll take a lock. So we're correct.
3045 	 * If we race with it entering cpu, unaccounted time is 0. This is
3046 	 * indistinguishable from the read occurring a few cycles earlier.
3047 	 * If we see ->on_cpu without ->on_rq, the task is leaving, and has
3048 	 * been accounted, so we're correct here as well.
3049 	 */
3050 	if (!p->on_cpu || !task_on_rq_queued(p))
3051 		return p->se.sum_exec_runtime;
3052 #endif
3053 
3054 	rq = task_rq_lock(p, &rf);
3055 	/*
3056 	 * Must be ->curr _and_ ->on_rq.  If dequeued, we would
3057 	 * project cycles that may never be accounted to this
3058 	 * thread, breaking clock_gettime().
3059 	 */
3060 	if (task_current(rq, p) && task_on_rq_queued(p)) {
3061 		prefetch_curr_exec_start(p);
3062 		update_rq_clock(rq);
3063 		p->sched_class->update_curr(rq);
3064 	}
3065 	ns = p->se.sum_exec_runtime;
3066 	task_rq_unlock(rq, p, &rf);
3067 
3068 	return ns;
3069 }
3070 
3071 /*
3072  * This function gets called by the timer code, with HZ frequency.
3073  * We call it with interrupts disabled.
3074  */
3075 void scheduler_tick(void)
3076 {
3077 	int cpu = smp_processor_id();
3078 	struct rq *rq = cpu_rq(cpu);
3079 	struct task_struct *curr = rq->curr;
3080 
3081 	sched_clock_tick();
3082 
3083 	raw_spin_lock(&rq->lock);
3084 	update_rq_clock(rq);
3085 	curr->sched_class->task_tick(rq, curr, 0);
3086 	cpu_load_update_active(rq);
3087 	calc_global_load_tick(rq);
3088 	raw_spin_unlock(&rq->lock);
3089 
3090 	perf_event_task_tick();
3091 
3092 #ifdef CONFIG_SMP
3093 	rq->idle_balance = idle_cpu(cpu);
3094 	trigger_load_balance(rq);
3095 #endif
3096 	rq_last_tick_reset(rq);
3097 }
3098 
3099 #ifdef CONFIG_NO_HZ_FULL
3100 /**
3101  * scheduler_tick_max_deferment
3102  *
3103  * Keep at least one tick per second when a single
3104  * active task is running because the scheduler doesn't
3105  * yet completely support full dynticks environment.
3106  *
3107  * This makes sure that uptime, CFS vruntime, load
3108  * balancing, etc... continue to move forward, even
3109  * with a very low granularity.
3110  *
3111  * Return: Maximum deferment in nanoseconds.
3112  */
3113 u64 scheduler_tick_max_deferment(void)
3114 {
3115 	struct rq *rq = this_rq();
3116 	unsigned long next, now = READ_ONCE(jiffies);
3117 
3118 	next = rq->last_sched_tick + HZ;
3119 
3120 	if (time_before_eq(next, now))
3121 		return 0;
3122 
3123 	return jiffies_to_nsecs(next - now);
3124 }
3125 #endif
3126 
3127 #if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
3128 				defined(CONFIG_PREEMPT_TRACER))
3129 /*
3130  * If the value passed in is equal to the current preempt count
3131  * then we just disabled preemption. Start timing the latency.
3132  */
3133 static inline void preempt_latency_start(int val)
3134 {
3135 	if (preempt_count() == val) {
3136 		unsigned long ip = get_lock_parent_ip();
3137 #ifdef CONFIG_DEBUG_PREEMPT
3138 		current->preempt_disable_ip = ip;
3139 #endif
3140 		trace_preempt_off(CALLER_ADDR0, ip);
3141 	}
3142 }
3143 
3144 void preempt_count_add(int val)
3145 {
3146 #ifdef CONFIG_DEBUG_PREEMPT
3147 	/*
3148 	 * Underflow?
3149 	 */
3150 	if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
3151 		return;
3152 #endif
3153 	__preempt_count_add(val);
3154 #ifdef CONFIG_DEBUG_PREEMPT
3155 	/*
3156 	 * Spinlock count overflowing soon?
3157 	 */
3158 	DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
3159 				PREEMPT_MASK - 10);
3160 #endif
3161 	preempt_latency_start(val);
3162 }
3163 EXPORT_SYMBOL(preempt_count_add);
3164 NOKPROBE_SYMBOL(preempt_count_add);
3165 
3166 /*
3167  * If the value passed in equals to the current preempt count
3168  * then we just enabled preemption. Stop timing the latency.
3169  */
3170 static inline void preempt_latency_stop(int val)
3171 {
3172 	if (preempt_count() == val)
3173 		trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip());
3174 }
3175 
3176 void preempt_count_sub(int val)
3177 {
3178 #ifdef CONFIG_DEBUG_PREEMPT
3179 	/*
3180 	 * Underflow?
3181 	 */
3182 	if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
3183 		return;
3184 	/*
3185 	 * Is the spinlock portion underflowing?
3186 	 */
3187 	if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
3188 			!(preempt_count() & PREEMPT_MASK)))
3189 		return;
3190 #endif
3191 
3192 	preempt_latency_stop(val);
3193 	__preempt_count_sub(val);
3194 }
3195 EXPORT_SYMBOL(preempt_count_sub);
3196 NOKPROBE_SYMBOL(preempt_count_sub);
3197 
3198 #else
3199 static inline void preempt_latency_start(int val) { }
3200 static inline void preempt_latency_stop(int val) { }
3201 #endif
3202 
3203 /*
3204  * Print scheduling while atomic bug:
3205  */
3206 static noinline void __schedule_bug(struct task_struct *prev)
3207 {
3208 	/* Save this before calling printk(), since that will clobber it */
3209 	unsigned long preempt_disable_ip = get_preempt_disable_ip(current);
3210 
3211 	if (oops_in_progress)
3212 		return;
3213 
3214 	printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
3215 		prev->comm, prev->pid, preempt_count());
3216 
3217 	debug_show_held_locks(prev);
3218 	print_modules();
3219 	if (irqs_disabled())
3220 		print_irqtrace_events(prev);
3221 	if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)
3222 	    && in_atomic_preempt_off()) {
3223 		pr_err("Preemption disabled at:");
3224 		print_ip_sym(preempt_disable_ip);
3225 		pr_cont("\n");
3226 	}
3227 	if (panic_on_warn)
3228 		panic("scheduling while atomic\n");
3229 
3230 	dump_stack();
3231 	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
3232 }
3233 
3234 /*
3235  * Various schedule()-time debugging checks and statistics:
3236  */
3237 static inline void schedule_debug(struct task_struct *prev)
3238 {
3239 #ifdef CONFIG_SCHED_STACK_END_CHECK
3240 	if (task_stack_end_corrupted(prev))
3241 		panic("corrupted stack end detected inside scheduler\n");
3242 #endif
3243 
3244 	if (unlikely(in_atomic_preempt_off())) {
3245 		__schedule_bug(prev);
3246 		preempt_count_set(PREEMPT_DISABLED);
3247 	}
3248 	rcu_sleep_check();
3249 
3250 	profile_hit(SCHED_PROFILING, __builtin_return_address(0));
3251 
3252 	schedstat_inc(this_rq()->sched_count);
3253 }
3254 
3255 /*
3256  * Pick up the highest-prio task:
3257  */
3258 static inline struct task_struct *
3259 pick_next_task(struct rq *rq, struct task_struct *prev, struct pin_cookie cookie)
3260 {
3261 	const struct sched_class *class = &fair_sched_class;
3262 	struct task_struct *p;
3263 
3264 	/*
3265 	 * Optimization: we know that if all tasks are in
3266 	 * the fair class we can call that function directly:
3267 	 */
3268 	if (likely(prev->sched_class == class &&
3269 		   rq->nr_running == rq->cfs.h_nr_running)) {
3270 		p = fair_sched_class.pick_next_task(rq, prev, cookie);
3271 		if (unlikely(p == RETRY_TASK))
3272 			goto again;
3273 
3274 		/* assumes fair_sched_class->next == idle_sched_class */
3275 		if (unlikely(!p))
3276 			p = idle_sched_class.pick_next_task(rq, prev, cookie);
3277 
3278 		return p;
3279 	}
3280 
3281 again:
3282 	for_each_class(class) {
3283 		p = class->pick_next_task(rq, prev, cookie);
3284 		if (p) {
3285 			if (unlikely(p == RETRY_TASK))
3286 				goto again;
3287 			return p;
3288 		}
3289 	}
3290 
3291 	BUG(); /* the idle class will always have a runnable task */
3292 }
3293 
3294 /*
3295  * __schedule() is the main scheduler function.
3296  *
3297  * The main means of driving the scheduler and thus entering this function are:
3298  *
3299  *   1. Explicit blocking: mutex, semaphore, waitqueue, etc.
3300  *
3301  *   2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return
3302  *      paths. For example, see arch/x86/entry_64.S.
3303  *
3304  *      To drive preemption between tasks, the scheduler sets the flag in timer
3305  *      interrupt handler scheduler_tick().
3306  *
3307  *   3. Wakeups don't really cause entry into schedule(). They add a
3308  *      task to the run-queue and that's it.
3309  *
3310  *      Now, if the new task added to the run-queue preempts the current
3311  *      task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets
3312  *      called on the nearest possible occasion:
3313  *
3314  *       - If the kernel is preemptible (CONFIG_PREEMPT=y):
3315  *
3316  *         - in syscall or exception context, at the next outmost
3317  *           preempt_enable(). (this might be as soon as the wake_up()'s
3318  *           spin_unlock()!)
3319  *
3320  *         - in IRQ context, return from interrupt-handler to
3321  *           preemptible context
3322  *
3323  *       - If the kernel is not preemptible (CONFIG_PREEMPT is not set)
3324  *         then at the next:
3325  *
3326  *          - cond_resched() call
3327  *          - explicit schedule() call
3328  *          - return from syscall or exception to user-space
3329  *          - return from interrupt-handler to user-space
3330  *
3331  * WARNING: must be called with preemption disabled!
3332  */
3333 static void __sched notrace __schedule(bool preempt)
3334 {
3335 	struct task_struct *prev, *next;
3336 	unsigned long *switch_count;
3337 	struct pin_cookie cookie;
3338 	struct rq *rq;
3339 	int cpu;
3340 
3341 	cpu = smp_processor_id();
3342 	rq = cpu_rq(cpu);
3343 	prev = rq->curr;
3344 
3345 	schedule_debug(prev);
3346 
3347 	if (sched_feat(HRTICK))
3348 		hrtick_clear(rq);
3349 
3350 	local_irq_disable();
3351 	rcu_note_context_switch();
3352 
3353 	/*
3354 	 * Make sure that signal_pending_state()->signal_pending() below
3355 	 * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE)
3356 	 * done by the caller to avoid the race with signal_wake_up().
3357 	 */
3358 	smp_mb__before_spinlock();
3359 	raw_spin_lock(&rq->lock);
3360 	cookie = lockdep_pin_lock(&rq->lock);
3361 
3362 	rq->clock_skip_update <<= 1; /* promote REQ to ACT */
3363 
3364 	switch_count = &prev->nivcsw;
3365 	if (!preempt && prev->state) {
3366 		if (unlikely(signal_pending_state(prev->state, prev))) {
3367 			prev->state = TASK_RUNNING;
3368 		} else {
3369 			deactivate_task(rq, prev, DEQUEUE_SLEEP);
3370 			prev->on_rq = 0;
3371 
3372 			/*
3373 			 * If a worker went to sleep, notify and ask workqueue
3374 			 * whether it wants to wake up a task to maintain
3375 			 * concurrency.
3376 			 */
3377 			if (prev->flags & PF_WQ_WORKER) {
3378 				struct task_struct *to_wakeup;
3379 
3380 				to_wakeup = wq_worker_sleeping(prev);
3381 				if (to_wakeup)
3382 					try_to_wake_up_local(to_wakeup, cookie);
3383 			}
3384 		}
3385 		switch_count = &prev->nvcsw;
3386 	}
3387 
3388 	if (task_on_rq_queued(prev))
3389 		update_rq_clock(rq);
3390 
3391 	next = pick_next_task(rq, prev, cookie);
3392 	clear_tsk_need_resched(prev);
3393 	clear_preempt_need_resched();
3394 	rq->clock_skip_update = 0;
3395 
3396 	if (likely(prev != next)) {
3397 		rq->nr_switches++;
3398 		rq->curr = next;
3399 		++*switch_count;
3400 
3401 		trace_sched_switch(preempt, prev, next);
3402 		rq = context_switch(rq, prev, next, cookie); /* unlocks the rq */
3403 	} else {
3404 		lockdep_unpin_lock(&rq->lock, cookie);
3405 		raw_spin_unlock_irq(&rq->lock);
3406 	}
3407 
3408 	balance_callback(rq);
3409 }
3410 
3411 void __noreturn do_task_dead(void)
3412 {
3413 	/*
3414 	 * The setting of TASK_RUNNING by try_to_wake_up() may be delayed
3415 	 * when the following two conditions become true.
3416 	 *   - There is race condition of mmap_sem (It is acquired by
3417 	 *     exit_mm()), and
3418 	 *   - SMI occurs before setting TASK_RUNINNG.
3419 	 *     (or hypervisor of virtual machine switches to other guest)
3420 	 *  As a result, we may become TASK_RUNNING after becoming TASK_DEAD
3421 	 *
3422 	 * To avoid it, we have to wait for releasing tsk->pi_lock which
3423 	 * is held by try_to_wake_up()
3424 	 */
3425 	smp_mb();
3426 	raw_spin_unlock_wait(&current->pi_lock);
3427 
3428 	/* causes final put_task_struct in finish_task_switch(). */
3429 	__set_current_state(TASK_DEAD);
3430 	current->flags |= PF_NOFREEZE;	/* tell freezer to ignore us */
3431 	__schedule(false);
3432 	BUG();
3433 	/* Avoid "noreturn function does return".  */
3434 	for (;;)
3435 		cpu_relax();	/* For when BUG is null */
3436 }
3437 
3438 static inline void sched_submit_work(struct task_struct *tsk)
3439 {
3440 	if (!tsk->state || tsk_is_pi_blocked(tsk))
3441 		return;
3442 	/*
3443 	 * If we are going to sleep and we have plugged IO queued,
3444 	 * make sure to submit it to avoid deadlocks.
3445 	 */
3446 	if (blk_needs_flush_plug(tsk))
3447 		blk_schedule_flush_plug(tsk);
3448 }
3449 
3450 asmlinkage __visible void __sched schedule(void)
3451 {
3452 	struct task_struct *tsk = current;
3453 
3454 	sched_submit_work(tsk);
3455 	do {
3456 		preempt_disable();
3457 		__schedule(false);
3458 		sched_preempt_enable_no_resched();
3459 	} while (need_resched());
3460 }
3461 EXPORT_SYMBOL(schedule);
3462 
3463 #ifdef CONFIG_CONTEXT_TRACKING
3464 asmlinkage __visible void __sched schedule_user(void)
3465 {
3466 	/*
3467 	 * If we come here after a random call to set_need_resched(),
3468 	 * or we have been woken up remotely but the IPI has not yet arrived,
3469 	 * we haven't yet exited the RCU idle mode. Do it here manually until
3470 	 * we find a better solution.
3471 	 *
3472 	 * NB: There are buggy callers of this function.  Ideally we
3473 	 * should warn if prev_state != CONTEXT_USER, but that will trigger
3474 	 * too frequently to make sense yet.
3475 	 */
3476 	enum ctx_state prev_state = exception_enter();
3477 	schedule();
3478 	exception_exit(prev_state);
3479 }
3480 #endif
3481 
3482 /**
3483  * schedule_preempt_disabled - called with preemption disabled
3484  *
3485  * Returns with preemption disabled. Note: preempt_count must be 1
3486  */
3487 void __sched schedule_preempt_disabled(void)
3488 {
3489 	sched_preempt_enable_no_resched();
3490 	schedule();
3491 	preempt_disable();
3492 }
3493 
3494 static void __sched notrace preempt_schedule_common(void)
3495 {
3496 	do {
3497 		/*
3498 		 * Because the function tracer can trace preempt_count_sub()
3499 		 * and it also uses preempt_enable/disable_notrace(), if
3500 		 * NEED_RESCHED is set, the preempt_enable_notrace() called
3501 		 * by the function tracer will call this function again and
3502 		 * cause infinite recursion.
3503 		 *
3504 		 * Preemption must be disabled here before the function
3505 		 * tracer can trace. Break up preempt_disable() into two
3506 		 * calls. One to disable preemption without fear of being
3507 		 * traced. The other to still record the preemption latency,
3508 		 * which can also be traced by the function tracer.
3509 		 */
3510 		preempt_disable_notrace();
3511 		preempt_latency_start(1);
3512 		__schedule(true);
3513 		preempt_latency_stop(1);
3514 		preempt_enable_no_resched_notrace();
3515 
3516 		/*
3517 		 * Check again in case we missed a preemption opportunity
3518 		 * between schedule and now.
3519 		 */
3520 	} while (need_resched());
3521 }
3522 
3523 #ifdef CONFIG_PREEMPT
3524 /*
3525  * this is the entry point to schedule() from in-kernel preemption
3526  * off of preempt_enable. Kernel preemptions off return from interrupt
3527  * occur there and call schedule directly.
3528  */
3529 asmlinkage __visible void __sched notrace preempt_schedule(void)
3530 {
3531 	/*
3532 	 * If there is a non-zero preempt_count or interrupts are disabled,
3533 	 * we do not want to preempt the current task. Just return..
3534 	 */
3535 	if (likely(!preemptible()))
3536 		return;
3537 
3538 	preempt_schedule_common();
3539 }
3540 NOKPROBE_SYMBOL(preempt_schedule);
3541 EXPORT_SYMBOL(preempt_schedule);
3542 
3543 /**
3544  * preempt_schedule_notrace - preempt_schedule called by tracing
3545  *
3546  * The tracing infrastructure uses preempt_enable_notrace to prevent
3547  * recursion and tracing preempt enabling caused by the tracing
3548  * infrastructure itself. But as tracing can happen in areas coming
3549  * from userspace or just about to enter userspace, a preempt enable
3550  * can occur before user_exit() is called. This will cause the scheduler
3551  * to be called when the system is still in usermode.
3552  *
3553  * To prevent this, the preempt_enable_notrace will use this function
3554  * instead of preempt_schedule() to exit user context if needed before
3555  * calling the scheduler.
3556  */
3557 asmlinkage __visible void __sched notrace preempt_schedule_notrace(void)
3558 {
3559 	enum ctx_state prev_ctx;
3560 
3561 	if (likely(!preemptible()))
3562 		return;
3563 
3564 	do {
3565 		/*
3566 		 * Because the function tracer can trace preempt_count_sub()
3567 		 * and it also uses preempt_enable/disable_notrace(), if
3568 		 * NEED_RESCHED is set, the preempt_enable_notrace() called
3569 		 * by the function tracer will call this function again and
3570 		 * cause infinite recursion.
3571 		 *
3572 		 * Preemption must be disabled here before the function
3573 		 * tracer can trace. Break up preempt_disable() into two
3574 		 * calls. One to disable preemption without fear of being
3575 		 * traced. The other to still record the preemption latency,
3576 		 * which can also be traced by the function tracer.
3577 		 */
3578 		preempt_disable_notrace();
3579 		preempt_latency_start(1);
3580 		/*
3581 		 * Needs preempt disabled in case user_exit() is traced
3582 		 * and the tracer calls preempt_enable_notrace() causing
3583 		 * an infinite recursion.
3584 		 */
3585 		prev_ctx = exception_enter();
3586 		__schedule(true);
3587 		exception_exit(prev_ctx);
3588 
3589 		preempt_latency_stop(1);
3590 		preempt_enable_no_resched_notrace();
3591 	} while (need_resched());
3592 }
3593 EXPORT_SYMBOL_GPL(preempt_schedule_notrace);
3594 
3595 #endif /* CONFIG_PREEMPT */
3596 
3597 /*
3598  * this is the entry point to schedule() from kernel preemption
3599  * off of irq context.
3600  * Note, that this is called and return with irqs disabled. This will
3601  * protect us against recursive calling from irq.
3602  */
3603 asmlinkage __visible void __sched preempt_schedule_irq(void)
3604 {
3605 	enum ctx_state prev_state;
3606 
3607 	/* Catch callers which need to be fixed */
3608 	BUG_ON(preempt_count() || !irqs_disabled());
3609 
3610 	prev_state = exception_enter();
3611 
3612 	do {
3613 		preempt_disable();
3614 		local_irq_enable();
3615 		__schedule(true);
3616 		local_irq_disable();
3617 		sched_preempt_enable_no_resched();
3618 	} while (need_resched());
3619 
3620 	exception_exit(prev_state);
3621 }
3622 
3623 int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags,
3624 			  void *key)
3625 {
3626 	return try_to_wake_up(curr->private, mode, wake_flags);
3627 }
3628 EXPORT_SYMBOL(default_wake_function);
3629 
3630 #ifdef CONFIG_RT_MUTEXES
3631 
3632 /*
3633  * rt_mutex_setprio - set the current priority of a task
3634  * @p: task
3635  * @prio: prio value (kernel-internal form)
3636  *
3637  * This function changes the 'effective' priority of a task. It does
3638  * not touch ->normal_prio like __setscheduler().
3639  *
3640  * Used by the rt_mutex code to implement priority inheritance
3641  * logic. Call site only calls if the priority of the task changed.
3642  */
3643 void rt_mutex_setprio(struct task_struct *p, int prio)
3644 {
3645 	int oldprio, queued, running, queue_flag = DEQUEUE_SAVE | DEQUEUE_MOVE;
3646 	const struct sched_class *prev_class;
3647 	struct rq_flags rf;
3648 	struct rq *rq;
3649 
3650 	BUG_ON(prio > MAX_PRIO);
3651 
3652 	rq = __task_rq_lock(p, &rf);
3653 
3654 	/*
3655 	 * Idle task boosting is a nono in general. There is one
3656 	 * exception, when PREEMPT_RT and NOHZ is active:
3657 	 *
3658 	 * The idle task calls get_next_timer_interrupt() and holds
3659 	 * the timer wheel base->lock on the CPU and another CPU wants
3660 	 * to access the timer (probably to cancel it). We can safely
3661 	 * ignore the boosting request, as the idle CPU runs this code
3662 	 * with interrupts disabled and will complete the lock
3663 	 * protected section without being interrupted. So there is no
3664 	 * real need to boost.
3665 	 */
3666 	if (unlikely(p == rq->idle)) {
3667 		WARN_ON(p != rq->curr);
3668 		WARN_ON(p->pi_blocked_on);
3669 		goto out_unlock;
3670 	}
3671 
3672 	trace_sched_pi_setprio(p, prio);
3673 	oldprio = p->prio;
3674 
3675 	if (oldprio == prio)
3676 		queue_flag &= ~DEQUEUE_MOVE;
3677 
3678 	prev_class = p->sched_class;
3679 	queued = task_on_rq_queued(p);
3680 	running = task_current(rq, p);
3681 	if (queued)
3682 		dequeue_task(rq, p, queue_flag);
3683 	if (running)
3684 		put_prev_task(rq, p);
3685 
3686 	/*
3687 	 * Boosting condition are:
3688 	 * 1. -rt task is running and holds mutex A
3689 	 *      --> -dl task blocks on mutex A
3690 	 *
3691 	 * 2. -dl task is running and holds mutex A
3692 	 *      --> -dl task blocks on mutex A and could preempt the
3693 	 *          running task
3694 	 */
3695 	if (dl_prio(prio)) {
3696 		struct task_struct *pi_task = rt_mutex_get_top_task(p);
3697 		if (!dl_prio(p->normal_prio) ||
3698 		    (pi_task && dl_entity_preempt(&pi_task->dl, &p->dl))) {
3699 			p->dl.dl_boosted = 1;
3700 			queue_flag |= ENQUEUE_REPLENISH;
3701 		} else
3702 			p->dl.dl_boosted = 0;
3703 		p->sched_class = &dl_sched_class;
3704 	} else if (rt_prio(prio)) {
3705 		if (dl_prio(oldprio))
3706 			p->dl.dl_boosted = 0;
3707 		if (oldprio < prio)
3708 			queue_flag |= ENQUEUE_HEAD;
3709 		p->sched_class = &rt_sched_class;
3710 	} else {
3711 		if (dl_prio(oldprio))
3712 			p->dl.dl_boosted = 0;
3713 		if (rt_prio(oldprio))
3714 			p->rt.timeout = 0;
3715 		p->sched_class = &fair_sched_class;
3716 	}
3717 
3718 	p->prio = prio;
3719 
3720 	if (queued)
3721 		enqueue_task(rq, p, queue_flag);
3722 	if (running)
3723 		set_curr_task(rq, p);
3724 
3725 	check_class_changed(rq, p, prev_class, oldprio);
3726 out_unlock:
3727 	preempt_disable(); /* avoid rq from going away on us */
3728 	__task_rq_unlock(rq, &rf);
3729 
3730 	balance_callback(rq);
3731 	preempt_enable();
3732 }
3733 #endif
3734 
3735 void set_user_nice(struct task_struct *p, long nice)
3736 {
3737 	bool queued, running;
3738 	int old_prio, delta;
3739 	struct rq_flags rf;
3740 	struct rq *rq;
3741 
3742 	if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE)
3743 		return;
3744 	/*
3745 	 * We have to be careful, if called from sys_setpriority(),
3746 	 * the task might be in the middle of scheduling on another CPU.
3747 	 */
3748 	rq = task_rq_lock(p, &rf);
3749 	/*
3750 	 * The RT priorities are set via sched_setscheduler(), but we still
3751 	 * allow the 'normal' nice value to be set - but as expected
3752 	 * it wont have any effect on scheduling until the task is
3753 	 * SCHED_DEADLINE, SCHED_FIFO or SCHED_RR:
3754 	 */
3755 	if (task_has_dl_policy(p) || task_has_rt_policy(p)) {
3756 		p->static_prio = NICE_TO_PRIO(nice);
3757 		goto out_unlock;
3758 	}
3759 	queued = task_on_rq_queued(p);
3760 	running = task_current(rq, p);
3761 	if (queued)
3762 		dequeue_task(rq, p, DEQUEUE_SAVE);
3763 	if (running)
3764 		put_prev_task(rq, p);
3765 
3766 	p->static_prio = NICE_TO_PRIO(nice);
3767 	set_load_weight(p);
3768 	old_prio = p->prio;
3769 	p->prio = effective_prio(p);
3770 	delta = p->prio - old_prio;
3771 
3772 	if (queued) {
3773 		enqueue_task(rq, p, ENQUEUE_RESTORE);
3774 		/*
3775 		 * If the task increased its priority or is running and
3776 		 * lowered its priority, then reschedule its CPU:
3777 		 */
3778 		if (delta < 0 || (delta > 0 && task_running(rq, p)))
3779 			resched_curr(rq);
3780 	}
3781 	if (running)
3782 		set_curr_task(rq, p);
3783 out_unlock:
3784 	task_rq_unlock(rq, p, &rf);
3785 }
3786 EXPORT_SYMBOL(set_user_nice);
3787 
3788 /*
3789  * can_nice - check if a task can reduce its nice value
3790  * @p: task
3791  * @nice: nice value
3792  */
3793 int can_nice(const struct task_struct *p, const int nice)
3794 {
3795 	/* convert nice value [19,-20] to rlimit style value [1,40] */
3796 	int nice_rlim = nice_to_rlimit(nice);
3797 
3798 	return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) ||
3799 		capable(CAP_SYS_NICE));
3800 }
3801 
3802 #ifdef __ARCH_WANT_SYS_NICE
3803 
3804 /*
3805  * sys_nice - change the priority of the current process.
3806  * @increment: priority increment
3807  *
3808  * sys_setpriority is a more generic, but much slower function that
3809  * does similar things.
3810  */
3811 SYSCALL_DEFINE1(nice, int, increment)
3812 {
3813 	long nice, retval;
3814 
3815 	/*
3816 	 * Setpriority might change our priority at the same moment.
3817 	 * We don't have to worry. Conceptually one call occurs first
3818 	 * and we have a single winner.
3819 	 */
3820 	increment = clamp(increment, -NICE_WIDTH, NICE_WIDTH);
3821 	nice = task_nice(current) + increment;
3822 
3823 	nice = clamp_val(nice, MIN_NICE, MAX_NICE);
3824 	if (increment < 0 && !can_nice(current, nice))
3825 		return -EPERM;
3826 
3827 	retval = security_task_setnice(current, nice);
3828 	if (retval)
3829 		return retval;
3830 
3831 	set_user_nice(current, nice);
3832 	return 0;
3833 }
3834 
3835 #endif
3836 
3837 /**
3838  * task_prio - return the priority value of a given task.
3839  * @p: the task in question.
3840  *
3841  * Return: The priority value as seen by users in /proc.
3842  * RT tasks are offset by -200. Normal tasks are centered
3843  * around 0, value goes from -16 to +15.
3844  */
3845 int task_prio(const struct task_struct *p)
3846 {
3847 	return p->prio - MAX_RT_PRIO;
3848 }
3849 
3850 /**
3851  * idle_cpu - is a given cpu idle currently?
3852  * @cpu: the processor in question.
3853  *
3854  * Return: 1 if the CPU is currently idle. 0 otherwise.
3855  */
3856 int idle_cpu(int cpu)
3857 {
3858 	struct rq *rq = cpu_rq(cpu);
3859 
3860 	if (rq->curr != rq->idle)
3861 		return 0;
3862 
3863 	if (rq->nr_running)
3864 		return 0;
3865 
3866 #ifdef CONFIG_SMP
3867 	if (!llist_empty(&rq->wake_list))
3868 		return 0;
3869 #endif
3870 
3871 	return 1;
3872 }
3873 
3874 /**
3875  * idle_task - return the idle task for a given cpu.
3876  * @cpu: the processor in question.
3877  *
3878  * Return: The idle task for the cpu @cpu.
3879  */
3880 struct task_struct *idle_task(int cpu)
3881 {
3882 	return cpu_rq(cpu)->idle;
3883 }
3884 
3885 /**
3886  * find_process_by_pid - find a process with a matching PID value.
3887  * @pid: the pid in question.
3888  *
3889  * The task of @pid, if found. %NULL otherwise.
3890  */
3891 static struct task_struct *find_process_by_pid(pid_t pid)
3892 {
3893 	return pid ? find_task_by_vpid(pid) : current;
3894 }
3895 
3896 /*
3897  * This function initializes the sched_dl_entity of a newly becoming
3898  * SCHED_DEADLINE task.
3899  *
3900  * Only the static values are considered here, the actual runtime and the
3901  * absolute deadline will be properly calculated when the task is enqueued
3902  * for the first time with its new policy.
3903  */
3904 static void
3905 __setparam_dl(struct task_struct *p, const struct sched_attr *attr)
3906 {
3907 	struct sched_dl_entity *dl_se = &p->dl;
3908 
3909 	dl_se->dl_runtime = attr->sched_runtime;
3910 	dl_se->dl_deadline = attr->sched_deadline;
3911 	dl_se->dl_period = attr->sched_period ?: dl_se->dl_deadline;
3912 	dl_se->flags = attr->sched_flags;
3913 	dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime);
3914 
3915 	/*
3916 	 * Changing the parameters of a task is 'tricky' and we're not doing
3917 	 * the correct thing -- also see task_dead_dl() and switched_from_dl().
3918 	 *
3919 	 * What we SHOULD do is delay the bandwidth release until the 0-lag
3920 	 * point. This would include retaining the task_struct until that time
3921 	 * and change dl_overflow() to not immediately decrement the current
3922 	 * amount.
3923 	 *
3924 	 * Instead we retain the current runtime/deadline and let the new
3925 	 * parameters take effect after the current reservation period lapses.
3926 	 * This is safe (albeit pessimistic) because the 0-lag point is always
3927 	 * before the current scheduling deadline.
3928 	 *
3929 	 * We can still have temporary overloads because we do not delay the
3930 	 * change in bandwidth until that time; so admission control is
3931 	 * not on the safe side. It does however guarantee tasks will never
3932 	 * consume more than promised.
3933 	 */
3934 }
3935 
3936 /*
3937  * sched_setparam() passes in -1 for its policy, to let the functions
3938  * it calls know not to change it.
3939  */
3940 #define SETPARAM_POLICY	-1
3941 
3942 static void __setscheduler_params(struct task_struct *p,
3943 		const struct sched_attr *attr)
3944 {
3945 	int policy = attr->sched_policy;
3946 
3947 	if (policy == SETPARAM_POLICY)
3948 		policy = p->policy;
3949 
3950 	p->policy = policy;
3951 
3952 	if (dl_policy(policy))
3953 		__setparam_dl(p, attr);
3954 	else if (fair_policy(policy))
3955 		p->static_prio = NICE_TO_PRIO(attr->sched_nice);
3956 
3957 	/*
3958 	 * __sched_setscheduler() ensures attr->sched_priority == 0 when
3959 	 * !rt_policy. Always setting this ensures that things like
3960 	 * getparam()/getattr() don't report silly values for !rt tasks.
3961 	 */
3962 	p->rt_priority = attr->sched_priority;
3963 	p->normal_prio = normal_prio(p);
3964 	set_load_weight(p);
3965 }
3966 
3967 /* Actually do priority change: must hold pi & rq lock. */
3968 static void __setscheduler(struct rq *rq, struct task_struct *p,
3969 			   const struct sched_attr *attr, bool keep_boost)
3970 {
3971 	__setscheduler_params(p, attr);
3972 
3973 	/*
3974 	 * Keep a potential priority boosting if called from
3975 	 * sched_setscheduler().
3976 	 */
3977 	if (keep_boost)
3978 		p->prio = rt_mutex_get_effective_prio(p, normal_prio(p));
3979 	else
3980 		p->prio = normal_prio(p);
3981 
3982 	if (dl_prio(p->prio))
3983 		p->sched_class = &dl_sched_class;
3984 	else if (rt_prio(p->prio))
3985 		p->sched_class = &rt_sched_class;
3986 	else
3987 		p->sched_class = &fair_sched_class;
3988 }
3989 
3990 static void
3991 __getparam_dl(struct task_struct *p, struct sched_attr *attr)
3992 {
3993 	struct sched_dl_entity *dl_se = &p->dl;
3994 
3995 	attr->sched_priority = p->rt_priority;
3996 	attr->sched_runtime = dl_se->dl_runtime;
3997 	attr->sched_deadline = dl_se->dl_deadline;
3998 	attr->sched_period = dl_se->dl_period;
3999 	attr->sched_flags = dl_se->flags;
4000 }
4001 
4002 /*
4003  * This function validates the new parameters of a -deadline task.
4004  * We ask for the deadline not being zero, and greater or equal
4005  * than the runtime, as well as the period of being zero or
4006  * greater than deadline. Furthermore, we have to be sure that
4007  * user parameters are above the internal resolution of 1us (we
4008  * check sched_runtime only since it is always the smaller one) and
4009  * below 2^63 ns (we have to check both sched_deadline and
4010  * sched_period, as the latter can be zero).
4011  */
4012 static bool
4013 __checkparam_dl(const struct sched_attr *attr)
4014 {
4015 	/* deadline != 0 */
4016 	if (attr->sched_deadline == 0)
4017 		return false;
4018 
4019 	/*
4020 	 * Since we truncate DL_SCALE bits, make sure we're at least
4021 	 * that big.
4022 	 */
4023 	if (attr->sched_runtime < (1ULL << DL_SCALE))
4024 		return false;
4025 
4026 	/*
4027 	 * Since we use the MSB for wrap-around and sign issues, make
4028 	 * sure it's not set (mind that period can be equal to zero).
4029 	 */
4030 	if (attr->sched_deadline & (1ULL << 63) ||
4031 	    attr->sched_period & (1ULL << 63))
4032 		return false;
4033 
4034 	/* runtime <= deadline <= period (if period != 0) */
4035 	if ((attr->sched_period != 0 &&
4036 	     attr->sched_period < attr->sched_deadline) ||
4037 	    attr->sched_deadline < attr->sched_runtime)
4038 		return false;
4039 
4040 	return true;
4041 }
4042 
4043 /*
4044  * check the target process has a UID that matches the current process's
4045  */
4046 static bool check_same_owner(struct task_struct *p)
4047 {
4048 	const struct cred *cred = current_cred(), *pcred;
4049 	bool match;
4050 
4051 	rcu_read_lock();
4052 	pcred = __task_cred(p);
4053 	match = (uid_eq(cred->euid, pcred->euid) ||
4054 		 uid_eq(cred->euid, pcred->uid));
4055 	rcu_read_unlock();
4056 	return match;
4057 }
4058 
4059 static bool dl_param_changed(struct task_struct *p,
4060 		const struct sched_attr *attr)
4061 {
4062 	struct sched_dl_entity *dl_se = &p->dl;
4063 
4064 	if (dl_se->dl_runtime != attr->sched_runtime ||
4065 		dl_se->dl_deadline != attr->sched_deadline ||
4066 		dl_se->dl_period != attr->sched_period ||
4067 		dl_se->flags != attr->sched_flags)
4068 		return true;
4069 
4070 	return false;
4071 }
4072 
4073 static int __sched_setscheduler(struct task_struct *p,
4074 				const struct sched_attr *attr,
4075 				bool user, bool pi)
4076 {
4077 	int newprio = dl_policy(attr->sched_policy) ? MAX_DL_PRIO - 1 :
4078 		      MAX_RT_PRIO - 1 - attr->sched_priority;
4079 	int retval, oldprio, oldpolicy = -1, queued, running;
4080 	int new_effective_prio, policy = attr->sched_policy;
4081 	const struct sched_class *prev_class;
4082 	struct rq_flags rf;
4083 	int reset_on_fork;
4084 	int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE;
4085 	struct rq *rq;
4086 
4087 	/* may grab non-irq protected spin_locks */
4088 	BUG_ON(in_interrupt());
4089 recheck:
4090 	/* double check policy once rq lock held */
4091 	if (policy < 0) {
4092 		reset_on_fork = p->sched_reset_on_fork;
4093 		policy = oldpolicy = p->policy;
4094 	} else {
4095 		reset_on_fork = !!(attr->sched_flags & SCHED_FLAG_RESET_ON_FORK);
4096 
4097 		if (!valid_policy(policy))
4098 			return -EINVAL;
4099 	}
4100 
4101 	if (attr->sched_flags & ~(SCHED_FLAG_RESET_ON_FORK))
4102 		return -EINVAL;
4103 
4104 	/*
4105 	 * Valid priorities for SCHED_FIFO and SCHED_RR are
4106 	 * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
4107 	 * SCHED_BATCH and SCHED_IDLE is 0.
4108 	 */
4109 	if ((p->mm && attr->sched_priority > MAX_USER_RT_PRIO-1) ||
4110 	    (!p->mm && attr->sched_priority > MAX_RT_PRIO-1))
4111 		return -EINVAL;
4112 	if ((dl_policy(policy) && !__checkparam_dl(attr)) ||
4113 	    (rt_policy(policy) != (attr->sched_priority != 0)))
4114 		return -EINVAL;
4115 
4116 	/*
4117 	 * Allow unprivileged RT tasks to decrease priority:
4118 	 */
4119 	if (user && !capable(CAP_SYS_NICE)) {
4120 		if (fair_policy(policy)) {
4121 			if (attr->sched_nice < task_nice(p) &&
4122 			    !can_nice(p, attr->sched_nice))
4123 				return -EPERM;
4124 		}
4125 
4126 		if (rt_policy(policy)) {
4127 			unsigned long rlim_rtprio =
4128 					task_rlimit(p, RLIMIT_RTPRIO);
4129 
4130 			/* can't set/change the rt policy */
4131 			if (policy != p->policy && !rlim_rtprio)
4132 				return -EPERM;
4133 
4134 			/* can't increase priority */
4135 			if (attr->sched_priority > p->rt_priority &&
4136 			    attr->sched_priority > rlim_rtprio)
4137 				return -EPERM;
4138 		}
4139 
4140 		 /*
4141 		  * Can't set/change SCHED_DEADLINE policy at all for now
4142 		  * (safest behavior); in the future we would like to allow
4143 		  * unprivileged DL tasks to increase their relative deadline
4144 		  * or reduce their runtime (both ways reducing utilization)
4145 		  */
4146 		if (dl_policy(policy))
4147 			return -EPERM;
4148 
4149 		/*
4150 		 * Treat SCHED_IDLE as nice 20. Only allow a switch to
4151 		 * SCHED_NORMAL if the RLIMIT_NICE would normally permit it.
4152 		 */
4153 		if (idle_policy(p->policy) && !idle_policy(policy)) {
4154 			if (!can_nice(p, task_nice(p)))
4155 				return -EPERM;
4156 		}
4157 
4158 		/* can't change other user's priorities */
4159 		if (!check_same_owner(p))
4160 			return -EPERM;
4161 
4162 		/* Normal users shall not reset the sched_reset_on_fork flag */
4163 		if (p->sched_reset_on_fork && !reset_on_fork)
4164 			return -EPERM;
4165 	}
4166 
4167 	if (user) {
4168 		retval = security_task_setscheduler(p);
4169 		if (retval)
4170 			return retval;
4171 	}
4172 
4173 	/*
4174 	 * make sure no PI-waiters arrive (or leave) while we are
4175 	 * changing the priority of the task:
4176 	 *
4177 	 * To be able to change p->policy safely, the appropriate
4178 	 * runqueue lock must be held.
4179 	 */
4180 	rq = task_rq_lock(p, &rf);
4181 
4182 	/*
4183 	 * Changing the policy of the stop threads its a very bad idea
4184 	 */
4185 	if (p == rq->stop) {
4186 		task_rq_unlock(rq, p, &rf);
4187 		return -EINVAL;
4188 	}
4189 
4190 	/*
4191 	 * If not changing anything there's no need to proceed further,
4192 	 * but store a possible modification of reset_on_fork.
4193 	 */
4194 	if (unlikely(policy == p->policy)) {
4195 		if (fair_policy(policy) && attr->sched_nice != task_nice(p))
4196 			goto change;
4197 		if (rt_policy(policy) && attr->sched_priority != p->rt_priority)
4198 			goto change;
4199 		if (dl_policy(policy) && dl_param_changed(p, attr))
4200 			goto change;
4201 
4202 		p->sched_reset_on_fork = reset_on_fork;
4203 		task_rq_unlock(rq, p, &rf);
4204 		return 0;
4205 	}
4206 change:
4207 
4208 	if (user) {
4209 #ifdef CONFIG_RT_GROUP_SCHED
4210 		/*
4211 		 * Do not allow realtime tasks into groups that have no runtime
4212 		 * assigned.
4213 		 */
4214 		if (rt_bandwidth_enabled() && rt_policy(policy) &&
4215 				task_group(p)->rt_bandwidth.rt_runtime == 0 &&
4216 				!task_group_is_autogroup(task_group(p))) {
4217 			task_rq_unlock(rq, p, &rf);
4218 			return -EPERM;
4219 		}
4220 #endif
4221 #ifdef CONFIG_SMP
4222 		if (dl_bandwidth_enabled() && dl_policy(policy)) {
4223 			cpumask_t *span = rq->rd->span;
4224 
4225 			/*
4226 			 * Don't allow tasks with an affinity mask smaller than
4227 			 * the entire root_domain to become SCHED_DEADLINE. We
4228 			 * will also fail if there's no bandwidth available.
4229 			 */
4230 			if (!cpumask_subset(span, &p->cpus_allowed) ||
4231 			    rq->rd->dl_bw.bw == 0) {
4232 				task_rq_unlock(rq, p, &rf);
4233 				return -EPERM;
4234 			}
4235 		}
4236 #endif
4237 	}
4238 
4239 	/* recheck policy now with rq lock held */
4240 	if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
4241 		policy = oldpolicy = -1;
4242 		task_rq_unlock(rq, p, &rf);
4243 		goto recheck;
4244 	}
4245 
4246 	/*
4247 	 * If setscheduling to SCHED_DEADLINE (or changing the parameters
4248 	 * of a SCHED_DEADLINE task) we need to check if enough bandwidth
4249 	 * is available.
4250 	 */
4251 	if ((dl_policy(policy) || dl_task(p)) && dl_overflow(p, policy, attr)) {
4252 		task_rq_unlock(rq, p, &rf);
4253 		return -EBUSY;
4254 	}
4255 
4256 	p->sched_reset_on_fork = reset_on_fork;
4257 	oldprio = p->prio;
4258 
4259 	if (pi) {
4260 		/*
4261 		 * Take priority boosted tasks into account. If the new
4262 		 * effective priority is unchanged, we just store the new
4263 		 * normal parameters and do not touch the scheduler class and
4264 		 * the runqueue. This will be done when the task deboost
4265 		 * itself.
4266 		 */
4267 		new_effective_prio = rt_mutex_get_effective_prio(p, newprio);
4268 		if (new_effective_prio == oldprio)
4269 			queue_flags &= ~DEQUEUE_MOVE;
4270 	}
4271 
4272 	queued = task_on_rq_queued(p);
4273 	running = task_current(rq, p);
4274 	if (queued)
4275 		dequeue_task(rq, p, queue_flags);
4276 	if (running)
4277 		put_prev_task(rq, p);
4278 
4279 	prev_class = p->sched_class;
4280 	__setscheduler(rq, p, attr, pi);
4281 
4282 	if (queued) {
4283 		/*
4284 		 * We enqueue to tail when the priority of a task is
4285 		 * increased (user space view).
4286 		 */
4287 		if (oldprio < p->prio)
4288 			queue_flags |= ENQUEUE_HEAD;
4289 
4290 		enqueue_task(rq, p, queue_flags);
4291 	}
4292 	if (running)
4293 		set_curr_task(rq, p);
4294 
4295 	check_class_changed(rq, p, prev_class, oldprio);
4296 	preempt_disable(); /* avoid rq from going away on us */
4297 	task_rq_unlock(rq, p, &rf);
4298 
4299 	if (pi)
4300 		rt_mutex_adjust_pi(p);
4301 
4302 	/*
4303 	 * Run balance callbacks after we've adjusted the PI chain.
4304 	 */
4305 	balance_callback(rq);
4306 	preempt_enable();
4307 
4308 	return 0;
4309 }
4310 
4311 static int _sched_setscheduler(struct task_struct *p, int policy,
4312 			       const struct sched_param *param, bool check)
4313 {
4314 	struct sched_attr attr = {
4315 		.sched_policy   = policy,
4316 		.sched_priority = param->sched_priority,
4317 		.sched_nice	= PRIO_TO_NICE(p->static_prio),
4318 	};
4319 
4320 	/* Fixup the legacy SCHED_RESET_ON_FORK hack. */
4321 	if ((policy != SETPARAM_POLICY) && (policy & SCHED_RESET_ON_FORK)) {
4322 		attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK;
4323 		policy &= ~SCHED_RESET_ON_FORK;
4324 		attr.sched_policy = policy;
4325 	}
4326 
4327 	return __sched_setscheduler(p, &attr, check, true);
4328 }
4329 /**
4330  * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
4331  * @p: the task in question.
4332  * @policy: new policy.
4333  * @param: structure containing the new RT priority.
4334  *
4335  * Return: 0 on success. An error code otherwise.
4336  *
4337  * NOTE that the task may be already dead.
4338  */
4339 int sched_setscheduler(struct task_struct *p, int policy,
4340 		       const struct sched_param *param)
4341 {
4342 	return _sched_setscheduler(p, policy, param, true);
4343 }
4344 EXPORT_SYMBOL_GPL(sched_setscheduler);
4345 
4346 int sched_setattr(struct task_struct *p, const struct sched_attr *attr)
4347 {
4348 	return __sched_setscheduler(p, attr, true, true);
4349 }
4350 EXPORT_SYMBOL_GPL(sched_setattr);
4351 
4352 /**
4353  * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
4354  * @p: the task in question.
4355  * @policy: new policy.
4356  * @param: structure containing the new RT priority.
4357  *
4358  * Just like sched_setscheduler, only don't bother checking if the
4359  * current context has permission.  For example, this is needed in
4360  * stop_machine(): we create temporary high priority worker threads,
4361  * but our caller might not have that capability.
4362  *
4363  * Return: 0 on success. An error code otherwise.
4364  */
4365 int sched_setscheduler_nocheck(struct task_struct *p, int policy,
4366 			       const struct sched_param *param)
4367 {
4368 	return _sched_setscheduler(p, policy, param, false);
4369 }
4370 EXPORT_SYMBOL_GPL(sched_setscheduler_nocheck);
4371 
4372 static int
4373 do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
4374 {
4375 	struct sched_param lparam;
4376 	struct task_struct *p;
4377 	int retval;
4378 
4379 	if (!param || pid < 0)
4380 		return -EINVAL;
4381 	if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
4382 		return -EFAULT;
4383 
4384 	rcu_read_lock();
4385 	retval = -ESRCH;
4386 	p = find_process_by_pid(pid);
4387 	if (p != NULL)
4388 		retval = sched_setscheduler(p, policy, &lparam);
4389 	rcu_read_unlock();
4390 
4391 	return retval;
4392 }
4393 
4394 /*
4395  * Mimics kernel/events/core.c perf_copy_attr().
4396  */
4397 static int sched_copy_attr(struct sched_attr __user *uattr,
4398 			   struct sched_attr *attr)
4399 {
4400 	u32 size;
4401 	int ret;
4402 
4403 	if (!access_ok(VERIFY_WRITE, uattr, SCHED_ATTR_SIZE_VER0))
4404 		return -EFAULT;
4405 
4406 	/*
4407 	 * zero the full structure, so that a short copy will be nice.
4408 	 */
4409 	memset(attr, 0, sizeof(*attr));
4410 
4411 	ret = get_user(size, &uattr->size);
4412 	if (ret)
4413 		return ret;
4414 
4415 	if (size > PAGE_SIZE)	/* silly large */
4416 		goto err_size;
4417 
4418 	if (!size)		/* abi compat */
4419 		size = SCHED_ATTR_SIZE_VER0;
4420 
4421 	if (size < SCHED_ATTR_SIZE_VER0)
4422 		goto err_size;
4423 
4424 	/*
4425 	 * If we're handed a bigger struct than we know of,
4426 	 * ensure all the unknown bits are 0 - i.e. new
4427 	 * user-space does not rely on any kernel feature
4428 	 * extensions we dont know about yet.
4429 	 */
4430 	if (size > sizeof(*attr)) {
4431 		unsigned char __user *addr;
4432 		unsigned char __user *end;
4433 		unsigned char val;
4434 
4435 		addr = (void __user *)uattr + sizeof(*attr);
4436 		end  = (void __user *)uattr + size;
4437 
4438 		for (; addr < end; addr++) {
4439 			ret = get_user(val, addr);
4440 			if (ret)
4441 				return ret;
4442 			if (val)
4443 				goto err_size;
4444 		}
4445 		size = sizeof(*attr);
4446 	}
4447 
4448 	ret = copy_from_user(attr, uattr, size);
4449 	if (ret)
4450 		return -EFAULT;
4451 
4452 	/*
4453 	 * XXX: do we want to be lenient like existing syscalls; or do we want
4454 	 * to be strict and return an error on out-of-bounds values?
4455 	 */
4456 	attr->sched_nice = clamp(attr->sched_nice, MIN_NICE, MAX_NICE);
4457 
4458 	return 0;
4459 
4460 err_size:
4461 	put_user(sizeof(*attr), &uattr->size);
4462 	return -E2BIG;
4463 }
4464 
4465 /**
4466  * sys_sched_setscheduler - set/change the scheduler policy and RT priority
4467  * @pid: the pid in question.
4468  * @policy: new policy.
4469  * @param: structure containing the new RT priority.
4470  *
4471  * Return: 0 on success. An error code otherwise.
4472  */
4473 SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy,
4474 		struct sched_param __user *, param)
4475 {
4476 	/* negative values for policy are not valid */
4477 	if (policy < 0)
4478 		return -EINVAL;
4479 
4480 	return do_sched_setscheduler(pid, policy, param);
4481 }
4482 
4483 /**
4484  * sys_sched_setparam - set/change the RT priority of a thread
4485  * @pid: the pid in question.
4486  * @param: structure containing the new RT priority.
4487  *
4488  * Return: 0 on success. An error code otherwise.
4489  */
4490 SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param)
4491 {
4492 	return do_sched_setscheduler(pid, SETPARAM_POLICY, param);
4493 }
4494 
4495 /**
4496  * sys_sched_setattr - same as above, but with extended sched_attr
4497  * @pid: the pid in question.
4498  * @uattr: structure containing the extended parameters.
4499  * @flags: for future extension.
4500  */
4501 SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr,
4502 			       unsigned int, flags)
4503 {
4504 	struct sched_attr attr;
4505 	struct task_struct *p;
4506 	int retval;
4507 
4508 	if (!uattr || pid < 0 || flags)
4509 		return -EINVAL;
4510 
4511 	retval = sched_copy_attr(uattr, &attr);
4512 	if (retval)
4513 		return retval;
4514 
4515 	if ((int)attr.sched_policy < 0)
4516 		return -EINVAL;
4517 
4518 	rcu_read_lock();
4519 	retval = -ESRCH;
4520 	p = find_process_by_pid(pid);
4521 	if (p != NULL)
4522 		retval = sched_setattr(p, &attr);
4523 	rcu_read_unlock();
4524 
4525 	return retval;
4526 }
4527 
4528 /**
4529  * sys_sched_getscheduler - get the policy (scheduling class) of a thread
4530  * @pid: the pid in question.
4531  *
4532  * Return: On success, the policy of the thread. Otherwise, a negative error
4533  * code.
4534  */
4535 SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
4536 {
4537 	struct task_struct *p;
4538 	int retval;
4539 
4540 	if (pid < 0)
4541 		return -EINVAL;
4542 
4543 	retval = -ESRCH;
4544 	rcu_read_lock();
4545 	p = find_process_by_pid(pid);
4546 	if (p) {
4547 		retval = security_task_getscheduler(p);
4548 		if (!retval)
4549 			retval = p->policy
4550 				| (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0);
4551 	}
4552 	rcu_read_unlock();
4553 	return retval;
4554 }
4555 
4556 /**
4557  * sys_sched_getparam - get the RT priority of a thread
4558  * @pid: the pid in question.
4559  * @param: structure containing the RT priority.
4560  *
4561  * Return: On success, 0 and the RT priority is in @param. Otherwise, an error
4562  * code.
4563  */
4564 SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
4565 {
4566 	struct sched_param lp = { .sched_priority = 0 };
4567 	struct task_struct *p;
4568 	int retval;
4569 
4570 	if (!param || pid < 0)
4571 		return -EINVAL;
4572 
4573 	rcu_read_lock();
4574 	p = find_process_by_pid(pid);
4575 	retval = -ESRCH;
4576 	if (!p)
4577 		goto out_unlock;
4578 
4579 	retval = security_task_getscheduler(p);
4580 	if (retval)
4581 		goto out_unlock;
4582 
4583 	if (task_has_rt_policy(p))
4584 		lp.sched_priority = p->rt_priority;
4585 	rcu_read_unlock();
4586 
4587 	/*
4588 	 * This one might sleep, we cannot do it with a spinlock held ...
4589 	 */
4590 	retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;
4591 
4592 	return retval;
4593 
4594 out_unlock:
4595 	rcu_read_unlock();
4596 	return retval;
4597 }
4598 
4599 static int sched_read_attr(struct sched_attr __user *uattr,
4600 			   struct sched_attr *attr,
4601 			   unsigned int usize)
4602 {
4603 	int ret;
4604 
4605 	if (!access_ok(VERIFY_WRITE, uattr, usize))
4606 		return -EFAULT;
4607 
4608 	/*
4609 	 * If we're handed a smaller struct than we know of,
4610 	 * ensure all the unknown bits are 0 - i.e. old
4611 	 * user-space does not get uncomplete information.
4612 	 */
4613 	if (usize < sizeof(*attr)) {
4614 		unsigned char *addr;
4615 		unsigned char *end;
4616 
4617 		addr = (void *)attr + usize;
4618 		end  = (void *)attr + sizeof(*attr);
4619 
4620 		for (; addr < end; addr++) {
4621 			if (*addr)
4622 				return -EFBIG;
4623 		}
4624 
4625 		attr->size = usize;
4626 	}
4627 
4628 	ret = copy_to_user(uattr, attr, attr->size);
4629 	if (ret)
4630 		return -EFAULT;
4631 
4632 	return 0;
4633 }
4634 
4635 /**
4636  * sys_sched_getattr - similar to sched_getparam, but with sched_attr
4637  * @pid: the pid in question.
4638  * @uattr: structure containing the extended parameters.
4639  * @size: sizeof(attr) for fwd/bwd comp.
4640  * @flags: for future extension.
4641  */
4642 SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr,
4643 		unsigned int, size, unsigned int, flags)
4644 {
4645 	struct sched_attr attr = {
4646 		.size = sizeof(struct sched_attr),
4647 	};
4648 	struct task_struct *p;
4649 	int retval;
4650 
4651 	if (!uattr || pid < 0 || size > PAGE_SIZE ||
4652 	    size < SCHED_ATTR_SIZE_VER0 || flags)
4653 		return -EINVAL;
4654 
4655 	rcu_read_lock();
4656 	p = find_process_by_pid(pid);
4657 	retval = -ESRCH;
4658 	if (!p)
4659 		goto out_unlock;
4660 
4661 	retval = security_task_getscheduler(p);
4662 	if (retval)
4663 		goto out_unlock;
4664 
4665 	attr.sched_policy = p->policy;
4666 	if (p->sched_reset_on_fork)
4667 		attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK;
4668 	if (task_has_dl_policy(p))
4669 		__getparam_dl(p, &attr);
4670 	else if (task_has_rt_policy(p))
4671 		attr.sched_priority = p->rt_priority;
4672 	else
4673 		attr.sched_nice = task_nice(p);
4674 
4675 	rcu_read_unlock();
4676 
4677 	retval = sched_read_attr(uattr, &attr, size);
4678 	return retval;
4679 
4680 out_unlock:
4681 	rcu_read_unlock();
4682 	return retval;
4683 }
4684 
4685 long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
4686 {
4687 	cpumask_var_t cpus_allowed, new_mask;
4688 	struct task_struct *p;
4689 	int retval;
4690 
4691 	rcu_read_lock();
4692 
4693 	p = find_process_by_pid(pid);
4694 	if (!p) {
4695 		rcu_read_unlock();
4696 		return -ESRCH;
4697 	}
4698 
4699 	/* Prevent p going away */
4700 	get_task_struct(p);
4701 	rcu_read_unlock();
4702 
4703 	if (p->flags & PF_NO_SETAFFINITY) {
4704 		retval = -EINVAL;
4705 		goto out_put_task;
4706 	}
4707 	if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
4708 		retval = -ENOMEM;
4709 		goto out_put_task;
4710 	}
4711 	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
4712 		retval = -ENOMEM;
4713 		goto out_free_cpus_allowed;
4714 	}
4715 	retval = -EPERM;
4716 	if (!check_same_owner(p)) {
4717 		rcu_read_lock();
4718 		if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) {
4719 			rcu_read_unlock();
4720 			goto out_free_new_mask;
4721 		}
4722 		rcu_read_unlock();
4723 	}
4724 
4725 	retval = security_task_setscheduler(p);
4726 	if (retval)
4727 		goto out_free_new_mask;
4728 
4729 
4730 	cpuset_cpus_allowed(p, cpus_allowed);
4731 	cpumask_and(new_mask, in_mask, cpus_allowed);
4732 
4733 	/*
4734 	 * Since bandwidth control happens on root_domain basis,
4735 	 * if admission test is enabled, we only admit -deadline
4736 	 * tasks allowed to run on all the CPUs in the task's
4737 	 * root_domain.
4738 	 */
4739 #ifdef CONFIG_SMP
4740 	if (task_has_dl_policy(p) && dl_bandwidth_enabled()) {
4741 		rcu_read_lock();
4742 		if (!cpumask_subset(task_rq(p)->rd->span, new_mask)) {
4743 			retval = -EBUSY;
4744 			rcu_read_unlock();
4745 			goto out_free_new_mask;
4746 		}
4747 		rcu_read_unlock();
4748 	}
4749 #endif
4750 again:
4751 	retval = __set_cpus_allowed_ptr(p, new_mask, true);
4752 
4753 	if (!retval) {
4754 		cpuset_cpus_allowed(p, cpus_allowed);
4755 		if (!cpumask_subset(new_mask, cpus_allowed)) {
4756 			/*
4757 			 * We must have raced with a concurrent cpuset
4758 			 * update. Just reset the cpus_allowed to the
4759 			 * cpuset's cpus_allowed
4760 			 */
4761 			cpumask_copy(new_mask, cpus_allowed);
4762 			goto again;
4763 		}
4764 	}
4765 out_free_new_mask:
4766 	free_cpumask_var(new_mask);
4767 out_free_cpus_allowed:
4768 	free_cpumask_var(cpus_allowed);
4769 out_put_task:
4770 	put_task_struct(p);
4771 	return retval;
4772 }
4773 
4774 static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
4775 			     struct cpumask *new_mask)
4776 {
4777 	if (len < cpumask_size())
4778 		cpumask_clear(new_mask);
4779 	else if (len > cpumask_size())
4780 		len = cpumask_size();
4781 
4782 	return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
4783 }
4784 
4785 /**
4786  * sys_sched_setaffinity - set the cpu affinity of a process
4787  * @pid: pid of the process
4788  * @len: length in bytes of the bitmask pointed to by user_mask_ptr
4789  * @user_mask_ptr: user-space pointer to the new cpu mask
4790  *
4791  * Return: 0 on success. An error code otherwise.
4792  */
4793 SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len,
4794 		unsigned long __user *, user_mask_ptr)
4795 {
4796 	cpumask_var_t new_mask;
4797 	int retval;
4798 
4799 	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
4800 		return -ENOMEM;
4801 
4802 	retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
4803 	if (retval == 0)
4804 		retval = sched_setaffinity(pid, new_mask);
4805 	free_cpumask_var(new_mask);
4806 	return retval;
4807 }
4808 
4809 long sched_getaffinity(pid_t pid, struct cpumask *mask)
4810 {
4811 	struct task_struct *p;
4812 	unsigned long flags;
4813 	int retval;
4814 
4815 	rcu_read_lock();
4816 
4817 	retval = -ESRCH;
4818 	p = find_process_by_pid(pid);
4819 	if (!p)
4820 		goto out_unlock;
4821 
4822 	retval = security_task_getscheduler(p);
4823 	if (retval)
4824 		goto out_unlock;
4825 
4826 	raw_spin_lock_irqsave(&p->pi_lock, flags);
4827 	cpumask_and(mask, &p->cpus_allowed, cpu_active_mask);
4828 	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
4829 
4830 out_unlock:
4831 	rcu_read_unlock();
4832 
4833 	return retval;
4834 }
4835 
4836 /**
4837  * sys_sched_getaffinity - get the cpu affinity of a process
4838  * @pid: pid of the process
4839  * @len: length in bytes of the bitmask pointed to by user_mask_ptr
4840  * @user_mask_ptr: user-space pointer to hold the current cpu mask
4841  *
4842  * Return: size of CPU mask copied to user_mask_ptr on success. An
4843  * error code otherwise.
4844  */
4845 SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len,
4846 		unsigned long __user *, user_mask_ptr)
4847 {
4848 	int ret;
4849 	cpumask_var_t mask;
4850 
4851 	if ((len * BITS_PER_BYTE) < nr_cpu_ids)
4852 		return -EINVAL;
4853 	if (len & (sizeof(unsigned long)-1))
4854 		return -EINVAL;
4855 
4856 	if (!alloc_cpumask_var(&mask, GFP_KERNEL))
4857 		return -ENOMEM;
4858 
4859 	ret = sched_getaffinity(pid, mask);
4860 	if (ret == 0) {
4861 		size_t retlen = min_t(size_t, len, cpumask_size());
4862 
4863 		if (copy_to_user(user_mask_ptr, mask, retlen))
4864 			ret = -EFAULT;
4865 		else
4866 			ret = retlen;
4867 	}
4868 	free_cpumask_var(mask);
4869 
4870 	return ret;
4871 }
4872 
4873 /**
4874  * sys_sched_yield - yield the current processor to other threads.
4875  *
4876  * This function yields the current CPU to other tasks. If there are no
4877  * other threads running on this CPU then this function will return.
4878  *
4879  * Return: 0.
4880  */
4881 SYSCALL_DEFINE0(sched_yield)
4882 {
4883 	struct rq *rq = this_rq_lock();
4884 
4885 	schedstat_inc(rq->yld_count);
4886 	current->sched_class->yield_task(rq);
4887 
4888 	/*
4889 	 * Since we are going to call schedule() anyway, there's
4890 	 * no need to preempt or enable interrupts:
4891 	 */
4892 	__release(rq->lock);
4893 	spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
4894 	do_raw_spin_unlock(&rq->lock);
4895 	sched_preempt_enable_no_resched();
4896 
4897 	schedule();
4898 
4899 	return 0;
4900 }
4901 
4902 #ifndef CONFIG_PREEMPT
4903 int __sched _cond_resched(void)
4904 {
4905 	if (should_resched(0)) {
4906 		preempt_schedule_common();
4907 		return 1;
4908 	}
4909 	return 0;
4910 }
4911 EXPORT_SYMBOL(_cond_resched);
4912 #endif
4913 
4914 /*
4915  * __cond_resched_lock() - if a reschedule is pending, drop the given lock,
4916  * call schedule, and on return reacquire the lock.
4917  *
4918  * This works OK both with and without CONFIG_PREEMPT. We do strange low-level
4919  * operations here to prevent schedule() from being called twice (once via
4920  * spin_unlock(), once by hand).
4921  */
4922 int __cond_resched_lock(spinlock_t *lock)
4923 {
4924 	int resched = should_resched(PREEMPT_LOCK_OFFSET);
4925 	int ret = 0;
4926 
4927 	lockdep_assert_held(lock);
4928 
4929 	if (spin_needbreak(lock) || resched) {
4930 		spin_unlock(lock);
4931 		if (resched)
4932 			preempt_schedule_common();
4933 		else
4934 			cpu_relax();
4935 		ret = 1;
4936 		spin_lock(lock);
4937 	}
4938 	return ret;
4939 }
4940 EXPORT_SYMBOL(__cond_resched_lock);
4941 
4942 int __sched __cond_resched_softirq(void)
4943 {
4944 	BUG_ON(!in_softirq());
4945 
4946 	if (should_resched(SOFTIRQ_DISABLE_OFFSET)) {
4947 		local_bh_enable();
4948 		preempt_schedule_common();
4949 		local_bh_disable();
4950 		return 1;
4951 	}
4952 	return 0;
4953 }
4954 EXPORT_SYMBOL(__cond_resched_softirq);
4955 
4956 /**
4957  * yield - yield the current processor to other threads.
4958  *
4959  * Do not ever use this function, there's a 99% chance you're doing it wrong.
4960  *
4961  * The scheduler is at all times free to pick the calling task as the most
4962  * eligible task to run, if removing the yield() call from your code breaks
4963  * it, its already broken.
4964  *
4965  * Typical broken usage is:
4966  *
4967  * while (!event)
4968  * 	yield();
4969  *
4970  * where one assumes that yield() will let 'the other' process run that will
4971  * make event true. If the current task is a SCHED_FIFO task that will never
4972  * happen. Never use yield() as a progress guarantee!!
4973  *
4974  * If you want to use yield() to wait for something, use wait_event().
4975  * If you want to use yield() to be 'nice' for others, use cond_resched().
4976  * If you still want to use yield(), do not!
4977  */
4978 void __sched yield(void)
4979 {
4980 	set_current_state(TASK_RUNNING);
4981 	sys_sched_yield();
4982 }
4983 EXPORT_SYMBOL(yield);
4984 
4985 /**
4986  * yield_to - yield the current processor to another thread in
4987  * your thread group, or accelerate that thread toward the
4988  * processor it's on.
4989  * @p: target task
4990  * @preempt: whether task preemption is allowed or not
4991  *
4992  * It's the caller's job to ensure that the target task struct
4993  * can't go away on us before we can do any checks.
4994  *
4995  * Return:
4996  *	true (>0) if we indeed boosted the target task.
4997  *	false (0) if we failed to boost the target.
4998  *	-ESRCH if there's no task to yield to.
4999  */
5000 int __sched yield_to(struct task_struct *p, bool preempt)
5001 {
5002 	struct task_struct *curr = current;
5003 	struct rq *rq, *p_rq;
5004 	unsigned long flags;
5005 	int yielded = 0;
5006 
5007 	local_irq_save(flags);
5008 	rq = this_rq();
5009 
5010 again:
5011 	p_rq = task_rq(p);
5012 	/*
5013 	 * If we're the only runnable task on the rq and target rq also
5014 	 * has only one task, there's absolutely no point in yielding.
5015 	 */
5016 	if (rq->nr_running == 1 && p_rq->nr_running == 1) {
5017 		yielded = -ESRCH;
5018 		goto out_irq;
5019 	}
5020 
5021 	double_rq_lock(rq, p_rq);
5022 	if (task_rq(p) != p_rq) {
5023 		double_rq_unlock(rq, p_rq);
5024 		goto again;
5025 	}
5026 
5027 	if (!curr->sched_class->yield_to_task)
5028 		goto out_unlock;
5029 
5030 	if (curr->sched_class != p->sched_class)
5031 		goto out_unlock;
5032 
5033 	if (task_running(p_rq, p) || p->state)
5034 		goto out_unlock;
5035 
5036 	yielded = curr->sched_class->yield_to_task(rq, p, preempt);
5037 	if (yielded) {
5038 		schedstat_inc(rq->yld_count);
5039 		/*
5040 		 * Make p's CPU reschedule; pick_next_entity takes care of
5041 		 * fairness.
5042 		 */
5043 		if (preempt && rq != p_rq)
5044 			resched_curr(p_rq);
5045 	}
5046 
5047 out_unlock:
5048 	double_rq_unlock(rq, p_rq);
5049 out_irq:
5050 	local_irq_restore(flags);
5051 
5052 	if (yielded > 0)
5053 		schedule();
5054 
5055 	return yielded;
5056 }
5057 EXPORT_SYMBOL_GPL(yield_to);
5058 
5059 /*
5060  * This task is about to go to sleep on IO. Increment rq->nr_iowait so
5061  * that process accounting knows that this is a task in IO wait state.
5062  */
5063 long __sched io_schedule_timeout(long timeout)
5064 {
5065 	int old_iowait = current->in_iowait;
5066 	struct rq *rq;
5067 	long ret;
5068 
5069 	current->in_iowait = 1;
5070 	blk_schedule_flush_plug(current);
5071 
5072 	delayacct_blkio_start();
5073 	rq = raw_rq();
5074 	atomic_inc(&rq->nr_iowait);
5075 	ret = schedule_timeout(timeout);
5076 	current->in_iowait = old_iowait;
5077 	atomic_dec(&rq->nr_iowait);
5078 	delayacct_blkio_end();
5079 
5080 	return ret;
5081 }
5082 EXPORT_SYMBOL(io_schedule_timeout);
5083 
5084 /**
5085  * sys_sched_get_priority_max - return maximum RT priority.
5086  * @policy: scheduling class.
5087  *
5088  * Return: On success, this syscall returns the maximum
5089  * rt_priority that can be used by a given scheduling class.
5090  * On failure, a negative error code is returned.
5091  */
5092 SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
5093 {
5094 	int ret = -EINVAL;
5095 
5096 	switch (policy) {
5097 	case SCHED_FIFO:
5098 	case SCHED_RR:
5099 		ret = MAX_USER_RT_PRIO-1;
5100 		break;
5101 	case SCHED_DEADLINE:
5102 	case SCHED_NORMAL:
5103 	case SCHED_BATCH:
5104 	case SCHED_IDLE:
5105 		ret = 0;
5106 		break;
5107 	}
5108 	return ret;
5109 }
5110 
5111 /**
5112  * sys_sched_get_priority_min - return minimum RT priority.
5113  * @policy: scheduling class.
5114  *
5115  * Return: On success, this syscall returns the minimum
5116  * rt_priority that can be used by a given scheduling class.
5117  * On failure, a negative error code is returned.
5118  */
5119 SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
5120 {
5121 	int ret = -EINVAL;
5122 
5123 	switch (policy) {
5124 	case SCHED_FIFO:
5125 	case SCHED_RR:
5126 		ret = 1;
5127 		break;
5128 	case SCHED_DEADLINE:
5129 	case SCHED_NORMAL:
5130 	case SCHED_BATCH:
5131 	case SCHED_IDLE:
5132 		ret = 0;
5133 	}
5134 	return ret;
5135 }
5136 
5137 /**
5138  * sys_sched_rr_get_interval - return the default timeslice of a process.
5139  * @pid: pid of the process.
5140  * @interval: userspace pointer to the timeslice value.
5141  *
5142  * this syscall writes the default timeslice value of a given process
5143  * into the user-space timespec buffer. A value of '0' means infinity.
5144  *
5145  * Return: On success, 0 and the timeslice is in @interval. Otherwise,
5146  * an error code.
5147  */
5148 SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
5149 		struct timespec __user *, interval)
5150 {
5151 	struct task_struct *p;
5152 	unsigned int time_slice;
5153 	struct rq_flags rf;
5154 	struct timespec t;
5155 	struct rq *rq;
5156 	int retval;
5157 
5158 	if (pid < 0)
5159 		return -EINVAL;
5160 
5161 	retval = -ESRCH;
5162 	rcu_read_lock();
5163 	p = find_process_by_pid(pid);
5164 	if (!p)
5165 		goto out_unlock;
5166 
5167 	retval = security_task_getscheduler(p);
5168 	if (retval)
5169 		goto out_unlock;
5170 
5171 	rq = task_rq_lock(p, &rf);
5172 	time_slice = 0;
5173 	if (p->sched_class->get_rr_interval)
5174 		time_slice = p->sched_class->get_rr_interval(rq, p);
5175 	task_rq_unlock(rq, p, &rf);
5176 
5177 	rcu_read_unlock();
5178 	jiffies_to_timespec(time_slice, &t);
5179 	retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
5180 	return retval;
5181 
5182 out_unlock:
5183 	rcu_read_unlock();
5184 	return retval;
5185 }
5186 
5187 static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
5188 
5189 void sched_show_task(struct task_struct *p)
5190 {
5191 	unsigned long free = 0;
5192 	int ppid;
5193 	unsigned long state = p->state;
5194 
5195 	if (state)
5196 		state = __ffs(state) + 1;
5197 	printk(KERN_INFO "%-15.15s %c", p->comm,
5198 		state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
5199 #if BITS_PER_LONG == 32
5200 	if (state == TASK_RUNNING)
5201 		printk(KERN_CONT " running  ");
5202 	else
5203 		printk(KERN_CONT " %08lx ", thread_saved_pc(p));
5204 #else
5205 	if (state == TASK_RUNNING)
5206 		printk(KERN_CONT "  running task    ");
5207 	else
5208 		printk(KERN_CONT " %016lx ", thread_saved_pc(p));
5209 #endif
5210 #ifdef CONFIG_DEBUG_STACK_USAGE
5211 	free = stack_not_used(p);
5212 #endif
5213 	ppid = 0;
5214 	rcu_read_lock();
5215 	if (pid_alive(p))
5216 		ppid = task_pid_nr(rcu_dereference(p->real_parent));
5217 	rcu_read_unlock();
5218 	printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free,
5219 		task_pid_nr(p), ppid,
5220 		(unsigned long)task_thread_info(p)->flags);
5221 
5222 	print_worker_info(KERN_INFO, p);
5223 	show_stack(p, NULL);
5224 }
5225 
5226 void show_state_filter(unsigned long state_filter)
5227 {
5228 	struct task_struct *g, *p;
5229 
5230 #if BITS_PER_LONG == 32
5231 	printk(KERN_INFO
5232 		"  task                PC stack   pid father\n");
5233 #else
5234 	printk(KERN_INFO
5235 		"  task                        PC stack   pid father\n");
5236 #endif
5237 	rcu_read_lock();
5238 	for_each_process_thread(g, p) {
5239 		/*
5240 		 * reset the NMI-timeout, listing all files on a slow
5241 		 * console might take a lot of time:
5242 		 * Also, reset softlockup watchdogs on all CPUs, because
5243 		 * another CPU might be blocked waiting for us to process
5244 		 * an IPI.
5245 		 */
5246 		touch_nmi_watchdog();
5247 		touch_all_softlockup_watchdogs();
5248 		if (!state_filter || (p->state & state_filter))
5249 			sched_show_task(p);
5250 	}
5251 
5252 #ifdef CONFIG_SCHED_DEBUG
5253 	if (!state_filter)
5254 		sysrq_sched_debug_show();
5255 #endif
5256 	rcu_read_unlock();
5257 	/*
5258 	 * Only show locks if all tasks are dumped:
5259 	 */
5260 	if (!state_filter)
5261 		debug_show_all_locks();
5262 }
5263 
5264 void init_idle_bootup_task(struct task_struct *idle)
5265 {
5266 	idle->sched_class = &idle_sched_class;
5267 }
5268 
5269 /**
5270  * init_idle - set up an idle thread for a given CPU
5271  * @idle: task in question
5272  * @cpu: cpu the idle task belongs to
5273  *
5274  * NOTE: this function does not set the idle thread's NEED_RESCHED
5275  * flag, to make booting more robust.
5276  */
5277 void init_idle(struct task_struct *idle, int cpu)
5278 {
5279 	struct rq *rq = cpu_rq(cpu);
5280 	unsigned long flags;
5281 
5282 	raw_spin_lock_irqsave(&idle->pi_lock, flags);
5283 	raw_spin_lock(&rq->lock);
5284 
5285 	__sched_fork(0, idle);
5286 	idle->state = TASK_RUNNING;
5287 	idle->se.exec_start = sched_clock();
5288 
5289 	kasan_unpoison_task_stack(idle);
5290 
5291 #ifdef CONFIG_SMP
5292 	/*
5293 	 * Its possible that init_idle() gets called multiple times on a task,
5294 	 * in that case do_set_cpus_allowed() will not do the right thing.
5295 	 *
5296 	 * And since this is boot we can forgo the serialization.
5297 	 */
5298 	set_cpus_allowed_common(idle, cpumask_of(cpu));
5299 #endif
5300 	/*
5301 	 * We're having a chicken and egg problem, even though we are
5302 	 * holding rq->lock, the cpu isn't yet set to this cpu so the
5303 	 * lockdep check in task_group() will fail.
5304 	 *
5305 	 * Similar case to sched_fork(). / Alternatively we could
5306 	 * use task_rq_lock() here and obtain the other rq->lock.
5307 	 *
5308 	 * Silence PROVE_RCU
5309 	 */
5310 	rcu_read_lock();
5311 	__set_task_cpu(idle, cpu);
5312 	rcu_read_unlock();
5313 
5314 	rq->curr = rq->idle = idle;
5315 	idle->on_rq = TASK_ON_RQ_QUEUED;
5316 #ifdef CONFIG_SMP
5317 	idle->on_cpu = 1;
5318 #endif
5319 	raw_spin_unlock(&rq->lock);
5320 	raw_spin_unlock_irqrestore(&idle->pi_lock, flags);
5321 
5322 	/* Set the preempt count _outside_ the spinlocks! */
5323 	init_idle_preempt_count(idle, cpu);
5324 
5325 	/*
5326 	 * The idle tasks have their own, simple scheduling class:
5327 	 */
5328 	idle->sched_class = &idle_sched_class;
5329 	ftrace_graph_init_idle_task(idle, cpu);
5330 	vtime_init_idle(idle, cpu);
5331 #ifdef CONFIG_SMP
5332 	sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu);
5333 #endif
5334 }
5335 
5336 int cpuset_cpumask_can_shrink(const struct cpumask *cur,
5337 			      const struct cpumask *trial)
5338 {
5339 	int ret = 1, trial_cpus;
5340 	struct dl_bw *cur_dl_b;
5341 	unsigned long flags;
5342 
5343 	if (!cpumask_weight(cur))
5344 		return ret;
5345 
5346 	rcu_read_lock_sched();
5347 	cur_dl_b = dl_bw_of(cpumask_any(cur));
5348 	trial_cpus = cpumask_weight(trial);
5349 
5350 	raw_spin_lock_irqsave(&cur_dl_b->lock, flags);
5351 	if (cur_dl_b->bw != -1 &&
5352 	    cur_dl_b->bw * trial_cpus < cur_dl_b->total_bw)
5353 		ret = 0;
5354 	raw_spin_unlock_irqrestore(&cur_dl_b->lock, flags);
5355 	rcu_read_unlock_sched();
5356 
5357 	return ret;
5358 }
5359 
5360 int task_can_attach(struct task_struct *p,
5361 		    const struct cpumask *cs_cpus_allowed)
5362 {
5363 	int ret = 0;
5364 
5365 	/*
5366 	 * Kthreads which disallow setaffinity shouldn't be moved
5367 	 * to a new cpuset; we don't want to change their cpu
5368 	 * affinity and isolating such threads by their set of
5369 	 * allowed nodes is unnecessary.  Thus, cpusets are not
5370 	 * applicable for such threads.  This prevents checking for
5371 	 * success of set_cpus_allowed_ptr() on all attached tasks
5372 	 * before cpus_allowed may be changed.
5373 	 */
5374 	if (p->flags & PF_NO_SETAFFINITY) {
5375 		ret = -EINVAL;
5376 		goto out;
5377 	}
5378 
5379 #ifdef CONFIG_SMP
5380 	if (dl_task(p) && !cpumask_intersects(task_rq(p)->rd->span,
5381 					      cs_cpus_allowed)) {
5382 		unsigned int dest_cpu = cpumask_any_and(cpu_active_mask,
5383 							cs_cpus_allowed);
5384 		struct dl_bw *dl_b;
5385 		bool overflow;
5386 		int cpus;
5387 		unsigned long flags;
5388 
5389 		rcu_read_lock_sched();
5390 		dl_b = dl_bw_of(dest_cpu);
5391 		raw_spin_lock_irqsave(&dl_b->lock, flags);
5392 		cpus = dl_bw_cpus(dest_cpu);
5393 		overflow = __dl_overflow(dl_b, cpus, 0, p->dl.dl_bw);
5394 		if (overflow)
5395 			ret = -EBUSY;
5396 		else {
5397 			/*
5398 			 * We reserve space for this task in the destination
5399 			 * root_domain, as we can't fail after this point.
5400 			 * We will free resources in the source root_domain
5401 			 * later on (see set_cpus_allowed_dl()).
5402 			 */
5403 			__dl_add(dl_b, p->dl.dl_bw);
5404 		}
5405 		raw_spin_unlock_irqrestore(&dl_b->lock, flags);
5406 		rcu_read_unlock_sched();
5407 
5408 	}
5409 #endif
5410 out:
5411 	return ret;
5412 }
5413 
5414 #ifdef CONFIG_SMP
5415 
5416 static bool sched_smp_initialized __read_mostly;
5417 
5418 #ifdef CONFIG_NUMA_BALANCING
5419 /* Migrate current task p to target_cpu */
5420 int migrate_task_to(struct task_struct *p, int target_cpu)
5421 {
5422 	struct migration_arg arg = { p, target_cpu };
5423 	int curr_cpu = task_cpu(p);
5424 
5425 	if (curr_cpu == target_cpu)
5426 		return 0;
5427 
5428 	if (!cpumask_test_cpu(target_cpu, tsk_cpus_allowed(p)))
5429 		return -EINVAL;
5430 
5431 	/* TODO: This is not properly updating schedstats */
5432 
5433 	trace_sched_move_numa(p, curr_cpu, target_cpu);
5434 	return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg);
5435 }
5436 
5437 /*
5438  * Requeue a task on a given node and accurately track the number of NUMA
5439  * tasks on the runqueues
5440  */
5441 void sched_setnuma(struct task_struct *p, int nid)
5442 {
5443 	bool queued, running;
5444 	struct rq_flags rf;
5445 	struct rq *rq;
5446 
5447 	rq = task_rq_lock(p, &rf);
5448 	queued = task_on_rq_queued(p);
5449 	running = task_current(rq, p);
5450 
5451 	if (queued)
5452 		dequeue_task(rq, p, DEQUEUE_SAVE);
5453 	if (running)
5454 		put_prev_task(rq, p);
5455 
5456 	p->numa_preferred_nid = nid;
5457 
5458 	if (queued)
5459 		enqueue_task(rq, p, ENQUEUE_RESTORE);
5460 	if (running)
5461 		set_curr_task(rq, p);
5462 	task_rq_unlock(rq, p, &rf);
5463 }
5464 #endif /* CONFIG_NUMA_BALANCING */
5465 
5466 #ifdef CONFIG_HOTPLUG_CPU
5467 /*
5468  * Ensures that the idle task is using init_mm right before its cpu goes
5469  * offline.
5470  */
5471 void idle_task_exit(void)
5472 {
5473 	struct mm_struct *mm = current->active_mm;
5474 
5475 	BUG_ON(cpu_online(smp_processor_id()));
5476 
5477 	if (mm != &init_mm) {
5478 		switch_mm_irqs_off(mm, &init_mm, current);
5479 		finish_arch_post_lock_switch();
5480 	}
5481 	mmdrop(mm);
5482 }
5483 
5484 /*
5485  * Since this CPU is going 'away' for a while, fold any nr_active delta
5486  * we might have. Assumes we're called after migrate_tasks() so that the
5487  * nr_active count is stable. We need to take the teardown thread which
5488  * is calling this into account, so we hand in adjust = 1 to the load
5489  * calculation.
5490  *
5491  * Also see the comment "Global load-average calculations".
5492  */
5493 static void calc_load_migrate(struct rq *rq)
5494 {
5495 	long delta = calc_load_fold_active(rq, 1);
5496 	if (delta)
5497 		atomic_long_add(delta, &calc_load_tasks);
5498 }
5499 
5500 static void put_prev_task_fake(struct rq *rq, struct task_struct *prev)
5501 {
5502 }
5503 
5504 static const struct sched_class fake_sched_class = {
5505 	.put_prev_task = put_prev_task_fake,
5506 };
5507 
5508 static struct task_struct fake_task = {
5509 	/*
5510 	 * Avoid pull_{rt,dl}_task()
5511 	 */
5512 	.prio = MAX_PRIO + 1,
5513 	.sched_class = &fake_sched_class,
5514 };
5515 
5516 /*
5517  * Migrate all tasks from the rq, sleeping tasks will be migrated by
5518  * try_to_wake_up()->select_task_rq().
5519  *
5520  * Called with rq->lock held even though we'er in stop_machine() and
5521  * there's no concurrency possible, we hold the required locks anyway
5522  * because of lock validation efforts.
5523  */
5524 static void migrate_tasks(struct rq *dead_rq)
5525 {
5526 	struct rq *rq = dead_rq;
5527 	struct task_struct *next, *stop = rq->stop;
5528 	struct pin_cookie cookie;
5529 	int dest_cpu;
5530 
5531 	/*
5532 	 * Fudge the rq selection such that the below task selection loop
5533 	 * doesn't get stuck on the currently eligible stop task.
5534 	 *
5535 	 * We're currently inside stop_machine() and the rq is either stuck
5536 	 * in the stop_machine_cpu_stop() loop, or we're executing this code,
5537 	 * either way we should never end up calling schedule() until we're
5538 	 * done here.
5539 	 */
5540 	rq->stop = NULL;
5541 
5542 	/*
5543 	 * put_prev_task() and pick_next_task() sched
5544 	 * class method both need to have an up-to-date
5545 	 * value of rq->clock[_task]
5546 	 */
5547 	update_rq_clock(rq);
5548 
5549 	for (;;) {
5550 		/*
5551 		 * There's this thread running, bail when that's the only
5552 		 * remaining thread.
5553 		 */
5554 		if (rq->nr_running == 1)
5555 			break;
5556 
5557 		/*
5558 		 * pick_next_task assumes pinned rq->lock.
5559 		 */
5560 		cookie = lockdep_pin_lock(&rq->lock);
5561 		next = pick_next_task(rq, &fake_task, cookie);
5562 		BUG_ON(!next);
5563 		next->sched_class->put_prev_task(rq, next);
5564 
5565 		/*
5566 		 * Rules for changing task_struct::cpus_allowed are holding
5567 		 * both pi_lock and rq->lock, such that holding either
5568 		 * stabilizes the mask.
5569 		 *
5570 		 * Drop rq->lock is not quite as disastrous as it usually is
5571 		 * because !cpu_active at this point, which means load-balance
5572 		 * will not interfere. Also, stop-machine.
5573 		 */
5574 		lockdep_unpin_lock(&rq->lock, cookie);
5575 		raw_spin_unlock(&rq->lock);
5576 		raw_spin_lock(&next->pi_lock);
5577 		raw_spin_lock(&rq->lock);
5578 
5579 		/*
5580 		 * Since we're inside stop-machine, _nothing_ should have
5581 		 * changed the task, WARN if weird stuff happened, because in
5582 		 * that case the above rq->lock drop is a fail too.
5583 		 */
5584 		if (WARN_ON(task_rq(next) != rq || !task_on_rq_queued(next))) {
5585 			raw_spin_unlock(&next->pi_lock);
5586 			continue;
5587 		}
5588 
5589 		/* Find suitable destination for @next, with force if needed. */
5590 		dest_cpu = select_fallback_rq(dead_rq->cpu, next);
5591 
5592 		rq = __migrate_task(rq, next, dest_cpu);
5593 		if (rq != dead_rq) {
5594 			raw_spin_unlock(&rq->lock);
5595 			rq = dead_rq;
5596 			raw_spin_lock(&rq->lock);
5597 		}
5598 		raw_spin_unlock(&next->pi_lock);
5599 	}
5600 
5601 	rq->stop = stop;
5602 }
5603 #endif /* CONFIG_HOTPLUG_CPU */
5604 
5605 static void set_rq_online(struct rq *rq)
5606 {
5607 	if (!rq->online) {
5608 		const struct sched_class *class;
5609 
5610 		cpumask_set_cpu(rq->cpu, rq->rd->online);
5611 		rq->online = 1;
5612 
5613 		for_each_class(class) {
5614 			if (class->rq_online)
5615 				class->rq_online(rq);
5616 		}
5617 	}
5618 }
5619 
5620 static void set_rq_offline(struct rq *rq)
5621 {
5622 	if (rq->online) {
5623 		const struct sched_class *class;
5624 
5625 		for_each_class(class) {
5626 			if (class->rq_offline)
5627 				class->rq_offline(rq);
5628 		}
5629 
5630 		cpumask_clear_cpu(rq->cpu, rq->rd->online);
5631 		rq->online = 0;
5632 	}
5633 }
5634 
5635 static void set_cpu_rq_start_time(unsigned int cpu)
5636 {
5637 	struct rq *rq = cpu_rq(cpu);
5638 
5639 	rq->age_stamp = sched_clock_cpu(cpu);
5640 }
5641 
5642 static cpumask_var_t sched_domains_tmpmask; /* sched_domains_mutex */
5643 
5644 #ifdef CONFIG_SCHED_DEBUG
5645 
5646 static __read_mostly int sched_debug_enabled;
5647 
5648 static int __init sched_debug_setup(char *str)
5649 {
5650 	sched_debug_enabled = 1;
5651 
5652 	return 0;
5653 }
5654 early_param("sched_debug", sched_debug_setup);
5655 
5656 static inline bool sched_debug(void)
5657 {
5658 	return sched_debug_enabled;
5659 }
5660 
5661 static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
5662 				  struct cpumask *groupmask)
5663 {
5664 	struct sched_group *group = sd->groups;
5665 
5666 	cpumask_clear(groupmask);
5667 
5668 	printk(KERN_DEBUG "%*s domain %d: ", level, "", level);
5669 
5670 	if (!(sd->flags & SD_LOAD_BALANCE)) {
5671 		printk("does not load-balance\n");
5672 		if (sd->parent)
5673 			printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
5674 					" has parent");
5675 		return -1;
5676 	}
5677 
5678 	printk(KERN_CONT "span %*pbl level %s\n",
5679 	       cpumask_pr_args(sched_domain_span(sd)), sd->name);
5680 
5681 	if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
5682 		printk(KERN_ERR "ERROR: domain->span does not contain "
5683 				"CPU%d\n", cpu);
5684 	}
5685 	if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
5686 		printk(KERN_ERR "ERROR: domain->groups does not contain"
5687 				" CPU%d\n", cpu);
5688 	}
5689 
5690 	printk(KERN_DEBUG "%*s groups:", level + 1, "");
5691 	do {
5692 		if (!group) {
5693 			printk("\n");
5694 			printk(KERN_ERR "ERROR: group is NULL\n");
5695 			break;
5696 		}
5697 
5698 		if (!cpumask_weight(sched_group_cpus(group))) {
5699 			printk(KERN_CONT "\n");
5700 			printk(KERN_ERR "ERROR: empty group\n");
5701 			break;
5702 		}
5703 
5704 		if (!(sd->flags & SD_OVERLAP) &&
5705 		    cpumask_intersects(groupmask, sched_group_cpus(group))) {
5706 			printk(KERN_CONT "\n");
5707 			printk(KERN_ERR "ERROR: repeated CPUs\n");
5708 			break;
5709 		}
5710 
5711 		cpumask_or(groupmask, groupmask, sched_group_cpus(group));
5712 
5713 		printk(KERN_CONT " %*pbl",
5714 		       cpumask_pr_args(sched_group_cpus(group)));
5715 		if (group->sgc->capacity != SCHED_CAPACITY_SCALE) {
5716 			printk(KERN_CONT " (cpu_capacity = %d)",
5717 				group->sgc->capacity);
5718 		}
5719 
5720 		group = group->next;
5721 	} while (group != sd->groups);
5722 	printk(KERN_CONT "\n");
5723 
5724 	if (!cpumask_equal(sched_domain_span(sd), groupmask))
5725 		printk(KERN_ERR "ERROR: groups don't span domain->span\n");
5726 
5727 	if (sd->parent &&
5728 	    !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
5729 		printk(KERN_ERR "ERROR: parent span is not a superset "
5730 			"of domain->span\n");
5731 	return 0;
5732 }
5733 
5734 static void sched_domain_debug(struct sched_domain *sd, int cpu)
5735 {
5736 	int level = 0;
5737 
5738 	if (!sched_debug_enabled)
5739 		return;
5740 
5741 	if (!sd) {
5742 		printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
5743 		return;
5744 	}
5745 
5746 	printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);
5747 
5748 	for (;;) {
5749 		if (sched_domain_debug_one(sd, cpu, level, sched_domains_tmpmask))
5750 			break;
5751 		level++;
5752 		sd = sd->parent;
5753 		if (!sd)
5754 			break;
5755 	}
5756 }
5757 #else /* !CONFIG_SCHED_DEBUG */
5758 
5759 # define sched_debug_enabled 0
5760 # define sched_domain_debug(sd, cpu) do { } while (0)
5761 static inline bool sched_debug(void)
5762 {
5763 	return false;
5764 }
5765 #endif /* CONFIG_SCHED_DEBUG */
5766 
5767 static int sd_degenerate(struct sched_domain *sd)
5768 {
5769 	if (cpumask_weight(sched_domain_span(sd)) == 1)
5770 		return 1;
5771 
5772 	/* Following flags need at least 2 groups */
5773 	if (sd->flags & (SD_LOAD_BALANCE |
5774 			 SD_BALANCE_NEWIDLE |
5775 			 SD_BALANCE_FORK |
5776 			 SD_BALANCE_EXEC |
5777 			 SD_SHARE_CPUCAPACITY |
5778 			 SD_ASYM_CPUCAPACITY |
5779 			 SD_SHARE_PKG_RESOURCES |
5780 			 SD_SHARE_POWERDOMAIN)) {
5781 		if (sd->groups != sd->groups->next)
5782 			return 0;
5783 	}
5784 
5785 	/* Following flags don't use groups */
5786 	if (sd->flags & (SD_WAKE_AFFINE))
5787 		return 0;
5788 
5789 	return 1;
5790 }
5791 
5792 static int
5793 sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
5794 {
5795 	unsigned long cflags = sd->flags, pflags = parent->flags;
5796 
5797 	if (sd_degenerate(parent))
5798 		return 1;
5799 
5800 	if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
5801 		return 0;
5802 
5803 	/* Flags needing groups don't count if only 1 group in parent */
5804 	if (parent->groups == parent->groups->next) {
5805 		pflags &= ~(SD_LOAD_BALANCE |
5806 				SD_BALANCE_NEWIDLE |
5807 				SD_BALANCE_FORK |
5808 				SD_BALANCE_EXEC |
5809 				SD_ASYM_CPUCAPACITY |
5810 				SD_SHARE_CPUCAPACITY |
5811 				SD_SHARE_PKG_RESOURCES |
5812 				SD_PREFER_SIBLING |
5813 				SD_SHARE_POWERDOMAIN);
5814 		if (nr_node_ids == 1)
5815 			pflags &= ~SD_SERIALIZE;
5816 	}
5817 	if (~cflags & pflags)
5818 		return 0;
5819 
5820 	return 1;
5821 }
5822 
5823 static void free_rootdomain(struct rcu_head *rcu)
5824 {
5825 	struct root_domain *rd = container_of(rcu, struct root_domain, rcu);
5826 
5827 	cpupri_cleanup(&rd->cpupri);
5828 	cpudl_cleanup(&rd->cpudl);
5829 	free_cpumask_var(rd->dlo_mask);
5830 	free_cpumask_var(rd->rto_mask);
5831 	free_cpumask_var(rd->online);
5832 	free_cpumask_var(rd->span);
5833 	kfree(rd);
5834 }
5835 
5836 static void rq_attach_root(struct rq *rq, struct root_domain *rd)
5837 {
5838 	struct root_domain *old_rd = NULL;
5839 	unsigned long flags;
5840 
5841 	raw_spin_lock_irqsave(&rq->lock, flags);
5842 
5843 	if (rq->rd) {
5844 		old_rd = rq->rd;
5845 
5846 		if (cpumask_test_cpu(rq->cpu, old_rd->online))
5847 			set_rq_offline(rq);
5848 
5849 		cpumask_clear_cpu(rq->cpu, old_rd->span);
5850 
5851 		/*
5852 		 * If we dont want to free the old_rd yet then
5853 		 * set old_rd to NULL to skip the freeing later
5854 		 * in this function:
5855 		 */
5856 		if (!atomic_dec_and_test(&old_rd->refcount))
5857 			old_rd = NULL;
5858 	}
5859 
5860 	atomic_inc(&rd->refcount);
5861 	rq->rd = rd;
5862 
5863 	cpumask_set_cpu(rq->cpu, rd->span);
5864 	if (cpumask_test_cpu(rq->cpu, cpu_active_mask))
5865 		set_rq_online(rq);
5866 
5867 	raw_spin_unlock_irqrestore(&rq->lock, flags);
5868 
5869 	if (old_rd)
5870 		call_rcu_sched(&old_rd->rcu, free_rootdomain);
5871 }
5872 
5873 static int init_rootdomain(struct root_domain *rd)
5874 {
5875 	memset(rd, 0, sizeof(*rd));
5876 
5877 	if (!zalloc_cpumask_var(&rd->span, GFP_KERNEL))
5878 		goto out;
5879 	if (!zalloc_cpumask_var(&rd->online, GFP_KERNEL))
5880 		goto free_span;
5881 	if (!zalloc_cpumask_var(&rd->dlo_mask, GFP_KERNEL))
5882 		goto free_online;
5883 	if (!zalloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
5884 		goto free_dlo_mask;
5885 
5886 	init_dl_bw(&rd->dl_bw);
5887 	if (cpudl_init(&rd->cpudl) != 0)
5888 		goto free_dlo_mask;
5889 
5890 	if (cpupri_init(&rd->cpupri) != 0)
5891 		goto free_rto_mask;
5892 	return 0;
5893 
5894 free_rto_mask:
5895 	free_cpumask_var(rd->rto_mask);
5896 free_dlo_mask:
5897 	free_cpumask_var(rd->dlo_mask);
5898 free_online:
5899 	free_cpumask_var(rd->online);
5900 free_span:
5901 	free_cpumask_var(rd->span);
5902 out:
5903 	return -ENOMEM;
5904 }
5905 
5906 /*
5907  * By default the system creates a single root-domain with all cpus as
5908  * members (mimicking the global state we have today).
5909  */
5910 struct root_domain def_root_domain;
5911 
5912 static void init_defrootdomain(void)
5913 {
5914 	init_rootdomain(&def_root_domain);
5915 
5916 	atomic_set(&def_root_domain.refcount, 1);
5917 }
5918 
5919 static struct root_domain *alloc_rootdomain(void)
5920 {
5921 	struct root_domain *rd;
5922 
5923 	rd = kmalloc(sizeof(*rd), GFP_KERNEL);
5924 	if (!rd)
5925 		return NULL;
5926 
5927 	if (init_rootdomain(rd) != 0) {
5928 		kfree(rd);
5929 		return NULL;
5930 	}
5931 
5932 	return rd;
5933 }
5934 
5935 static void free_sched_groups(struct sched_group *sg, int free_sgc)
5936 {
5937 	struct sched_group *tmp, *first;
5938 
5939 	if (!sg)
5940 		return;
5941 
5942 	first = sg;
5943 	do {
5944 		tmp = sg->next;
5945 
5946 		if (free_sgc && atomic_dec_and_test(&sg->sgc->ref))
5947 			kfree(sg->sgc);
5948 
5949 		kfree(sg);
5950 		sg = tmp;
5951 	} while (sg != first);
5952 }
5953 
5954 static void destroy_sched_domain(struct sched_domain *sd)
5955 {
5956 	/*
5957 	 * If its an overlapping domain it has private groups, iterate and
5958 	 * nuke them all.
5959 	 */
5960 	if (sd->flags & SD_OVERLAP) {
5961 		free_sched_groups(sd->groups, 1);
5962 	} else if (atomic_dec_and_test(&sd->groups->ref)) {
5963 		kfree(sd->groups->sgc);
5964 		kfree(sd->groups);
5965 	}
5966 	if (sd->shared && atomic_dec_and_test(&sd->shared->ref))
5967 		kfree(sd->shared);
5968 	kfree(sd);
5969 }
5970 
5971 static void destroy_sched_domains_rcu(struct rcu_head *rcu)
5972 {
5973 	struct sched_domain *sd = container_of(rcu, struct sched_domain, rcu);
5974 
5975 	while (sd) {
5976 		struct sched_domain *parent = sd->parent;
5977 		destroy_sched_domain(sd);
5978 		sd = parent;
5979 	}
5980 }
5981 
5982 static void destroy_sched_domains(struct sched_domain *sd)
5983 {
5984 	if (sd)
5985 		call_rcu(&sd->rcu, destroy_sched_domains_rcu);
5986 }
5987 
5988 /*
5989  * Keep a special pointer to the highest sched_domain that has
5990  * SD_SHARE_PKG_RESOURCE set (Last Level Cache Domain) for this
5991  * allows us to avoid some pointer chasing select_idle_sibling().
5992  *
5993  * Also keep a unique ID per domain (we use the first cpu number in
5994  * the cpumask of the domain), this allows us to quickly tell if
5995  * two cpus are in the same cache domain, see cpus_share_cache().
5996  */
5997 DEFINE_PER_CPU(struct sched_domain *, sd_llc);
5998 DEFINE_PER_CPU(int, sd_llc_size);
5999 DEFINE_PER_CPU(int, sd_llc_id);
6000 DEFINE_PER_CPU(struct sched_domain_shared *, sd_llc_shared);
6001 DEFINE_PER_CPU(struct sched_domain *, sd_numa);
6002 DEFINE_PER_CPU(struct sched_domain *, sd_asym);
6003 
6004 static void update_top_cache_domain(int cpu)
6005 {
6006 	struct sched_domain_shared *sds = NULL;
6007 	struct sched_domain *sd;
6008 	int id = cpu;
6009 	int size = 1;
6010 
6011 	sd = highest_flag_domain(cpu, SD_SHARE_PKG_RESOURCES);
6012 	if (sd) {
6013 		id = cpumask_first(sched_domain_span(sd));
6014 		size = cpumask_weight(sched_domain_span(sd));
6015 		sds = sd->shared;
6016 	}
6017 
6018 	rcu_assign_pointer(per_cpu(sd_llc, cpu), sd);
6019 	per_cpu(sd_llc_size, cpu) = size;
6020 	per_cpu(sd_llc_id, cpu) = id;
6021 	rcu_assign_pointer(per_cpu(sd_llc_shared, cpu), sds);
6022 
6023 	sd = lowest_flag_domain(cpu, SD_NUMA);
6024 	rcu_assign_pointer(per_cpu(sd_numa, cpu), sd);
6025 
6026 	sd = highest_flag_domain(cpu, SD_ASYM_PACKING);
6027 	rcu_assign_pointer(per_cpu(sd_asym, cpu), sd);
6028 }
6029 
6030 /*
6031  * Attach the domain 'sd' to 'cpu' as its base domain. Callers must
6032  * hold the hotplug lock.
6033  */
6034 static void
6035 cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
6036 {
6037 	struct rq *rq = cpu_rq(cpu);
6038 	struct sched_domain *tmp;
6039 
6040 	/* Remove the sched domains which do not contribute to scheduling. */
6041 	for (tmp = sd; tmp; ) {
6042 		struct sched_domain *parent = tmp->parent;
6043 		if (!parent)
6044 			break;
6045 
6046 		if (sd_parent_degenerate(tmp, parent)) {
6047 			tmp->parent = parent->parent;
6048 			if (parent->parent)
6049 				parent->parent->child = tmp;
6050 			/*
6051 			 * Transfer SD_PREFER_SIBLING down in case of a
6052 			 * degenerate parent; the spans match for this
6053 			 * so the property transfers.
6054 			 */
6055 			if (parent->flags & SD_PREFER_SIBLING)
6056 				tmp->flags |= SD_PREFER_SIBLING;
6057 			destroy_sched_domain(parent);
6058 		} else
6059 			tmp = tmp->parent;
6060 	}
6061 
6062 	if (sd && sd_degenerate(sd)) {
6063 		tmp = sd;
6064 		sd = sd->parent;
6065 		destroy_sched_domain(tmp);
6066 		if (sd)
6067 			sd->child = NULL;
6068 	}
6069 
6070 	sched_domain_debug(sd, cpu);
6071 
6072 	rq_attach_root(rq, rd);
6073 	tmp = rq->sd;
6074 	rcu_assign_pointer(rq->sd, sd);
6075 	destroy_sched_domains(tmp);
6076 
6077 	update_top_cache_domain(cpu);
6078 }
6079 
6080 /* Setup the mask of cpus configured for isolated domains */
6081 static int __init isolated_cpu_setup(char *str)
6082 {
6083 	int ret;
6084 
6085 	alloc_bootmem_cpumask_var(&cpu_isolated_map);
6086 	ret = cpulist_parse(str, cpu_isolated_map);
6087 	if (ret) {
6088 		pr_err("sched: Error, all isolcpus= values must be between 0 and %d\n", nr_cpu_ids);
6089 		return 0;
6090 	}
6091 	return 1;
6092 }
6093 __setup("isolcpus=", isolated_cpu_setup);
6094 
6095 struct s_data {
6096 	struct sched_domain ** __percpu sd;
6097 	struct root_domain	*rd;
6098 };
6099 
6100 enum s_alloc {
6101 	sa_rootdomain,
6102 	sa_sd,
6103 	sa_sd_storage,
6104 	sa_none,
6105 };
6106 
6107 /*
6108  * Build an iteration mask that can exclude certain CPUs from the upwards
6109  * domain traversal.
6110  *
6111  * Asymmetric node setups can result in situations where the domain tree is of
6112  * unequal depth, make sure to skip domains that already cover the entire
6113  * range.
6114  *
6115  * In that case build_sched_domains() will have terminated the iteration early
6116  * and our sibling sd spans will be empty. Domains should always include the
6117  * cpu they're built on, so check that.
6118  *
6119  */
6120 static void build_group_mask(struct sched_domain *sd, struct sched_group *sg)
6121 {
6122 	const struct cpumask *span = sched_domain_span(sd);
6123 	struct sd_data *sdd = sd->private;
6124 	struct sched_domain *sibling;
6125 	int i;
6126 
6127 	for_each_cpu(i, span) {
6128 		sibling = *per_cpu_ptr(sdd->sd, i);
6129 		if (!cpumask_test_cpu(i, sched_domain_span(sibling)))
6130 			continue;
6131 
6132 		cpumask_set_cpu(i, sched_group_mask(sg));
6133 	}
6134 }
6135 
6136 /*
6137  * Return the canonical balance cpu for this group, this is the first cpu
6138  * of this group that's also in the iteration mask.
6139  */
6140 int group_balance_cpu(struct sched_group *sg)
6141 {
6142 	return cpumask_first_and(sched_group_cpus(sg), sched_group_mask(sg));
6143 }
6144 
6145 static int
6146 build_overlap_sched_groups(struct sched_domain *sd, int cpu)
6147 {
6148 	struct sched_group *first = NULL, *last = NULL, *groups = NULL, *sg;
6149 	const struct cpumask *span = sched_domain_span(sd);
6150 	struct cpumask *covered = sched_domains_tmpmask;
6151 	struct sd_data *sdd = sd->private;
6152 	struct sched_domain *sibling;
6153 	int i;
6154 
6155 	cpumask_clear(covered);
6156 
6157 	for_each_cpu(i, span) {
6158 		struct cpumask *sg_span;
6159 
6160 		if (cpumask_test_cpu(i, covered))
6161 			continue;
6162 
6163 		sibling = *per_cpu_ptr(sdd->sd, i);
6164 
6165 		/* See the comment near build_group_mask(). */
6166 		if (!cpumask_test_cpu(i, sched_domain_span(sibling)))
6167 			continue;
6168 
6169 		sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
6170 				GFP_KERNEL, cpu_to_node(cpu));
6171 
6172 		if (!sg)
6173 			goto fail;
6174 
6175 		sg_span = sched_group_cpus(sg);
6176 		if (sibling->child)
6177 			cpumask_copy(sg_span, sched_domain_span(sibling->child));
6178 		else
6179 			cpumask_set_cpu(i, sg_span);
6180 
6181 		cpumask_or(covered, covered, sg_span);
6182 
6183 		sg->sgc = *per_cpu_ptr(sdd->sgc, i);
6184 		if (atomic_inc_return(&sg->sgc->ref) == 1)
6185 			build_group_mask(sd, sg);
6186 
6187 		/*
6188 		 * Initialize sgc->capacity such that even if we mess up the
6189 		 * domains and no possible iteration will get us here, we won't
6190 		 * die on a /0 trap.
6191 		 */
6192 		sg->sgc->capacity = SCHED_CAPACITY_SCALE * cpumask_weight(sg_span);
6193 
6194 		/*
6195 		 * Make sure the first group of this domain contains the
6196 		 * canonical balance cpu. Otherwise the sched_domain iteration
6197 		 * breaks. See update_sg_lb_stats().
6198 		 */
6199 		if ((!groups && cpumask_test_cpu(cpu, sg_span)) ||
6200 		    group_balance_cpu(sg) == cpu)
6201 			groups = sg;
6202 
6203 		if (!first)
6204 			first = sg;
6205 		if (last)
6206 			last->next = sg;
6207 		last = sg;
6208 		last->next = first;
6209 	}
6210 	sd->groups = groups;
6211 
6212 	return 0;
6213 
6214 fail:
6215 	free_sched_groups(first, 0);
6216 
6217 	return -ENOMEM;
6218 }
6219 
6220 static int get_group(int cpu, struct sd_data *sdd, struct sched_group **sg)
6221 {
6222 	struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu);
6223 	struct sched_domain *child = sd->child;
6224 
6225 	if (child)
6226 		cpu = cpumask_first(sched_domain_span(child));
6227 
6228 	if (sg) {
6229 		*sg = *per_cpu_ptr(sdd->sg, cpu);
6230 		(*sg)->sgc = *per_cpu_ptr(sdd->sgc, cpu);
6231 		atomic_set(&(*sg)->sgc->ref, 1); /* for claim_allocations */
6232 	}
6233 
6234 	return cpu;
6235 }
6236 
6237 /*
6238  * build_sched_groups will build a circular linked list of the groups
6239  * covered by the given span, and will set each group's ->cpumask correctly,
6240  * and ->cpu_capacity to 0.
6241  *
6242  * Assumes the sched_domain tree is fully constructed
6243  */
6244 static int
6245 build_sched_groups(struct sched_domain *sd, int cpu)
6246 {
6247 	struct sched_group *first = NULL, *last = NULL;
6248 	struct sd_data *sdd = sd->private;
6249 	const struct cpumask *span = sched_domain_span(sd);
6250 	struct cpumask *covered;
6251 	int i;
6252 
6253 	get_group(cpu, sdd, &sd->groups);
6254 	atomic_inc(&sd->groups->ref);
6255 
6256 	if (cpu != cpumask_first(span))
6257 		return 0;
6258 
6259 	lockdep_assert_held(&sched_domains_mutex);
6260 	covered = sched_domains_tmpmask;
6261 
6262 	cpumask_clear(covered);
6263 
6264 	for_each_cpu(i, span) {
6265 		struct sched_group *sg;
6266 		int group, j;
6267 
6268 		if (cpumask_test_cpu(i, covered))
6269 			continue;
6270 
6271 		group = get_group(i, sdd, &sg);
6272 		cpumask_setall(sched_group_mask(sg));
6273 
6274 		for_each_cpu(j, span) {
6275 			if (get_group(j, sdd, NULL) != group)
6276 				continue;
6277 
6278 			cpumask_set_cpu(j, covered);
6279 			cpumask_set_cpu(j, sched_group_cpus(sg));
6280 		}
6281 
6282 		if (!first)
6283 			first = sg;
6284 		if (last)
6285 			last->next = sg;
6286 		last = sg;
6287 	}
6288 	last->next = first;
6289 
6290 	return 0;
6291 }
6292 
6293 /*
6294  * Initialize sched groups cpu_capacity.
6295  *
6296  * cpu_capacity indicates the capacity of sched group, which is used while
6297  * distributing the load between different sched groups in a sched domain.
6298  * Typically cpu_capacity for all the groups in a sched domain will be same
6299  * unless there are asymmetries in the topology. If there are asymmetries,
6300  * group having more cpu_capacity will pickup more load compared to the
6301  * group having less cpu_capacity.
6302  */
6303 static void init_sched_groups_capacity(int cpu, struct sched_domain *sd)
6304 {
6305 	struct sched_group *sg = sd->groups;
6306 
6307 	WARN_ON(!sg);
6308 
6309 	do {
6310 		sg->group_weight = cpumask_weight(sched_group_cpus(sg));
6311 		sg = sg->next;
6312 	} while (sg != sd->groups);
6313 
6314 	if (cpu != group_balance_cpu(sg))
6315 		return;
6316 
6317 	update_group_capacity(sd, cpu);
6318 }
6319 
6320 /*
6321  * Initializers for schedule domains
6322  * Non-inlined to reduce accumulated stack pressure in build_sched_domains()
6323  */
6324 
6325 static int default_relax_domain_level = -1;
6326 int sched_domain_level_max;
6327 
6328 static int __init setup_relax_domain_level(char *str)
6329 {
6330 	if (kstrtoint(str, 0, &default_relax_domain_level))
6331 		pr_warn("Unable to set relax_domain_level\n");
6332 
6333 	return 1;
6334 }
6335 __setup("relax_domain_level=", setup_relax_domain_level);
6336 
6337 static void set_domain_attribute(struct sched_domain *sd,
6338 				 struct sched_domain_attr *attr)
6339 {
6340 	int request;
6341 
6342 	if (!attr || attr->relax_domain_level < 0) {
6343 		if (default_relax_domain_level < 0)
6344 			return;
6345 		else
6346 			request = default_relax_domain_level;
6347 	} else
6348 		request = attr->relax_domain_level;
6349 	if (request < sd->level) {
6350 		/* turn off idle balance on this domain */
6351 		sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
6352 	} else {
6353 		/* turn on idle balance on this domain */
6354 		sd->flags |= (SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
6355 	}
6356 }
6357 
6358 static void __sdt_free(const struct cpumask *cpu_map);
6359 static int __sdt_alloc(const struct cpumask *cpu_map);
6360 
6361 static void __free_domain_allocs(struct s_data *d, enum s_alloc what,
6362 				 const struct cpumask *cpu_map)
6363 {
6364 	switch (what) {
6365 	case sa_rootdomain:
6366 		if (!atomic_read(&d->rd->refcount))
6367 			free_rootdomain(&d->rd->rcu); /* fall through */
6368 	case sa_sd:
6369 		free_percpu(d->sd); /* fall through */
6370 	case sa_sd_storage:
6371 		__sdt_free(cpu_map); /* fall through */
6372 	case sa_none:
6373 		break;
6374 	}
6375 }
6376 
6377 static enum s_alloc __visit_domain_allocation_hell(struct s_data *d,
6378 						   const struct cpumask *cpu_map)
6379 {
6380 	memset(d, 0, sizeof(*d));
6381 
6382 	if (__sdt_alloc(cpu_map))
6383 		return sa_sd_storage;
6384 	d->sd = alloc_percpu(struct sched_domain *);
6385 	if (!d->sd)
6386 		return sa_sd_storage;
6387 	d->rd = alloc_rootdomain();
6388 	if (!d->rd)
6389 		return sa_sd;
6390 	return sa_rootdomain;
6391 }
6392 
6393 /*
6394  * NULL the sd_data elements we've used to build the sched_domain and
6395  * sched_group structure so that the subsequent __free_domain_allocs()
6396  * will not free the data we're using.
6397  */
6398 static void claim_allocations(int cpu, struct sched_domain *sd)
6399 {
6400 	struct sd_data *sdd = sd->private;
6401 
6402 	WARN_ON_ONCE(*per_cpu_ptr(sdd->sd, cpu) != sd);
6403 	*per_cpu_ptr(sdd->sd, cpu) = NULL;
6404 
6405 	if (atomic_read(&(*per_cpu_ptr(sdd->sds, cpu))->ref))
6406 		*per_cpu_ptr(sdd->sds, cpu) = NULL;
6407 
6408 	if (atomic_read(&(*per_cpu_ptr(sdd->sg, cpu))->ref))
6409 		*per_cpu_ptr(sdd->sg, cpu) = NULL;
6410 
6411 	if (atomic_read(&(*per_cpu_ptr(sdd->sgc, cpu))->ref))
6412 		*per_cpu_ptr(sdd->sgc, cpu) = NULL;
6413 }
6414 
6415 #ifdef CONFIG_NUMA
6416 static int sched_domains_numa_levels;
6417 enum numa_topology_type sched_numa_topology_type;
6418 static int *sched_domains_numa_distance;
6419 int sched_max_numa_distance;
6420 static struct cpumask ***sched_domains_numa_masks;
6421 static int sched_domains_curr_level;
6422 #endif
6423 
6424 /*
6425  * SD_flags allowed in topology descriptions.
6426  *
6427  * These flags are purely descriptive of the topology and do not prescribe
6428  * behaviour. Behaviour is artificial and mapped in the below sd_init()
6429  * function:
6430  *
6431  *   SD_SHARE_CPUCAPACITY   - describes SMT topologies
6432  *   SD_SHARE_PKG_RESOURCES - describes shared caches
6433  *   SD_NUMA                - describes NUMA topologies
6434  *   SD_SHARE_POWERDOMAIN   - describes shared power domain
6435  *   SD_ASYM_CPUCAPACITY    - describes mixed capacity topologies
6436  *
6437  * Odd one out, which beside describing the topology has a quirk also
6438  * prescribes the desired behaviour that goes along with it:
6439  *
6440  *   SD_ASYM_PACKING        - describes SMT quirks
6441  */
6442 #define TOPOLOGY_SD_FLAGS		\
6443 	(SD_SHARE_CPUCAPACITY |		\
6444 	 SD_SHARE_PKG_RESOURCES |	\
6445 	 SD_NUMA |			\
6446 	 SD_ASYM_PACKING |		\
6447 	 SD_ASYM_CPUCAPACITY |		\
6448 	 SD_SHARE_POWERDOMAIN)
6449 
6450 static struct sched_domain *
6451 sd_init(struct sched_domain_topology_level *tl,
6452 	const struct cpumask *cpu_map,
6453 	struct sched_domain *child, int cpu)
6454 {
6455 	struct sd_data *sdd = &tl->data;
6456 	struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu);
6457 	int sd_id, sd_weight, sd_flags = 0;
6458 
6459 #ifdef CONFIG_NUMA
6460 	/*
6461 	 * Ugly hack to pass state to sd_numa_mask()...
6462 	 */
6463 	sched_domains_curr_level = tl->numa_level;
6464 #endif
6465 
6466 	sd_weight = cpumask_weight(tl->mask(cpu));
6467 
6468 	if (tl->sd_flags)
6469 		sd_flags = (*tl->sd_flags)();
6470 	if (WARN_ONCE(sd_flags & ~TOPOLOGY_SD_FLAGS,
6471 			"wrong sd_flags in topology description\n"))
6472 		sd_flags &= ~TOPOLOGY_SD_FLAGS;
6473 
6474 	*sd = (struct sched_domain){
6475 		.min_interval		= sd_weight,
6476 		.max_interval		= 2*sd_weight,
6477 		.busy_factor		= 32,
6478 		.imbalance_pct		= 125,
6479 
6480 		.cache_nice_tries	= 0,
6481 		.busy_idx		= 0,
6482 		.idle_idx		= 0,
6483 		.newidle_idx		= 0,
6484 		.wake_idx		= 0,
6485 		.forkexec_idx		= 0,
6486 
6487 		.flags			= 1*SD_LOAD_BALANCE
6488 					| 1*SD_BALANCE_NEWIDLE
6489 					| 1*SD_BALANCE_EXEC
6490 					| 1*SD_BALANCE_FORK
6491 					| 0*SD_BALANCE_WAKE
6492 					| 1*SD_WAKE_AFFINE
6493 					| 0*SD_SHARE_CPUCAPACITY
6494 					| 0*SD_SHARE_PKG_RESOURCES
6495 					| 0*SD_SERIALIZE
6496 					| 0*SD_PREFER_SIBLING
6497 					| 0*SD_NUMA
6498 					| sd_flags
6499 					,
6500 
6501 		.last_balance		= jiffies,
6502 		.balance_interval	= sd_weight,
6503 		.smt_gain		= 0,
6504 		.max_newidle_lb_cost	= 0,
6505 		.next_decay_max_lb_cost	= jiffies,
6506 		.child			= child,
6507 #ifdef CONFIG_SCHED_DEBUG
6508 		.name			= tl->name,
6509 #endif
6510 	};
6511 
6512 	cpumask_and(sched_domain_span(sd), cpu_map, tl->mask(cpu));
6513 	sd_id = cpumask_first(sched_domain_span(sd));
6514 
6515 	/*
6516 	 * Convert topological properties into behaviour.
6517 	 */
6518 
6519 	if (sd->flags & SD_ASYM_CPUCAPACITY) {
6520 		struct sched_domain *t = sd;
6521 
6522 		for_each_lower_domain(t)
6523 			t->flags |= SD_BALANCE_WAKE;
6524 	}
6525 
6526 	if (sd->flags & SD_SHARE_CPUCAPACITY) {
6527 		sd->flags |= SD_PREFER_SIBLING;
6528 		sd->imbalance_pct = 110;
6529 		sd->smt_gain = 1178; /* ~15% */
6530 
6531 	} else if (sd->flags & SD_SHARE_PKG_RESOURCES) {
6532 		sd->imbalance_pct = 117;
6533 		sd->cache_nice_tries = 1;
6534 		sd->busy_idx = 2;
6535 
6536 #ifdef CONFIG_NUMA
6537 	} else if (sd->flags & SD_NUMA) {
6538 		sd->cache_nice_tries = 2;
6539 		sd->busy_idx = 3;
6540 		sd->idle_idx = 2;
6541 
6542 		sd->flags |= SD_SERIALIZE;
6543 		if (sched_domains_numa_distance[tl->numa_level] > RECLAIM_DISTANCE) {
6544 			sd->flags &= ~(SD_BALANCE_EXEC |
6545 				       SD_BALANCE_FORK |
6546 				       SD_WAKE_AFFINE);
6547 		}
6548 
6549 #endif
6550 	} else {
6551 		sd->flags |= SD_PREFER_SIBLING;
6552 		sd->cache_nice_tries = 1;
6553 		sd->busy_idx = 2;
6554 		sd->idle_idx = 1;
6555 	}
6556 
6557 	/*
6558 	 * For all levels sharing cache; connect a sched_domain_shared
6559 	 * instance.
6560 	 */
6561 	if (sd->flags & SD_SHARE_PKG_RESOURCES) {
6562 		sd->shared = *per_cpu_ptr(sdd->sds, sd_id);
6563 		atomic_inc(&sd->shared->ref);
6564 		atomic_set(&sd->shared->nr_busy_cpus, sd_weight);
6565 	}
6566 
6567 	sd->private = sdd;
6568 
6569 	return sd;
6570 }
6571 
6572 /*
6573  * Topology list, bottom-up.
6574  */
6575 static struct sched_domain_topology_level default_topology[] = {
6576 #ifdef CONFIG_SCHED_SMT
6577 	{ cpu_smt_mask, cpu_smt_flags, SD_INIT_NAME(SMT) },
6578 #endif
6579 #ifdef CONFIG_SCHED_MC
6580 	{ cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) },
6581 #endif
6582 	{ cpu_cpu_mask, SD_INIT_NAME(DIE) },
6583 	{ NULL, },
6584 };
6585 
6586 static struct sched_domain_topology_level *sched_domain_topology =
6587 	default_topology;
6588 
6589 #define for_each_sd_topology(tl)			\
6590 	for (tl = sched_domain_topology; tl->mask; tl++)
6591 
6592 void set_sched_topology(struct sched_domain_topology_level *tl)
6593 {
6594 	if (WARN_ON_ONCE(sched_smp_initialized))
6595 		return;
6596 
6597 	sched_domain_topology = tl;
6598 }
6599 
6600 #ifdef CONFIG_NUMA
6601 
6602 static const struct cpumask *sd_numa_mask(int cpu)
6603 {
6604 	return sched_domains_numa_masks[sched_domains_curr_level][cpu_to_node(cpu)];
6605 }
6606 
6607 static void sched_numa_warn(const char *str)
6608 {
6609 	static int done = false;
6610 	int i,j;
6611 
6612 	if (done)
6613 		return;
6614 
6615 	done = true;
6616 
6617 	printk(KERN_WARNING "ERROR: %s\n\n", str);
6618 
6619 	for (i = 0; i < nr_node_ids; i++) {
6620 		printk(KERN_WARNING "  ");
6621 		for (j = 0; j < nr_node_ids; j++)
6622 			printk(KERN_CONT "%02d ", node_distance(i,j));
6623 		printk(KERN_CONT "\n");
6624 	}
6625 	printk(KERN_WARNING "\n");
6626 }
6627 
6628 bool find_numa_distance(int distance)
6629 {
6630 	int i;
6631 
6632 	if (distance == node_distance(0, 0))
6633 		return true;
6634 
6635 	for (i = 0; i < sched_domains_numa_levels; i++) {
6636 		if (sched_domains_numa_distance[i] == distance)
6637 			return true;
6638 	}
6639 
6640 	return false;
6641 }
6642 
6643 /*
6644  * A system can have three types of NUMA topology:
6645  * NUMA_DIRECT: all nodes are directly connected, or not a NUMA system
6646  * NUMA_GLUELESS_MESH: some nodes reachable through intermediary nodes
6647  * NUMA_BACKPLANE: nodes can reach other nodes through a backplane
6648  *
6649  * The difference between a glueless mesh topology and a backplane
6650  * topology lies in whether communication between not directly
6651  * connected nodes goes through intermediary nodes (where programs
6652  * could run), or through backplane controllers. This affects
6653  * placement of programs.
6654  *
6655  * The type of topology can be discerned with the following tests:
6656  * - If the maximum distance between any nodes is 1 hop, the system
6657  *   is directly connected.
6658  * - If for two nodes A and B, located N > 1 hops away from each other,
6659  *   there is an intermediary node C, which is < N hops away from both
6660  *   nodes A and B, the system is a glueless mesh.
6661  */
6662 static void init_numa_topology_type(void)
6663 {
6664 	int a, b, c, n;
6665 
6666 	n = sched_max_numa_distance;
6667 
6668 	if (sched_domains_numa_levels <= 1) {
6669 		sched_numa_topology_type = NUMA_DIRECT;
6670 		return;
6671 	}
6672 
6673 	for_each_online_node(a) {
6674 		for_each_online_node(b) {
6675 			/* Find two nodes furthest removed from each other. */
6676 			if (node_distance(a, b) < n)
6677 				continue;
6678 
6679 			/* Is there an intermediary node between a and b? */
6680 			for_each_online_node(c) {
6681 				if (node_distance(a, c) < n &&
6682 				    node_distance(b, c) < n) {
6683 					sched_numa_topology_type =
6684 							NUMA_GLUELESS_MESH;
6685 					return;
6686 				}
6687 			}
6688 
6689 			sched_numa_topology_type = NUMA_BACKPLANE;
6690 			return;
6691 		}
6692 	}
6693 }
6694 
6695 static void sched_init_numa(void)
6696 {
6697 	int next_distance, curr_distance = node_distance(0, 0);
6698 	struct sched_domain_topology_level *tl;
6699 	int level = 0;
6700 	int i, j, k;
6701 
6702 	sched_domains_numa_distance = kzalloc(sizeof(int) * nr_node_ids, GFP_KERNEL);
6703 	if (!sched_domains_numa_distance)
6704 		return;
6705 
6706 	/*
6707 	 * O(nr_nodes^2) deduplicating selection sort -- in order to find the
6708 	 * unique distances in the node_distance() table.
6709 	 *
6710 	 * Assumes node_distance(0,j) includes all distances in
6711 	 * node_distance(i,j) in order to avoid cubic time.
6712 	 */
6713 	next_distance = curr_distance;
6714 	for (i = 0; i < nr_node_ids; i++) {
6715 		for (j = 0; j < nr_node_ids; j++) {
6716 			for (k = 0; k < nr_node_ids; k++) {
6717 				int distance = node_distance(i, k);
6718 
6719 				if (distance > curr_distance &&
6720 				    (distance < next_distance ||
6721 				     next_distance == curr_distance))
6722 					next_distance = distance;
6723 
6724 				/*
6725 				 * While not a strong assumption it would be nice to know
6726 				 * about cases where if node A is connected to B, B is not
6727 				 * equally connected to A.
6728 				 */
6729 				if (sched_debug() && node_distance(k, i) != distance)
6730 					sched_numa_warn("Node-distance not symmetric");
6731 
6732 				if (sched_debug() && i && !find_numa_distance(distance))
6733 					sched_numa_warn("Node-0 not representative");
6734 			}
6735 			if (next_distance != curr_distance) {
6736 				sched_domains_numa_distance[level++] = next_distance;
6737 				sched_domains_numa_levels = level;
6738 				curr_distance = next_distance;
6739 			} else break;
6740 		}
6741 
6742 		/*
6743 		 * In case of sched_debug() we verify the above assumption.
6744 		 */
6745 		if (!sched_debug())
6746 			break;
6747 	}
6748 
6749 	if (!level)
6750 		return;
6751 
6752 	/*
6753 	 * 'level' contains the number of unique distances, excluding the
6754 	 * identity distance node_distance(i,i).
6755 	 *
6756 	 * The sched_domains_numa_distance[] array includes the actual distance
6757 	 * numbers.
6758 	 */
6759 
6760 	/*
6761 	 * Here, we should temporarily reset sched_domains_numa_levels to 0.
6762 	 * If it fails to allocate memory for array sched_domains_numa_masks[][],
6763 	 * the array will contain less then 'level' members. This could be
6764 	 * dangerous when we use it to iterate array sched_domains_numa_masks[][]
6765 	 * in other functions.
6766 	 *
6767 	 * We reset it to 'level' at the end of this function.
6768 	 */
6769 	sched_domains_numa_levels = 0;
6770 
6771 	sched_domains_numa_masks = kzalloc(sizeof(void *) * level, GFP_KERNEL);
6772 	if (!sched_domains_numa_masks)
6773 		return;
6774 
6775 	/*
6776 	 * Now for each level, construct a mask per node which contains all
6777 	 * cpus of nodes that are that many hops away from us.
6778 	 */
6779 	for (i = 0; i < level; i++) {
6780 		sched_domains_numa_masks[i] =
6781 			kzalloc(nr_node_ids * sizeof(void *), GFP_KERNEL);
6782 		if (!sched_domains_numa_masks[i])
6783 			return;
6784 
6785 		for (j = 0; j < nr_node_ids; j++) {
6786 			struct cpumask *mask = kzalloc(cpumask_size(), GFP_KERNEL);
6787 			if (!mask)
6788 				return;
6789 
6790 			sched_domains_numa_masks[i][j] = mask;
6791 
6792 			for_each_node(k) {
6793 				if (node_distance(j, k) > sched_domains_numa_distance[i])
6794 					continue;
6795 
6796 				cpumask_or(mask, mask, cpumask_of_node(k));
6797 			}
6798 		}
6799 	}
6800 
6801 	/* Compute default topology size */
6802 	for (i = 0; sched_domain_topology[i].mask; i++);
6803 
6804 	tl = kzalloc((i + level + 1) *
6805 			sizeof(struct sched_domain_topology_level), GFP_KERNEL);
6806 	if (!tl)
6807 		return;
6808 
6809 	/*
6810 	 * Copy the default topology bits..
6811 	 */
6812 	for (i = 0; sched_domain_topology[i].mask; i++)
6813 		tl[i] = sched_domain_topology[i];
6814 
6815 	/*
6816 	 * .. and append 'j' levels of NUMA goodness.
6817 	 */
6818 	for (j = 0; j < level; i++, j++) {
6819 		tl[i] = (struct sched_domain_topology_level){
6820 			.mask = sd_numa_mask,
6821 			.sd_flags = cpu_numa_flags,
6822 			.flags = SDTL_OVERLAP,
6823 			.numa_level = j,
6824 			SD_INIT_NAME(NUMA)
6825 		};
6826 	}
6827 
6828 	sched_domain_topology = tl;
6829 
6830 	sched_domains_numa_levels = level;
6831 	sched_max_numa_distance = sched_domains_numa_distance[level - 1];
6832 
6833 	init_numa_topology_type();
6834 }
6835 
6836 static void sched_domains_numa_masks_set(unsigned int cpu)
6837 {
6838 	int node = cpu_to_node(cpu);
6839 	int i, j;
6840 
6841 	for (i = 0; i < sched_domains_numa_levels; i++) {
6842 		for (j = 0; j < nr_node_ids; j++) {
6843 			if (node_distance(j, node) <= sched_domains_numa_distance[i])
6844 				cpumask_set_cpu(cpu, sched_domains_numa_masks[i][j]);
6845 		}
6846 	}
6847 }
6848 
6849 static void sched_domains_numa_masks_clear(unsigned int cpu)
6850 {
6851 	int i, j;
6852 
6853 	for (i = 0; i < sched_domains_numa_levels; i++) {
6854 		for (j = 0; j < nr_node_ids; j++)
6855 			cpumask_clear_cpu(cpu, sched_domains_numa_masks[i][j]);
6856 	}
6857 }
6858 
6859 #else
6860 static inline void sched_init_numa(void) { }
6861 static void sched_domains_numa_masks_set(unsigned int cpu) { }
6862 static void sched_domains_numa_masks_clear(unsigned int cpu) { }
6863 #endif /* CONFIG_NUMA */
6864 
6865 static int __sdt_alloc(const struct cpumask *cpu_map)
6866 {
6867 	struct sched_domain_topology_level *tl;
6868 	int j;
6869 
6870 	for_each_sd_topology(tl) {
6871 		struct sd_data *sdd = &tl->data;
6872 
6873 		sdd->sd = alloc_percpu(struct sched_domain *);
6874 		if (!sdd->sd)
6875 			return -ENOMEM;
6876 
6877 		sdd->sds = alloc_percpu(struct sched_domain_shared *);
6878 		if (!sdd->sds)
6879 			return -ENOMEM;
6880 
6881 		sdd->sg = alloc_percpu(struct sched_group *);
6882 		if (!sdd->sg)
6883 			return -ENOMEM;
6884 
6885 		sdd->sgc = alloc_percpu(struct sched_group_capacity *);
6886 		if (!sdd->sgc)
6887 			return -ENOMEM;
6888 
6889 		for_each_cpu(j, cpu_map) {
6890 			struct sched_domain *sd;
6891 			struct sched_domain_shared *sds;
6892 			struct sched_group *sg;
6893 			struct sched_group_capacity *sgc;
6894 
6895 			sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(),
6896 					GFP_KERNEL, cpu_to_node(j));
6897 			if (!sd)
6898 				return -ENOMEM;
6899 
6900 			*per_cpu_ptr(sdd->sd, j) = sd;
6901 
6902 			sds = kzalloc_node(sizeof(struct sched_domain_shared),
6903 					GFP_KERNEL, cpu_to_node(j));
6904 			if (!sds)
6905 				return -ENOMEM;
6906 
6907 			*per_cpu_ptr(sdd->sds, j) = sds;
6908 
6909 			sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
6910 					GFP_KERNEL, cpu_to_node(j));
6911 			if (!sg)
6912 				return -ENOMEM;
6913 
6914 			sg->next = sg;
6915 
6916 			*per_cpu_ptr(sdd->sg, j) = sg;
6917 
6918 			sgc = kzalloc_node(sizeof(struct sched_group_capacity) + cpumask_size(),
6919 					GFP_KERNEL, cpu_to_node(j));
6920 			if (!sgc)
6921 				return -ENOMEM;
6922 
6923 			*per_cpu_ptr(sdd->sgc, j) = sgc;
6924 		}
6925 	}
6926 
6927 	return 0;
6928 }
6929 
6930 static void __sdt_free(const struct cpumask *cpu_map)
6931 {
6932 	struct sched_domain_topology_level *tl;
6933 	int j;
6934 
6935 	for_each_sd_topology(tl) {
6936 		struct sd_data *sdd = &tl->data;
6937 
6938 		for_each_cpu(j, cpu_map) {
6939 			struct sched_domain *sd;
6940 
6941 			if (sdd->sd) {
6942 				sd = *per_cpu_ptr(sdd->sd, j);
6943 				if (sd && (sd->flags & SD_OVERLAP))
6944 					free_sched_groups(sd->groups, 0);
6945 				kfree(*per_cpu_ptr(sdd->sd, j));
6946 			}
6947 
6948 			if (sdd->sds)
6949 				kfree(*per_cpu_ptr(sdd->sds, j));
6950 			if (sdd->sg)
6951 				kfree(*per_cpu_ptr(sdd->sg, j));
6952 			if (sdd->sgc)
6953 				kfree(*per_cpu_ptr(sdd->sgc, j));
6954 		}
6955 		free_percpu(sdd->sd);
6956 		sdd->sd = NULL;
6957 		free_percpu(sdd->sds);
6958 		sdd->sds = NULL;
6959 		free_percpu(sdd->sg);
6960 		sdd->sg = NULL;
6961 		free_percpu(sdd->sgc);
6962 		sdd->sgc = NULL;
6963 	}
6964 }
6965 
6966 struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl,
6967 		const struct cpumask *cpu_map, struct sched_domain_attr *attr,
6968 		struct sched_domain *child, int cpu)
6969 {
6970 	struct sched_domain *sd = sd_init(tl, cpu_map, child, cpu);
6971 
6972 	if (child) {
6973 		sd->level = child->level + 1;
6974 		sched_domain_level_max = max(sched_domain_level_max, sd->level);
6975 		child->parent = sd;
6976 
6977 		if (!cpumask_subset(sched_domain_span(child),
6978 				    sched_domain_span(sd))) {
6979 			pr_err("BUG: arch topology borken\n");
6980 #ifdef CONFIG_SCHED_DEBUG
6981 			pr_err("     the %s domain not a subset of the %s domain\n",
6982 					child->name, sd->name);
6983 #endif
6984 			/* Fixup, ensure @sd has at least @child cpus. */
6985 			cpumask_or(sched_domain_span(sd),
6986 				   sched_domain_span(sd),
6987 				   sched_domain_span(child));
6988 		}
6989 
6990 	}
6991 	set_domain_attribute(sd, attr);
6992 
6993 	return sd;
6994 }
6995 
6996 /*
6997  * Build sched domains for a given set of cpus and attach the sched domains
6998  * to the individual cpus
6999  */
7000 static int build_sched_domains(const struct cpumask *cpu_map,
7001 			       struct sched_domain_attr *attr)
7002 {
7003 	enum s_alloc alloc_state;
7004 	struct sched_domain *sd;
7005 	struct s_data d;
7006 	struct rq *rq = NULL;
7007 	int i, ret = -ENOMEM;
7008 
7009 	alloc_state = __visit_domain_allocation_hell(&d, cpu_map);
7010 	if (alloc_state != sa_rootdomain)
7011 		goto error;
7012 
7013 	/* Set up domains for cpus specified by the cpu_map. */
7014 	for_each_cpu(i, cpu_map) {
7015 		struct sched_domain_topology_level *tl;
7016 
7017 		sd = NULL;
7018 		for_each_sd_topology(tl) {
7019 			sd = build_sched_domain(tl, cpu_map, attr, sd, i);
7020 			if (tl == sched_domain_topology)
7021 				*per_cpu_ptr(d.sd, i) = sd;
7022 			if (tl->flags & SDTL_OVERLAP || sched_feat(FORCE_SD_OVERLAP))
7023 				sd->flags |= SD_OVERLAP;
7024 			if (cpumask_equal(cpu_map, sched_domain_span(sd)))
7025 				break;
7026 		}
7027 	}
7028 
7029 	/* Build the groups for the domains */
7030 	for_each_cpu(i, cpu_map) {
7031 		for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
7032 			sd->span_weight = cpumask_weight(sched_domain_span(sd));
7033 			if (sd->flags & SD_OVERLAP) {
7034 				if (build_overlap_sched_groups(sd, i))
7035 					goto error;
7036 			} else {
7037 				if (build_sched_groups(sd, i))
7038 					goto error;
7039 			}
7040 		}
7041 	}
7042 
7043 	/* Calculate CPU capacity for physical packages and nodes */
7044 	for (i = nr_cpumask_bits-1; i >= 0; i--) {
7045 		if (!cpumask_test_cpu(i, cpu_map))
7046 			continue;
7047 
7048 		for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
7049 			claim_allocations(i, sd);
7050 			init_sched_groups_capacity(i, sd);
7051 		}
7052 	}
7053 
7054 	/* Attach the domains */
7055 	rcu_read_lock();
7056 	for_each_cpu(i, cpu_map) {
7057 		rq = cpu_rq(i);
7058 		sd = *per_cpu_ptr(d.sd, i);
7059 
7060 		/* Use READ_ONCE()/WRITE_ONCE() to avoid load/store tearing: */
7061 		if (rq->cpu_capacity_orig > READ_ONCE(d.rd->max_cpu_capacity))
7062 			WRITE_ONCE(d.rd->max_cpu_capacity, rq->cpu_capacity_orig);
7063 
7064 		cpu_attach_domain(sd, d.rd, i);
7065 	}
7066 	rcu_read_unlock();
7067 
7068 	if (rq && sched_debug_enabled) {
7069 		pr_info("span: %*pbl (max cpu_capacity = %lu)\n",
7070 			cpumask_pr_args(cpu_map), rq->rd->max_cpu_capacity);
7071 	}
7072 
7073 	ret = 0;
7074 error:
7075 	__free_domain_allocs(&d, alloc_state, cpu_map);
7076 	return ret;
7077 }
7078 
7079 static cpumask_var_t *doms_cur;	/* current sched domains */
7080 static int ndoms_cur;		/* number of sched domains in 'doms_cur' */
7081 static struct sched_domain_attr *dattr_cur;
7082 				/* attribues of custom domains in 'doms_cur' */
7083 
7084 /*
7085  * Special case: If a kmalloc of a doms_cur partition (array of
7086  * cpumask) fails, then fallback to a single sched domain,
7087  * as determined by the single cpumask fallback_doms.
7088  */
7089 static cpumask_var_t fallback_doms;
7090 
7091 /*
7092  * arch_update_cpu_topology lets virtualized architectures update the
7093  * cpu core maps. It is supposed to return 1 if the topology changed
7094  * or 0 if it stayed the same.
7095  */
7096 int __weak arch_update_cpu_topology(void)
7097 {
7098 	return 0;
7099 }
7100 
7101 cpumask_var_t *alloc_sched_domains(unsigned int ndoms)
7102 {
7103 	int i;
7104 	cpumask_var_t *doms;
7105 
7106 	doms = kmalloc(sizeof(*doms) * ndoms, GFP_KERNEL);
7107 	if (!doms)
7108 		return NULL;
7109 	for (i = 0; i < ndoms; i++) {
7110 		if (!alloc_cpumask_var(&doms[i], GFP_KERNEL)) {
7111 			free_sched_domains(doms, i);
7112 			return NULL;
7113 		}
7114 	}
7115 	return doms;
7116 }
7117 
7118 void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms)
7119 {
7120 	unsigned int i;
7121 	for (i = 0; i < ndoms; i++)
7122 		free_cpumask_var(doms[i]);
7123 	kfree(doms);
7124 }
7125 
7126 /*
7127  * Set up scheduler domains and groups. Callers must hold the hotplug lock.
7128  * For now this just excludes isolated cpus, but could be used to
7129  * exclude other special cases in the future.
7130  */
7131 static int init_sched_domains(const struct cpumask *cpu_map)
7132 {
7133 	int err;
7134 
7135 	arch_update_cpu_topology();
7136 	ndoms_cur = 1;
7137 	doms_cur = alloc_sched_domains(ndoms_cur);
7138 	if (!doms_cur)
7139 		doms_cur = &fallback_doms;
7140 	cpumask_andnot(doms_cur[0], cpu_map, cpu_isolated_map);
7141 	err = build_sched_domains(doms_cur[0], NULL);
7142 	register_sched_domain_sysctl();
7143 
7144 	return err;
7145 }
7146 
7147 /*
7148  * Detach sched domains from a group of cpus specified in cpu_map
7149  * These cpus will now be attached to the NULL domain
7150  */
7151 static void detach_destroy_domains(const struct cpumask *cpu_map)
7152 {
7153 	int i;
7154 
7155 	rcu_read_lock();
7156 	for_each_cpu(i, cpu_map)
7157 		cpu_attach_domain(NULL, &def_root_domain, i);
7158 	rcu_read_unlock();
7159 }
7160 
7161 /* handle null as "default" */
7162 static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
7163 			struct sched_domain_attr *new, int idx_new)
7164 {
7165 	struct sched_domain_attr tmp;
7166 
7167 	/* fast path */
7168 	if (!new && !cur)
7169 		return 1;
7170 
7171 	tmp = SD_ATTR_INIT;
7172 	return !memcmp(cur ? (cur + idx_cur) : &tmp,
7173 			new ? (new + idx_new) : &tmp,
7174 			sizeof(struct sched_domain_attr));
7175 }
7176 
7177 /*
7178  * Partition sched domains as specified by the 'ndoms_new'
7179  * cpumasks in the array doms_new[] of cpumasks. This compares
7180  * doms_new[] to the current sched domain partitioning, doms_cur[].
7181  * It destroys each deleted domain and builds each new domain.
7182  *
7183  * 'doms_new' is an array of cpumask_var_t's of length 'ndoms_new'.
7184  * The masks don't intersect (don't overlap.) We should setup one
7185  * sched domain for each mask. CPUs not in any of the cpumasks will
7186  * not be load balanced. If the same cpumask appears both in the
7187  * current 'doms_cur' domains and in the new 'doms_new', we can leave
7188  * it as it is.
7189  *
7190  * The passed in 'doms_new' should be allocated using
7191  * alloc_sched_domains.  This routine takes ownership of it and will
7192  * free_sched_domains it when done with it. If the caller failed the
7193  * alloc call, then it can pass in doms_new == NULL && ndoms_new == 1,
7194  * and partition_sched_domains() will fallback to the single partition
7195  * 'fallback_doms', it also forces the domains to be rebuilt.
7196  *
7197  * If doms_new == NULL it will be replaced with cpu_online_mask.
7198  * ndoms_new == 0 is a special case for destroying existing domains,
7199  * and it will not create the default domain.
7200  *
7201  * Call with hotplug lock held
7202  */
7203 void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
7204 			     struct sched_domain_attr *dattr_new)
7205 {
7206 	int i, j, n;
7207 	int new_topology;
7208 
7209 	mutex_lock(&sched_domains_mutex);
7210 
7211 	/* always unregister in case we don't destroy any domains */
7212 	unregister_sched_domain_sysctl();
7213 
7214 	/* Let architecture update cpu core mappings. */
7215 	new_topology = arch_update_cpu_topology();
7216 
7217 	n = doms_new ? ndoms_new : 0;
7218 
7219 	/* Destroy deleted domains */
7220 	for (i = 0; i < ndoms_cur; i++) {
7221 		for (j = 0; j < n && !new_topology; j++) {
7222 			if (cpumask_equal(doms_cur[i], doms_new[j])
7223 			    && dattrs_equal(dattr_cur, i, dattr_new, j))
7224 				goto match1;
7225 		}
7226 		/* no match - a current sched domain not in new doms_new[] */
7227 		detach_destroy_domains(doms_cur[i]);
7228 match1:
7229 		;
7230 	}
7231 
7232 	n = ndoms_cur;
7233 	if (doms_new == NULL) {
7234 		n = 0;
7235 		doms_new = &fallback_doms;
7236 		cpumask_andnot(doms_new[0], cpu_active_mask, cpu_isolated_map);
7237 		WARN_ON_ONCE(dattr_new);
7238 	}
7239 
7240 	/* Build new domains */
7241 	for (i = 0; i < ndoms_new; i++) {
7242 		for (j = 0; j < n && !new_topology; j++) {
7243 			if (cpumask_equal(doms_new[i], doms_cur[j])
7244 			    && dattrs_equal(dattr_new, i, dattr_cur, j))
7245 				goto match2;
7246 		}
7247 		/* no match - add a new doms_new */
7248 		build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL);
7249 match2:
7250 		;
7251 	}
7252 
7253 	/* Remember the new sched domains */
7254 	if (doms_cur != &fallback_doms)
7255 		free_sched_domains(doms_cur, ndoms_cur);
7256 	kfree(dattr_cur);	/* kfree(NULL) is safe */
7257 	doms_cur = doms_new;
7258 	dattr_cur = dattr_new;
7259 	ndoms_cur = ndoms_new;
7260 
7261 	register_sched_domain_sysctl();
7262 
7263 	mutex_unlock(&sched_domains_mutex);
7264 }
7265 
7266 static int num_cpus_frozen;	/* used to mark begin/end of suspend/resume */
7267 
7268 /*
7269  * Update cpusets according to cpu_active mask.  If cpusets are
7270  * disabled, cpuset_update_active_cpus() becomes a simple wrapper
7271  * around partition_sched_domains().
7272  *
7273  * If we come here as part of a suspend/resume, don't touch cpusets because we
7274  * want to restore it back to its original state upon resume anyway.
7275  */
7276 static void cpuset_cpu_active(void)
7277 {
7278 	if (cpuhp_tasks_frozen) {
7279 		/*
7280 		 * num_cpus_frozen tracks how many CPUs are involved in suspend
7281 		 * resume sequence. As long as this is not the last online
7282 		 * operation in the resume sequence, just build a single sched
7283 		 * domain, ignoring cpusets.
7284 		 */
7285 		num_cpus_frozen--;
7286 		if (likely(num_cpus_frozen)) {
7287 			partition_sched_domains(1, NULL, NULL);
7288 			return;
7289 		}
7290 		/*
7291 		 * This is the last CPU online operation. So fall through and
7292 		 * restore the original sched domains by considering the
7293 		 * cpuset configurations.
7294 		 */
7295 	}
7296 	cpuset_update_active_cpus(true);
7297 }
7298 
7299 static int cpuset_cpu_inactive(unsigned int cpu)
7300 {
7301 	unsigned long flags;
7302 	struct dl_bw *dl_b;
7303 	bool overflow;
7304 	int cpus;
7305 
7306 	if (!cpuhp_tasks_frozen) {
7307 		rcu_read_lock_sched();
7308 		dl_b = dl_bw_of(cpu);
7309 
7310 		raw_spin_lock_irqsave(&dl_b->lock, flags);
7311 		cpus = dl_bw_cpus(cpu);
7312 		overflow = __dl_overflow(dl_b, cpus, 0, 0);
7313 		raw_spin_unlock_irqrestore(&dl_b->lock, flags);
7314 
7315 		rcu_read_unlock_sched();
7316 
7317 		if (overflow)
7318 			return -EBUSY;
7319 		cpuset_update_active_cpus(false);
7320 	} else {
7321 		num_cpus_frozen++;
7322 		partition_sched_domains(1, NULL, NULL);
7323 	}
7324 	return 0;
7325 }
7326 
7327 int sched_cpu_activate(unsigned int cpu)
7328 {
7329 	struct rq *rq = cpu_rq(cpu);
7330 	unsigned long flags;
7331 
7332 	set_cpu_active(cpu, true);
7333 
7334 	if (sched_smp_initialized) {
7335 		sched_domains_numa_masks_set(cpu);
7336 		cpuset_cpu_active();
7337 	}
7338 
7339 	/*
7340 	 * Put the rq online, if not already. This happens:
7341 	 *
7342 	 * 1) In the early boot process, because we build the real domains
7343 	 *    after all cpus have been brought up.
7344 	 *
7345 	 * 2) At runtime, if cpuset_cpu_active() fails to rebuild the
7346 	 *    domains.
7347 	 */
7348 	raw_spin_lock_irqsave(&rq->lock, flags);
7349 	if (rq->rd) {
7350 		BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
7351 		set_rq_online(rq);
7352 	}
7353 	raw_spin_unlock_irqrestore(&rq->lock, flags);
7354 
7355 	update_max_interval();
7356 
7357 	return 0;
7358 }
7359 
7360 int sched_cpu_deactivate(unsigned int cpu)
7361 {
7362 	int ret;
7363 
7364 	set_cpu_active(cpu, false);
7365 	/*
7366 	 * We've cleared cpu_active_mask, wait for all preempt-disabled and RCU
7367 	 * users of this state to go away such that all new such users will
7368 	 * observe it.
7369 	 *
7370 	 * For CONFIG_PREEMPT we have preemptible RCU and its sync_rcu() might
7371 	 * not imply sync_sched(), so wait for both.
7372 	 *
7373 	 * Do sync before park smpboot threads to take care the rcu boost case.
7374 	 */
7375 	if (IS_ENABLED(CONFIG_PREEMPT))
7376 		synchronize_rcu_mult(call_rcu, call_rcu_sched);
7377 	else
7378 		synchronize_rcu();
7379 
7380 	if (!sched_smp_initialized)
7381 		return 0;
7382 
7383 	ret = cpuset_cpu_inactive(cpu);
7384 	if (ret) {
7385 		set_cpu_active(cpu, true);
7386 		return ret;
7387 	}
7388 	sched_domains_numa_masks_clear(cpu);
7389 	return 0;
7390 }
7391 
7392 static void sched_rq_cpu_starting(unsigned int cpu)
7393 {
7394 	struct rq *rq = cpu_rq(cpu);
7395 
7396 	rq->calc_load_update = calc_load_update;
7397 	update_max_interval();
7398 }
7399 
7400 int sched_cpu_starting(unsigned int cpu)
7401 {
7402 	set_cpu_rq_start_time(cpu);
7403 	sched_rq_cpu_starting(cpu);
7404 	return 0;
7405 }
7406 
7407 #ifdef CONFIG_HOTPLUG_CPU
7408 int sched_cpu_dying(unsigned int cpu)
7409 {
7410 	struct rq *rq = cpu_rq(cpu);
7411 	unsigned long flags;
7412 
7413 	/* Handle pending wakeups and then migrate everything off */
7414 	sched_ttwu_pending();
7415 	raw_spin_lock_irqsave(&rq->lock, flags);
7416 	if (rq->rd) {
7417 		BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
7418 		set_rq_offline(rq);
7419 	}
7420 	migrate_tasks(rq);
7421 	BUG_ON(rq->nr_running != 1);
7422 	raw_spin_unlock_irqrestore(&rq->lock, flags);
7423 	calc_load_migrate(rq);
7424 	update_max_interval();
7425 	nohz_balance_exit_idle(cpu);
7426 	hrtick_clear(rq);
7427 	return 0;
7428 }
7429 #endif
7430 
7431 #ifdef CONFIG_SCHED_SMT
7432 DEFINE_STATIC_KEY_FALSE(sched_smt_present);
7433 
7434 static void sched_init_smt(void)
7435 {
7436 	/*
7437 	 * We've enumerated all CPUs and will assume that if any CPU
7438 	 * has SMT siblings, CPU0 will too.
7439 	 */
7440 	if (cpumask_weight(cpu_smt_mask(0)) > 1)
7441 		static_branch_enable(&sched_smt_present);
7442 }
7443 #else
7444 static inline void sched_init_smt(void) { }
7445 #endif
7446 
7447 void __init sched_init_smp(void)
7448 {
7449 	cpumask_var_t non_isolated_cpus;
7450 
7451 	alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
7452 	alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
7453 
7454 	sched_init_numa();
7455 
7456 	/*
7457 	 * There's no userspace yet to cause hotplug operations; hence all the
7458 	 * cpu masks are stable and all blatant races in the below code cannot
7459 	 * happen.
7460 	 */
7461 	mutex_lock(&sched_domains_mutex);
7462 	init_sched_domains(cpu_active_mask);
7463 	cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
7464 	if (cpumask_empty(non_isolated_cpus))
7465 		cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
7466 	mutex_unlock(&sched_domains_mutex);
7467 
7468 	/* Move init over to a non-isolated CPU */
7469 	if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
7470 		BUG();
7471 	sched_init_granularity();
7472 	free_cpumask_var(non_isolated_cpus);
7473 
7474 	init_sched_rt_class();
7475 	init_sched_dl_class();
7476 
7477 	sched_init_smt();
7478 
7479 	sched_smp_initialized = true;
7480 }
7481 
7482 static int __init migration_init(void)
7483 {
7484 	sched_rq_cpu_starting(smp_processor_id());
7485 	return 0;
7486 }
7487 early_initcall(migration_init);
7488 
7489 #else
7490 void __init sched_init_smp(void)
7491 {
7492 	sched_init_granularity();
7493 }
7494 #endif /* CONFIG_SMP */
7495 
7496 int in_sched_functions(unsigned long addr)
7497 {
7498 	return in_lock_functions(addr) ||
7499 		(addr >= (unsigned long)__sched_text_start
7500 		&& addr < (unsigned long)__sched_text_end);
7501 }
7502 
7503 #ifdef CONFIG_CGROUP_SCHED
7504 /*
7505  * Default task group.
7506  * Every task in system belongs to this group at bootup.
7507  */
7508 struct task_group root_task_group;
7509 LIST_HEAD(task_groups);
7510 
7511 /* Cacheline aligned slab cache for task_group */
7512 static struct kmem_cache *task_group_cache __read_mostly;
7513 #endif
7514 
7515 DECLARE_PER_CPU(cpumask_var_t, load_balance_mask);
7516 DECLARE_PER_CPU(cpumask_var_t, select_idle_mask);
7517 
7518 void __init sched_init(void)
7519 {
7520 	int i, j;
7521 	unsigned long alloc_size = 0, ptr;
7522 
7523 #ifdef CONFIG_FAIR_GROUP_SCHED
7524 	alloc_size += 2 * nr_cpu_ids * sizeof(void **);
7525 #endif
7526 #ifdef CONFIG_RT_GROUP_SCHED
7527 	alloc_size += 2 * nr_cpu_ids * sizeof(void **);
7528 #endif
7529 	if (alloc_size) {
7530 		ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT);
7531 
7532 #ifdef CONFIG_FAIR_GROUP_SCHED
7533 		root_task_group.se = (struct sched_entity **)ptr;
7534 		ptr += nr_cpu_ids * sizeof(void **);
7535 
7536 		root_task_group.cfs_rq = (struct cfs_rq **)ptr;
7537 		ptr += nr_cpu_ids * sizeof(void **);
7538 
7539 #endif /* CONFIG_FAIR_GROUP_SCHED */
7540 #ifdef CONFIG_RT_GROUP_SCHED
7541 		root_task_group.rt_se = (struct sched_rt_entity **)ptr;
7542 		ptr += nr_cpu_ids * sizeof(void **);
7543 
7544 		root_task_group.rt_rq = (struct rt_rq **)ptr;
7545 		ptr += nr_cpu_ids * sizeof(void **);
7546 
7547 #endif /* CONFIG_RT_GROUP_SCHED */
7548 	}
7549 #ifdef CONFIG_CPUMASK_OFFSTACK
7550 	for_each_possible_cpu(i) {
7551 		per_cpu(load_balance_mask, i) = (cpumask_var_t)kzalloc_node(
7552 			cpumask_size(), GFP_KERNEL, cpu_to_node(i));
7553 		per_cpu(select_idle_mask, i) = (cpumask_var_t)kzalloc_node(
7554 			cpumask_size(), GFP_KERNEL, cpu_to_node(i));
7555 	}
7556 #endif /* CONFIG_CPUMASK_OFFSTACK */
7557 
7558 	init_rt_bandwidth(&def_rt_bandwidth,
7559 			global_rt_period(), global_rt_runtime());
7560 	init_dl_bandwidth(&def_dl_bandwidth,
7561 			global_rt_period(), global_rt_runtime());
7562 
7563 #ifdef CONFIG_SMP
7564 	init_defrootdomain();
7565 #endif
7566 
7567 #ifdef CONFIG_RT_GROUP_SCHED
7568 	init_rt_bandwidth(&root_task_group.rt_bandwidth,
7569 			global_rt_period(), global_rt_runtime());
7570 #endif /* CONFIG_RT_GROUP_SCHED */
7571 
7572 #ifdef CONFIG_CGROUP_SCHED
7573 	task_group_cache = KMEM_CACHE(task_group, 0);
7574 
7575 	list_add(&root_task_group.list, &task_groups);
7576 	INIT_LIST_HEAD(&root_task_group.children);
7577 	INIT_LIST_HEAD(&root_task_group.siblings);
7578 	autogroup_init(&init_task);
7579 #endif /* CONFIG_CGROUP_SCHED */
7580 
7581 	for_each_possible_cpu(i) {
7582 		struct rq *rq;
7583 
7584 		rq = cpu_rq(i);
7585 		raw_spin_lock_init(&rq->lock);
7586 		rq->nr_running = 0;
7587 		rq->calc_load_active = 0;
7588 		rq->calc_load_update = jiffies + LOAD_FREQ;
7589 		init_cfs_rq(&rq->cfs);
7590 		init_rt_rq(&rq->rt);
7591 		init_dl_rq(&rq->dl);
7592 #ifdef CONFIG_FAIR_GROUP_SCHED
7593 		root_task_group.shares = ROOT_TASK_GROUP_LOAD;
7594 		INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
7595 		/*
7596 		 * How much cpu bandwidth does root_task_group get?
7597 		 *
7598 		 * In case of task-groups formed thr' the cgroup filesystem, it
7599 		 * gets 100% of the cpu resources in the system. This overall
7600 		 * system cpu resource is divided among the tasks of
7601 		 * root_task_group and its child task-groups in a fair manner,
7602 		 * based on each entity's (task or task-group's) weight
7603 		 * (se->load.weight).
7604 		 *
7605 		 * In other words, if root_task_group has 10 tasks of weight
7606 		 * 1024) and two child groups A0 and A1 (of weight 1024 each),
7607 		 * then A0's share of the cpu resource is:
7608 		 *
7609 		 *	A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33%
7610 		 *
7611 		 * We achieve this by letting root_task_group's tasks sit
7612 		 * directly in rq->cfs (i.e root_task_group->se[] = NULL).
7613 		 */
7614 		init_cfs_bandwidth(&root_task_group.cfs_bandwidth);
7615 		init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL);
7616 #endif /* CONFIG_FAIR_GROUP_SCHED */
7617 
7618 		rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime;
7619 #ifdef CONFIG_RT_GROUP_SCHED
7620 		init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL);
7621 #endif
7622 
7623 		for (j = 0; j < CPU_LOAD_IDX_MAX; j++)
7624 			rq->cpu_load[j] = 0;
7625 
7626 #ifdef CONFIG_SMP
7627 		rq->sd = NULL;
7628 		rq->rd = NULL;
7629 		rq->cpu_capacity = rq->cpu_capacity_orig = SCHED_CAPACITY_SCALE;
7630 		rq->balance_callback = NULL;
7631 		rq->active_balance = 0;
7632 		rq->next_balance = jiffies;
7633 		rq->push_cpu = 0;
7634 		rq->cpu = i;
7635 		rq->online = 0;
7636 		rq->idle_stamp = 0;
7637 		rq->avg_idle = 2*sysctl_sched_migration_cost;
7638 		rq->max_idle_balance_cost = sysctl_sched_migration_cost;
7639 
7640 		INIT_LIST_HEAD(&rq->cfs_tasks);
7641 
7642 		rq_attach_root(rq, &def_root_domain);
7643 #ifdef CONFIG_NO_HZ_COMMON
7644 		rq->last_load_update_tick = jiffies;
7645 		rq->nohz_flags = 0;
7646 #endif
7647 #ifdef CONFIG_NO_HZ_FULL
7648 		rq->last_sched_tick = 0;
7649 #endif
7650 #endif /* CONFIG_SMP */
7651 		init_rq_hrtick(rq);
7652 		atomic_set(&rq->nr_iowait, 0);
7653 	}
7654 
7655 	set_load_weight(&init_task);
7656 
7657 	/*
7658 	 * The boot idle thread does lazy MMU switching as well:
7659 	 */
7660 	atomic_inc(&init_mm.mm_count);
7661 	enter_lazy_tlb(&init_mm, current);
7662 
7663 	/*
7664 	 * Make us the idle thread. Technically, schedule() should not be
7665 	 * called from this thread, however somewhere below it might be,
7666 	 * but because we are the idle thread, we just pick up running again
7667 	 * when this runqueue becomes "idle".
7668 	 */
7669 	init_idle(current, smp_processor_id());
7670 
7671 	calc_load_update = jiffies + LOAD_FREQ;
7672 
7673 #ifdef CONFIG_SMP
7674 	zalloc_cpumask_var(&sched_domains_tmpmask, GFP_NOWAIT);
7675 	/* May be allocated at isolcpus cmdline parse time */
7676 	if (cpu_isolated_map == NULL)
7677 		zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT);
7678 	idle_thread_set_boot_cpu();
7679 	set_cpu_rq_start_time(smp_processor_id());
7680 #endif
7681 	init_sched_fair_class();
7682 
7683 	init_schedstats();
7684 
7685 	scheduler_running = 1;
7686 }
7687 
7688 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
7689 static inline int preempt_count_equals(int preempt_offset)
7690 {
7691 	int nested = preempt_count() + rcu_preempt_depth();
7692 
7693 	return (nested == preempt_offset);
7694 }
7695 
7696 void __might_sleep(const char *file, int line, int preempt_offset)
7697 {
7698 	/*
7699 	 * Blocking primitives will set (and therefore destroy) current->state,
7700 	 * since we will exit with TASK_RUNNING make sure we enter with it,
7701 	 * otherwise we will destroy state.
7702 	 */
7703 	WARN_ONCE(current->state != TASK_RUNNING && current->task_state_change,
7704 			"do not call blocking ops when !TASK_RUNNING; "
7705 			"state=%lx set at [<%p>] %pS\n",
7706 			current->state,
7707 			(void *)current->task_state_change,
7708 			(void *)current->task_state_change);
7709 
7710 	___might_sleep(file, line, preempt_offset);
7711 }
7712 EXPORT_SYMBOL(__might_sleep);
7713 
7714 void ___might_sleep(const char *file, int line, int preempt_offset)
7715 {
7716 	static unsigned long prev_jiffy;	/* ratelimiting */
7717 	unsigned long preempt_disable_ip;
7718 
7719 	rcu_sleep_check(); /* WARN_ON_ONCE() by default, no rate limit reqd. */
7720 	if ((preempt_count_equals(preempt_offset) && !irqs_disabled() &&
7721 	     !is_idle_task(current)) ||
7722 	    system_state != SYSTEM_RUNNING || oops_in_progress)
7723 		return;
7724 	if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
7725 		return;
7726 	prev_jiffy = jiffies;
7727 
7728 	/* Save this before calling printk(), since that will clobber it */
7729 	preempt_disable_ip = get_preempt_disable_ip(current);
7730 
7731 	printk(KERN_ERR
7732 		"BUG: sleeping function called from invalid context at %s:%d\n",
7733 			file, line);
7734 	printk(KERN_ERR
7735 		"in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
7736 			in_atomic(), irqs_disabled(),
7737 			current->pid, current->comm);
7738 
7739 	if (task_stack_end_corrupted(current))
7740 		printk(KERN_EMERG "Thread overran stack, or stack corrupted\n");
7741 
7742 	debug_show_held_locks(current);
7743 	if (irqs_disabled())
7744 		print_irqtrace_events(current);
7745 	if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)
7746 	    && !preempt_count_equals(preempt_offset)) {
7747 		pr_err("Preemption disabled at:");
7748 		print_ip_sym(preempt_disable_ip);
7749 		pr_cont("\n");
7750 	}
7751 	dump_stack();
7752 	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
7753 }
7754 EXPORT_SYMBOL(___might_sleep);
7755 #endif
7756 
7757 #ifdef CONFIG_MAGIC_SYSRQ
7758 void normalize_rt_tasks(void)
7759 {
7760 	struct task_struct *g, *p;
7761 	struct sched_attr attr = {
7762 		.sched_policy = SCHED_NORMAL,
7763 	};
7764 
7765 	read_lock(&tasklist_lock);
7766 	for_each_process_thread(g, p) {
7767 		/*
7768 		 * Only normalize user tasks:
7769 		 */
7770 		if (p->flags & PF_KTHREAD)
7771 			continue;
7772 
7773 		p->se.exec_start = 0;
7774 		schedstat_set(p->se.statistics.wait_start,  0);
7775 		schedstat_set(p->se.statistics.sleep_start, 0);
7776 		schedstat_set(p->se.statistics.block_start, 0);
7777 
7778 		if (!dl_task(p) && !rt_task(p)) {
7779 			/*
7780 			 * Renice negative nice level userspace
7781 			 * tasks back to 0:
7782 			 */
7783 			if (task_nice(p) < 0)
7784 				set_user_nice(p, 0);
7785 			continue;
7786 		}
7787 
7788 		__sched_setscheduler(p, &attr, false, false);
7789 	}
7790 	read_unlock(&tasklist_lock);
7791 }
7792 
7793 #endif /* CONFIG_MAGIC_SYSRQ */
7794 
7795 #if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB)
7796 /*
7797  * These functions are only useful for the IA64 MCA handling, or kdb.
7798  *
7799  * They can only be called when the whole system has been
7800  * stopped - every CPU needs to be quiescent, and no scheduling
7801  * activity can take place. Using them for anything else would
7802  * be a serious bug, and as a result, they aren't even visible
7803  * under any other configuration.
7804  */
7805 
7806 /**
7807  * curr_task - return the current task for a given cpu.
7808  * @cpu: the processor in question.
7809  *
7810  * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
7811  *
7812  * Return: The current task for @cpu.
7813  */
7814 struct task_struct *curr_task(int cpu)
7815 {
7816 	return cpu_curr(cpu);
7817 }
7818 
7819 #endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */
7820 
7821 #ifdef CONFIG_IA64
7822 /**
7823  * set_curr_task - set the current task for a given cpu.
7824  * @cpu: the processor in question.
7825  * @p: the task pointer to set.
7826  *
7827  * Description: This function must only be used when non-maskable interrupts
7828  * are serviced on a separate stack. It allows the architecture to switch the
7829  * notion of the current task on a cpu in a non-blocking manner. This function
7830  * must be called with all CPU's synchronized, and interrupts disabled, the
7831  * and caller must save the original value of the current task (see
7832  * curr_task() above) and restore that value before reenabling interrupts and
7833  * re-starting the system.
7834  *
7835  * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
7836  */
7837 void ia64_set_curr_task(int cpu, struct task_struct *p)
7838 {
7839 	cpu_curr(cpu) = p;
7840 }
7841 
7842 #endif
7843 
7844 #ifdef CONFIG_CGROUP_SCHED
7845 /* task_group_lock serializes the addition/removal of task groups */
7846 static DEFINE_SPINLOCK(task_group_lock);
7847 
7848 static void sched_free_group(struct task_group *tg)
7849 {
7850 	free_fair_sched_group(tg);
7851 	free_rt_sched_group(tg);
7852 	autogroup_free(tg);
7853 	kmem_cache_free(task_group_cache, tg);
7854 }
7855 
7856 /* allocate runqueue etc for a new task group */
7857 struct task_group *sched_create_group(struct task_group *parent)
7858 {
7859 	struct task_group *tg;
7860 
7861 	tg = kmem_cache_alloc(task_group_cache, GFP_KERNEL | __GFP_ZERO);
7862 	if (!tg)
7863 		return ERR_PTR(-ENOMEM);
7864 
7865 	if (!alloc_fair_sched_group(tg, parent))
7866 		goto err;
7867 
7868 	if (!alloc_rt_sched_group(tg, parent))
7869 		goto err;
7870 
7871 	return tg;
7872 
7873 err:
7874 	sched_free_group(tg);
7875 	return ERR_PTR(-ENOMEM);
7876 }
7877 
7878 void sched_online_group(struct task_group *tg, struct task_group *parent)
7879 {
7880 	unsigned long flags;
7881 
7882 	spin_lock_irqsave(&task_group_lock, flags);
7883 	list_add_rcu(&tg->list, &task_groups);
7884 
7885 	WARN_ON(!parent); /* root should already exist */
7886 
7887 	tg->parent = parent;
7888 	INIT_LIST_HEAD(&tg->children);
7889 	list_add_rcu(&tg->siblings, &parent->children);
7890 	spin_unlock_irqrestore(&task_group_lock, flags);
7891 
7892 	online_fair_sched_group(tg);
7893 }
7894 
7895 /* rcu callback to free various structures associated with a task group */
7896 static void sched_free_group_rcu(struct rcu_head *rhp)
7897 {
7898 	/* now it should be safe to free those cfs_rqs */
7899 	sched_free_group(container_of(rhp, struct task_group, rcu));
7900 }
7901 
7902 void sched_destroy_group(struct task_group *tg)
7903 {
7904 	/* wait for possible concurrent references to cfs_rqs complete */
7905 	call_rcu(&tg->rcu, sched_free_group_rcu);
7906 }
7907 
7908 void sched_offline_group(struct task_group *tg)
7909 {
7910 	unsigned long flags;
7911 
7912 	/* end participation in shares distribution */
7913 	unregister_fair_sched_group(tg);
7914 
7915 	spin_lock_irqsave(&task_group_lock, flags);
7916 	list_del_rcu(&tg->list);
7917 	list_del_rcu(&tg->siblings);
7918 	spin_unlock_irqrestore(&task_group_lock, flags);
7919 }
7920 
7921 static void sched_change_group(struct task_struct *tsk, int type)
7922 {
7923 	struct task_group *tg;
7924 
7925 	/*
7926 	 * All callers are synchronized by task_rq_lock(); we do not use RCU
7927 	 * which is pointless here. Thus, we pass "true" to task_css_check()
7928 	 * to prevent lockdep warnings.
7929 	 */
7930 	tg = container_of(task_css_check(tsk, cpu_cgrp_id, true),
7931 			  struct task_group, css);
7932 	tg = autogroup_task_group(tsk, tg);
7933 	tsk->sched_task_group = tg;
7934 
7935 #ifdef CONFIG_FAIR_GROUP_SCHED
7936 	if (tsk->sched_class->task_change_group)
7937 		tsk->sched_class->task_change_group(tsk, type);
7938 	else
7939 #endif
7940 		set_task_rq(tsk, task_cpu(tsk));
7941 }
7942 
7943 /*
7944  * Change task's runqueue when it moves between groups.
7945  *
7946  * The caller of this function should have put the task in its new group by
7947  * now. This function just updates tsk->se.cfs_rq and tsk->se.parent to reflect
7948  * its new group.
7949  */
7950 void sched_move_task(struct task_struct *tsk)
7951 {
7952 	int queued, running;
7953 	struct rq_flags rf;
7954 	struct rq *rq;
7955 
7956 	rq = task_rq_lock(tsk, &rf);
7957 
7958 	running = task_current(rq, tsk);
7959 	queued = task_on_rq_queued(tsk);
7960 
7961 	if (queued)
7962 		dequeue_task(rq, tsk, DEQUEUE_SAVE | DEQUEUE_MOVE);
7963 	if (unlikely(running))
7964 		put_prev_task(rq, tsk);
7965 
7966 	sched_change_group(tsk, TASK_MOVE_GROUP);
7967 
7968 	if (queued)
7969 		enqueue_task(rq, tsk, ENQUEUE_RESTORE | ENQUEUE_MOVE);
7970 	if (unlikely(running))
7971 		set_curr_task(rq, tsk);
7972 
7973 	task_rq_unlock(rq, tsk, &rf);
7974 }
7975 #endif /* CONFIG_CGROUP_SCHED */
7976 
7977 #ifdef CONFIG_RT_GROUP_SCHED
7978 /*
7979  * Ensure that the real time constraints are schedulable.
7980  */
7981 static DEFINE_MUTEX(rt_constraints_mutex);
7982 
7983 /* Must be called with tasklist_lock held */
7984 static inline int tg_has_rt_tasks(struct task_group *tg)
7985 {
7986 	struct task_struct *g, *p;
7987 
7988 	/*
7989 	 * Autogroups do not have RT tasks; see autogroup_create().
7990 	 */
7991 	if (task_group_is_autogroup(tg))
7992 		return 0;
7993 
7994 	for_each_process_thread(g, p) {
7995 		if (rt_task(p) && task_group(p) == tg)
7996 			return 1;
7997 	}
7998 
7999 	return 0;
8000 }
8001 
8002 struct rt_schedulable_data {
8003 	struct task_group *tg;
8004 	u64 rt_period;
8005 	u64 rt_runtime;
8006 };
8007 
8008 static int tg_rt_schedulable(struct task_group *tg, void *data)
8009 {
8010 	struct rt_schedulable_data *d = data;
8011 	struct task_group *child;
8012 	unsigned long total, sum = 0;
8013 	u64 period, runtime;
8014 
8015 	period = ktime_to_ns(tg->rt_bandwidth.rt_period);
8016 	runtime = tg->rt_bandwidth.rt_runtime;
8017 
8018 	if (tg == d->tg) {
8019 		period = d->rt_period;
8020 		runtime = d->rt_runtime;
8021 	}
8022 
8023 	/*
8024 	 * Cannot have more runtime than the period.
8025 	 */
8026 	if (runtime > period && runtime != RUNTIME_INF)
8027 		return -EINVAL;
8028 
8029 	/*
8030 	 * Ensure we don't starve existing RT tasks.
8031 	 */
8032 	if (rt_bandwidth_enabled() && !runtime && tg_has_rt_tasks(tg))
8033 		return -EBUSY;
8034 
8035 	total = to_ratio(period, runtime);
8036 
8037 	/*
8038 	 * Nobody can have more than the global setting allows.
8039 	 */
8040 	if (total > to_ratio(global_rt_period(), global_rt_runtime()))
8041 		return -EINVAL;
8042 
8043 	/*
8044 	 * The sum of our children's runtime should not exceed our own.
8045 	 */
8046 	list_for_each_entry_rcu(child, &tg->children, siblings) {
8047 		period = ktime_to_ns(child->rt_bandwidth.rt_period);
8048 		runtime = child->rt_bandwidth.rt_runtime;
8049 
8050 		if (child == d->tg) {
8051 			period = d->rt_period;
8052 			runtime = d->rt_runtime;
8053 		}
8054 
8055 		sum += to_ratio(period, runtime);
8056 	}
8057 
8058 	if (sum > total)
8059 		return -EINVAL;
8060 
8061 	return 0;
8062 }
8063 
8064 static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
8065 {
8066 	int ret;
8067 
8068 	struct rt_schedulable_data data = {
8069 		.tg = tg,
8070 		.rt_period = period,
8071 		.rt_runtime = runtime,
8072 	};
8073 
8074 	rcu_read_lock();
8075 	ret = walk_tg_tree(tg_rt_schedulable, tg_nop, &data);
8076 	rcu_read_unlock();
8077 
8078 	return ret;
8079 }
8080 
8081 static int tg_set_rt_bandwidth(struct task_group *tg,
8082 		u64 rt_period, u64 rt_runtime)
8083 {
8084 	int i, err = 0;
8085 
8086 	/*
8087 	 * Disallowing the root group RT runtime is BAD, it would disallow the
8088 	 * kernel creating (and or operating) RT threads.
8089 	 */
8090 	if (tg == &root_task_group && rt_runtime == 0)
8091 		return -EINVAL;
8092 
8093 	/* No period doesn't make any sense. */
8094 	if (rt_period == 0)
8095 		return -EINVAL;
8096 
8097 	mutex_lock(&rt_constraints_mutex);
8098 	read_lock(&tasklist_lock);
8099 	err = __rt_schedulable(tg, rt_period, rt_runtime);
8100 	if (err)
8101 		goto unlock;
8102 
8103 	raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
8104 	tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
8105 	tg->rt_bandwidth.rt_runtime = rt_runtime;
8106 
8107 	for_each_possible_cpu(i) {
8108 		struct rt_rq *rt_rq = tg->rt_rq[i];
8109 
8110 		raw_spin_lock(&rt_rq->rt_runtime_lock);
8111 		rt_rq->rt_runtime = rt_runtime;
8112 		raw_spin_unlock(&rt_rq->rt_runtime_lock);
8113 	}
8114 	raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
8115 unlock:
8116 	read_unlock(&tasklist_lock);
8117 	mutex_unlock(&rt_constraints_mutex);
8118 
8119 	return err;
8120 }
8121 
8122 static int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
8123 {
8124 	u64 rt_runtime, rt_period;
8125 
8126 	rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
8127 	rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC;
8128 	if (rt_runtime_us < 0)
8129 		rt_runtime = RUNTIME_INF;
8130 
8131 	return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
8132 }
8133 
8134 static long sched_group_rt_runtime(struct task_group *tg)
8135 {
8136 	u64 rt_runtime_us;
8137 
8138 	if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF)
8139 		return -1;
8140 
8141 	rt_runtime_us = tg->rt_bandwidth.rt_runtime;
8142 	do_div(rt_runtime_us, NSEC_PER_USEC);
8143 	return rt_runtime_us;
8144 }
8145 
8146 static int sched_group_set_rt_period(struct task_group *tg, u64 rt_period_us)
8147 {
8148 	u64 rt_runtime, rt_period;
8149 
8150 	rt_period = rt_period_us * NSEC_PER_USEC;
8151 	rt_runtime = tg->rt_bandwidth.rt_runtime;
8152 
8153 	return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
8154 }
8155 
8156 static long sched_group_rt_period(struct task_group *tg)
8157 {
8158 	u64 rt_period_us;
8159 
8160 	rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period);
8161 	do_div(rt_period_us, NSEC_PER_USEC);
8162 	return rt_period_us;
8163 }
8164 #endif /* CONFIG_RT_GROUP_SCHED */
8165 
8166 #ifdef CONFIG_RT_GROUP_SCHED
8167 static int sched_rt_global_constraints(void)
8168 {
8169 	int ret = 0;
8170 
8171 	mutex_lock(&rt_constraints_mutex);
8172 	read_lock(&tasklist_lock);
8173 	ret = __rt_schedulable(NULL, 0, 0);
8174 	read_unlock(&tasklist_lock);
8175 	mutex_unlock(&rt_constraints_mutex);
8176 
8177 	return ret;
8178 }
8179 
8180 static int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk)
8181 {
8182 	/* Don't accept realtime tasks when there is no way for them to run */
8183 	if (rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0)
8184 		return 0;
8185 
8186 	return 1;
8187 }
8188 
8189 #else /* !CONFIG_RT_GROUP_SCHED */
8190 static int sched_rt_global_constraints(void)
8191 {
8192 	unsigned long flags;
8193 	int i;
8194 
8195 	raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
8196 	for_each_possible_cpu(i) {
8197 		struct rt_rq *rt_rq = &cpu_rq(i)->rt;
8198 
8199 		raw_spin_lock(&rt_rq->rt_runtime_lock);
8200 		rt_rq->rt_runtime = global_rt_runtime();
8201 		raw_spin_unlock(&rt_rq->rt_runtime_lock);
8202 	}
8203 	raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
8204 
8205 	return 0;
8206 }
8207 #endif /* CONFIG_RT_GROUP_SCHED */
8208 
8209 static int sched_dl_global_validate(void)
8210 {
8211 	u64 runtime = global_rt_runtime();
8212 	u64 period = global_rt_period();
8213 	u64 new_bw = to_ratio(period, runtime);
8214 	struct dl_bw *dl_b;
8215 	int cpu, ret = 0;
8216 	unsigned long flags;
8217 
8218 	/*
8219 	 * Here we want to check the bandwidth not being set to some
8220 	 * value smaller than the currently allocated bandwidth in
8221 	 * any of the root_domains.
8222 	 *
8223 	 * FIXME: Cycling on all the CPUs is overdoing, but simpler than
8224 	 * cycling on root_domains... Discussion on different/better
8225 	 * solutions is welcome!
8226 	 */
8227 	for_each_possible_cpu(cpu) {
8228 		rcu_read_lock_sched();
8229 		dl_b = dl_bw_of(cpu);
8230 
8231 		raw_spin_lock_irqsave(&dl_b->lock, flags);
8232 		if (new_bw < dl_b->total_bw)
8233 			ret = -EBUSY;
8234 		raw_spin_unlock_irqrestore(&dl_b->lock, flags);
8235 
8236 		rcu_read_unlock_sched();
8237 
8238 		if (ret)
8239 			break;
8240 	}
8241 
8242 	return ret;
8243 }
8244 
8245 static void sched_dl_do_global(void)
8246 {
8247 	u64 new_bw = -1;
8248 	struct dl_bw *dl_b;
8249 	int cpu;
8250 	unsigned long flags;
8251 
8252 	def_dl_bandwidth.dl_period = global_rt_period();
8253 	def_dl_bandwidth.dl_runtime = global_rt_runtime();
8254 
8255 	if (global_rt_runtime() != RUNTIME_INF)
8256 		new_bw = to_ratio(global_rt_period(), global_rt_runtime());
8257 
8258 	/*
8259 	 * FIXME: As above...
8260 	 */
8261 	for_each_possible_cpu(cpu) {
8262 		rcu_read_lock_sched();
8263 		dl_b = dl_bw_of(cpu);
8264 
8265 		raw_spin_lock_irqsave(&dl_b->lock, flags);
8266 		dl_b->bw = new_bw;
8267 		raw_spin_unlock_irqrestore(&dl_b->lock, flags);
8268 
8269 		rcu_read_unlock_sched();
8270 	}
8271 }
8272 
8273 static int sched_rt_global_validate(void)
8274 {
8275 	if (sysctl_sched_rt_period <= 0)
8276 		return -EINVAL;
8277 
8278 	if ((sysctl_sched_rt_runtime != RUNTIME_INF) &&
8279 		(sysctl_sched_rt_runtime > sysctl_sched_rt_period))
8280 		return -EINVAL;
8281 
8282 	return 0;
8283 }
8284 
8285 static void sched_rt_do_global(void)
8286 {
8287 	def_rt_bandwidth.rt_runtime = global_rt_runtime();
8288 	def_rt_bandwidth.rt_period = ns_to_ktime(global_rt_period());
8289 }
8290 
8291 int sched_rt_handler(struct ctl_table *table, int write,
8292 		void __user *buffer, size_t *lenp,
8293 		loff_t *ppos)
8294 {
8295 	int old_period, old_runtime;
8296 	static DEFINE_MUTEX(mutex);
8297 	int ret;
8298 
8299 	mutex_lock(&mutex);
8300 	old_period = sysctl_sched_rt_period;
8301 	old_runtime = sysctl_sched_rt_runtime;
8302 
8303 	ret = proc_dointvec(table, write, buffer, lenp, ppos);
8304 
8305 	if (!ret && write) {
8306 		ret = sched_rt_global_validate();
8307 		if (ret)
8308 			goto undo;
8309 
8310 		ret = sched_dl_global_validate();
8311 		if (ret)
8312 			goto undo;
8313 
8314 		ret = sched_rt_global_constraints();
8315 		if (ret)
8316 			goto undo;
8317 
8318 		sched_rt_do_global();
8319 		sched_dl_do_global();
8320 	}
8321 	if (0) {
8322 undo:
8323 		sysctl_sched_rt_period = old_period;
8324 		sysctl_sched_rt_runtime = old_runtime;
8325 	}
8326 	mutex_unlock(&mutex);
8327 
8328 	return ret;
8329 }
8330 
8331 int sched_rr_handler(struct ctl_table *table, int write,
8332 		void __user *buffer, size_t *lenp,
8333 		loff_t *ppos)
8334 {
8335 	int ret;
8336 	static DEFINE_MUTEX(mutex);
8337 
8338 	mutex_lock(&mutex);
8339 	ret = proc_dointvec(table, write, buffer, lenp, ppos);
8340 	/* make sure that internally we keep jiffies */
8341 	/* also, writing zero resets timeslice to default */
8342 	if (!ret && write) {
8343 		sched_rr_timeslice = sched_rr_timeslice <= 0 ?
8344 			RR_TIMESLICE : msecs_to_jiffies(sched_rr_timeslice);
8345 	}
8346 	mutex_unlock(&mutex);
8347 	return ret;
8348 }
8349 
8350 #ifdef CONFIG_CGROUP_SCHED
8351 
8352 static inline struct task_group *css_tg(struct cgroup_subsys_state *css)
8353 {
8354 	return css ? container_of(css, struct task_group, css) : NULL;
8355 }
8356 
8357 static struct cgroup_subsys_state *
8358 cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
8359 {
8360 	struct task_group *parent = css_tg(parent_css);
8361 	struct task_group *tg;
8362 
8363 	if (!parent) {
8364 		/* This is early initialization for the top cgroup */
8365 		return &root_task_group.css;
8366 	}
8367 
8368 	tg = sched_create_group(parent);
8369 	if (IS_ERR(tg))
8370 		return ERR_PTR(-ENOMEM);
8371 
8372 	sched_online_group(tg, parent);
8373 
8374 	return &tg->css;
8375 }
8376 
8377 static void cpu_cgroup_css_released(struct cgroup_subsys_state *css)
8378 {
8379 	struct task_group *tg = css_tg(css);
8380 
8381 	sched_offline_group(tg);
8382 }
8383 
8384 static void cpu_cgroup_css_free(struct cgroup_subsys_state *css)
8385 {
8386 	struct task_group *tg = css_tg(css);
8387 
8388 	/*
8389 	 * Relies on the RCU grace period between css_released() and this.
8390 	 */
8391 	sched_free_group(tg);
8392 }
8393 
8394 /*
8395  * This is called before wake_up_new_task(), therefore we really only
8396  * have to set its group bits, all the other stuff does not apply.
8397  */
8398 static void cpu_cgroup_fork(struct task_struct *task)
8399 {
8400 	struct rq_flags rf;
8401 	struct rq *rq;
8402 
8403 	rq = task_rq_lock(task, &rf);
8404 
8405 	sched_change_group(task, TASK_SET_GROUP);
8406 
8407 	task_rq_unlock(rq, task, &rf);
8408 }
8409 
8410 static int cpu_cgroup_can_attach(struct cgroup_taskset *tset)
8411 {
8412 	struct task_struct *task;
8413 	struct cgroup_subsys_state *css;
8414 	int ret = 0;
8415 
8416 	cgroup_taskset_for_each(task, css, tset) {
8417 #ifdef CONFIG_RT_GROUP_SCHED
8418 		if (!sched_rt_can_attach(css_tg(css), task))
8419 			return -EINVAL;
8420 #else
8421 		/* We don't support RT-tasks being in separate groups */
8422 		if (task->sched_class != &fair_sched_class)
8423 			return -EINVAL;
8424 #endif
8425 		/*
8426 		 * Serialize against wake_up_new_task() such that if its
8427 		 * running, we're sure to observe its full state.
8428 		 */
8429 		raw_spin_lock_irq(&task->pi_lock);
8430 		/*
8431 		 * Avoid calling sched_move_task() before wake_up_new_task()
8432 		 * has happened. This would lead to problems with PELT, due to
8433 		 * move wanting to detach+attach while we're not attached yet.
8434 		 */
8435 		if (task->state == TASK_NEW)
8436 			ret = -EINVAL;
8437 		raw_spin_unlock_irq(&task->pi_lock);
8438 
8439 		if (ret)
8440 			break;
8441 	}
8442 	return ret;
8443 }
8444 
8445 static void cpu_cgroup_attach(struct cgroup_taskset *tset)
8446 {
8447 	struct task_struct *task;
8448 	struct cgroup_subsys_state *css;
8449 
8450 	cgroup_taskset_for_each(task, css, tset)
8451 		sched_move_task(task);
8452 }
8453 
8454 #ifdef CONFIG_FAIR_GROUP_SCHED
8455 static int cpu_shares_write_u64(struct cgroup_subsys_state *css,
8456 				struct cftype *cftype, u64 shareval)
8457 {
8458 	return sched_group_set_shares(css_tg(css), scale_load(shareval));
8459 }
8460 
8461 static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css,
8462 			       struct cftype *cft)
8463 {
8464 	struct task_group *tg = css_tg(css);
8465 
8466 	return (u64) scale_load_down(tg->shares);
8467 }
8468 
8469 #ifdef CONFIG_CFS_BANDWIDTH
8470 static DEFINE_MUTEX(cfs_constraints_mutex);
8471 
8472 const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */
8473 const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */
8474 
8475 static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime);
8476 
8477 static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
8478 {
8479 	int i, ret = 0, runtime_enabled, runtime_was_enabled;
8480 	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
8481 
8482 	if (tg == &root_task_group)
8483 		return -EINVAL;
8484 
8485 	/*
8486 	 * Ensure we have at some amount of bandwidth every period.  This is
8487 	 * to prevent reaching a state of large arrears when throttled via
8488 	 * entity_tick() resulting in prolonged exit starvation.
8489 	 */
8490 	if (quota < min_cfs_quota_period || period < min_cfs_quota_period)
8491 		return -EINVAL;
8492 
8493 	/*
8494 	 * Likewise, bound things on the otherside by preventing insane quota
8495 	 * periods.  This also allows us to normalize in computing quota
8496 	 * feasibility.
8497 	 */
8498 	if (period > max_cfs_quota_period)
8499 		return -EINVAL;
8500 
8501 	/*
8502 	 * Prevent race between setting of cfs_rq->runtime_enabled and
8503 	 * unthrottle_offline_cfs_rqs().
8504 	 */
8505 	get_online_cpus();
8506 	mutex_lock(&cfs_constraints_mutex);
8507 	ret = __cfs_schedulable(tg, period, quota);
8508 	if (ret)
8509 		goto out_unlock;
8510 
8511 	runtime_enabled = quota != RUNTIME_INF;
8512 	runtime_was_enabled = cfs_b->quota != RUNTIME_INF;
8513 	/*
8514 	 * If we need to toggle cfs_bandwidth_used, off->on must occur
8515 	 * before making related changes, and on->off must occur afterwards
8516 	 */
8517 	if (runtime_enabled && !runtime_was_enabled)
8518 		cfs_bandwidth_usage_inc();
8519 	raw_spin_lock_irq(&cfs_b->lock);
8520 	cfs_b->period = ns_to_ktime(period);
8521 	cfs_b->quota = quota;
8522 
8523 	__refill_cfs_bandwidth_runtime(cfs_b);
8524 	/* restart the period timer (if active) to handle new period expiry */
8525 	if (runtime_enabled)
8526 		start_cfs_bandwidth(cfs_b);
8527 	raw_spin_unlock_irq(&cfs_b->lock);
8528 
8529 	for_each_online_cpu(i) {
8530 		struct cfs_rq *cfs_rq = tg->cfs_rq[i];
8531 		struct rq *rq = cfs_rq->rq;
8532 
8533 		raw_spin_lock_irq(&rq->lock);
8534 		cfs_rq->runtime_enabled = runtime_enabled;
8535 		cfs_rq->runtime_remaining = 0;
8536 
8537 		if (cfs_rq->throttled)
8538 			unthrottle_cfs_rq(cfs_rq);
8539 		raw_spin_unlock_irq(&rq->lock);
8540 	}
8541 	if (runtime_was_enabled && !runtime_enabled)
8542 		cfs_bandwidth_usage_dec();
8543 out_unlock:
8544 	mutex_unlock(&cfs_constraints_mutex);
8545 	put_online_cpus();
8546 
8547 	return ret;
8548 }
8549 
8550 int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us)
8551 {
8552 	u64 quota, period;
8553 
8554 	period = ktime_to_ns(tg->cfs_bandwidth.period);
8555 	if (cfs_quota_us < 0)
8556 		quota = RUNTIME_INF;
8557 	else
8558 		quota = (u64)cfs_quota_us * NSEC_PER_USEC;
8559 
8560 	return tg_set_cfs_bandwidth(tg, period, quota);
8561 }
8562 
8563 long tg_get_cfs_quota(struct task_group *tg)
8564 {
8565 	u64 quota_us;
8566 
8567 	if (tg->cfs_bandwidth.quota == RUNTIME_INF)
8568 		return -1;
8569 
8570 	quota_us = tg->cfs_bandwidth.quota;
8571 	do_div(quota_us, NSEC_PER_USEC);
8572 
8573 	return quota_us;
8574 }
8575 
8576 int tg_set_cfs_period(struct task_group *tg, long cfs_period_us)
8577 {
8578 	u64 quota, period;
8579 
8580 	period = (u64)cfs_period_us * NSEC_PER_USEC;
8581 	quota = tg->cfs_bandwidth.quota;
8582 
8583 	return tg_set_cfs_bandwidth(tg, period, quota);
8584 }
8585 
8586 long tg_get_cfs_period(struct task_group *tg)
8587 {
8588 	u64 cfs_period_us;
8589 
8590 	cfs_period_us = ktime_to_ns(tg->cfs_bandwidth.period);
8591 	do_div(cfs_period_us, NSEC_PER_USEC);
8592 
8593 	return cfs_period_us;
8594 }
8595 
8596 static s64 cpu_cfs_quota_read_s64(struct cgroup_subsys_state *css,
8597 				  struct cftype *cft)
8598 {
8599 	return tg_get_cfs_quota(css_tg(css));
8600 }
8601 
8602 static int cpu_cfs_quota_write_s64(struct cgroup_subsys_state *css,
8603 				   struct cftype *cftype, s64 cfs_quota_us)
8604 {
8605 	return tg_set_cfs_quota(css_tg(css), cfs_quota_us);
8606 }
8607 
8608 static u64 cpu_cfs_period_read_u64(struct cgroup_subsys_state *css,
8609 				   struct cftype *cft)
8610 {
8611 	return tg_get_cfs_period(css_tg(css));
8612 }
8613 
8614 static int cpu_cfs_period_write_u64(struct cgroup_subsys_state *css,
8615 				    struct cftype *cftype, u64 cfs_period_us)
8616 {
8617 	return tg_set_cfs_period(css_tg(css), cfs_period_us);
8618 }
8619 
8620 struct cfs_schedulable_data {
8621 	struct task_group *tg;
8622 	u64 period, quota;
8623 };
8624 
8625 /*
8626  * normalize group quota/period to be quota/max_period
8627  * note: units are usecs
8628  */
8629 static u64 normalize_cfs_quota(struct task_group *tg,
8630 			       struct cfs_schedulable_data *d)
8631 {
8632 	u64 quota, period;
8633 
8634 	if (tg == d->tg) {
8635 		period = d->period;
8636 		quota = d->quota;
8637 	} else {
8638 		period = tg_get_cfs_period(tg);
8639 		quota = tg_get_cfs_quota(tg);
8640 	}
8641 
8642 	/* note: these should typically be equivalent */
8643 	if (quota == RUNTIME_INF || quota == -1)
8644 		return RUNTIME_INF;
8645 
8646 	return to_ratio(period, quota);
8647 }
8648 
8649 static int tg_cfs_schedulable_down(struct task_group *tg, void *data)
8650 {
8651 	struct cfs_schedulable_data *d = data;
8652 	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
8653 	s64 quota = 0, parent_quota = -1;
8654 
8655 	if (!tg->parent) {
8656 		quota = RUNTIME_INF;
8657 	} else {
8658 		struct cfs_bandwidth *parent_b = &tg->parent->cfs_bandwidth;
8659 
8660 		quota = normalize_cfs_quota(tg, d);
8661 		parent_quota = parent_b->hierarchical_quota;
8662 
8663 		/*
8664 		 * ensure max(child_quota) <= parent_quota, inherit when no
8665 		 * limit is set
8666 		 */
8667 		if (quota == RUNTIME_INF)
8668 			quota = parent_quota;
8669 		else if (parent_quota != RUNTIME_INF && quota > parent_quota)
8670 			return -EINVAL;
8671 	}
8672 	cfs_b->hierarchical_quota = quota;
8673 
8674 	return 0;
8675 }
8676 
8677 static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota)
8678 {
8679 	int ret;
8680 	struct cfs_schedulable_data data = {
8681 		.tg = tg,
8682 		.period = period,
8683 		.quota = quota,
8684 	};
8685 
8686 	if (quota != RUNTIME_INF) {
8687 		do_div(data.period, NSEC_PER_USEC);
8688 		do_div(data.quota, NSEC_PER_USEC);
8689 	}
8690 
8691 	rcu_read_lock();
8692 	ret = walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data);
8693 	rcu_read_unlock();
8694 
8695 	return ret;
8696 }
8697 
8698 static int cpu_stats_show(struct seq_file *sf, void *v)
8699 {
8700 	struct task_group *tg = css_tg(seq_css(sf));
8701 	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
8702 
8703 	seq_printf(sf, "nr_periods %d\n", cfs_b->nr_periods);
8704 	seq_printf(sf, "nr_throttled %d\n", cfs_b->nr_throttled);
8705 	seq_printf(sf, "throttled_time %llu\n", cfs_b->throttled_time);
8706 
8707 	return 0;
8708 }
8709 #endif /* CONFIG_CFS_BANDWIDTH */
8710 #endif /* CONFIG_FAIR_GROUP_SCHED */
8711 
8712 #ifdef CONFIG_RT_GROUP_SCHED
8713 static int cpu_rt_runtime_write(struct cgroup_subsys_state *css,
8714 				struct cftype *cft, s64 val)
8715 {
8716 	return sched_group_set_rt_runtime(css_tg(css), val);
8717 }
8718 
8719 static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css,
8720 			       struct cftype *cft)
8721 {
8722 	return sched_group_rt_runtime(css_tg(css));
8723 }
8724 
8725 static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css,
8726 				    struct cftype *cftype, u64 rt_period_us)
8727 {
8728 	return sched_group_set_rt_period(css_tg(css), rt_period_us);
8729 }
8730 
8731 static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css,
8732 				   struct cftype *cft)
8733 {
8734 	return sched_group_rt_period(css_tg(css));
8735 }
8736 #endif /* CONFIG_RT_GROUP_SCHED */
8737 
8738 static struct cftype cpu_files[] = {
8739 #ifdef CONFIG_FAIR_GROUP_SCHED
8740 	{
8741 		.name = "shares",
8742 		.read_u64 = cpu_shares_read_u64,
8743 		.write_u64 = cpu_shares_write_u64,
8744 	},
8745 #endif
8746 #ifdef CONFIG_CFS_BANDWIDTH
8747 	{
8748 		.name = "cfs_quota_us",
8749 		.read_s64 = cpu_cfs_quota_read_s64,
8750 		.write_s64 = cpu_cfs_quota_write_s64,
8751 	},
8752 	{
8753 		.name = "cfs_period_us",
8754 		.read_u64 = cpu_cfs_period_read_u64,
8755 		.write_u64 = cpu_cfs_period_write_u64,
8756 	},
8757 	{
8758 		.name = "stat",
8759 		.seq_show = cpu_stats_show,
8760 	},
8761 #endif
8762 #ifdef CONFIG_RT_GROUP_SCHED
8763 	{
8764 		.name = "rt_runtime_us",
8765 		.read_s64 = cpu_rt_runtime_read,
8766 		.write_s64 = cpu_rt_runtime_write,
8767 	},
8768 	{
8769 		.name = "rt_period_us",
8770 		.read_u64 = cpu_rt_period_read_uint,
8771 		.write_u64 = cpu_rt_period_write_uint,
8772 	},
8773 #endif
8774 	{ }	/* terminate */
8775 };
8776 
8777 struct cgroup_subsys cpu_cgrp_subsys = {
8778 	.css_alloc	= cpu_cgroup_css_alloc,
8779 	.css_released	= cpu_cgroup_css_released,
8780 	.css_free	= cpu_cgroup_css_free,
8781 	.fork		= cpu_cgroup_fork,
8782 	.can_attach	= cpu_cgroup_can_attach,
8783 	.attach		= cpu_cgroup_attach,
8784 	.legacy_cftypes	= cpu_files,
8785 	.early_init	= true,
8786 };
8787 
8788 #endif	/* CONFIG_CGROUP_SCHED */
8789 
8790 void dump_cpu_task(int cpu)
8791 {
8792 	pr_info("Task dump for CPU %d:\n", cpu);
8793 	sched_show_task(cpu_curr(cpu));
8794 }
8795 
8796 /*
8797  * Nice levels are multiplicative, with a gentle 10% change for every
8798  * nice level changed. I.e. when a CPU-bound task goes from nice 0 to
8799  * nice 1, it will get ~10% less CPU time than another CPU-bound task
8800  * that remained on nice 0.
8801  *
8802  * The "10% effect" is relative and cumulative: from _any_ nice level,
8803  * if you go up 1 level, it's -10% CPU usage, if you go down 1 level
8804  * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25.
8805  * If a task goes up by ~10% and another task goes down by ~10% then
8806  * the relative distance between them is ~25%.)
8807  */
8808 const int sched_prio_to_weight[40] = {
8809  /* -20 */     88761,     71755,     56483,     46273,     36291,
8810  /* -15 */     29154,     23254,     18705,     14949,     11916,
8811  /* -10 */      9548,      7620,      6100,      4904,      3906,
8812  /*  -5 */      3121,      2501,      1991,      1586,      1277,
8813  /*   0 */      1024,       820,       655,       526,       423,
8814  /*   5 */       335,       272,       215,       172,       137,
8815  /*  10 */       110,        87,        70,        56,        45,
8816  /*  15 */        36,        29,        23,        18,        15,
8817 };
8818 
8819 /*
8820  * Inverse (2^32/x) values of the sched_prio_to_weight[] array, precalculated.
8821  *
8822  * In cases where the weight does not change often, we can use the
8823  * precalculated inverse to speed up arithmetics by turning divisions
8824  * into multiplications:
8825  */
8826 const u32 sched_prio_to_wmult[40] = {
8827  /* -20 */     48388,     59856,     76040,     92818,    118348,
8828  /* -15 */    147320,    184698,    229616,    287308,    360437,
8829  /* -10 */    449829,    563644,    704093,    875809,   1099582,
8830  /*  -5 */   1376151,   1717300,   2157191,   2708050,   3363326,
8831  /*   0 */   4194304,   5237765,   6557202,   8165337,  10153587,
8832  /*   5 */  12820798,  15790321,  19976592,  24970740,  31350126,
8833  /*  10 */  39045157,  49367440,  61356676,  76695844,  95443717,
8834  /*  15 */ 119304647, 148102320, 186737708, 238609294, 286331153,
8835 };
8836