xref: /openbmc/linux/kernel/sched/rt.c (revision c7fcb99877f9f542c918509b2801065adcaf46fa)
1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0
2391e43daSPeter Zijlstra /*
3391e43daSPeter Zijlstra  * Real-Time Scheduling Class (mapped to the SCHED_FIFO and SCHED_RR
4391e43daSPeter Zijlstra  * policies)
5391e43daSPeter Zijlstra  */
6371bf427SVincent Guittot 
7ce0dbbbbSClark Williams int sched_rr_timeslice = RR_TIMESLICE;
8d505b8afSHuaixin Chang /* More than 4 hours if BW_SHIFT equals 20. */
9d505b8afSHuaixin Chang static const u64 max_rt_runtime = MAX_BW;
10ce0dbbbbSClark Williams 
11391e43daSPeter Zijlstra static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);
12391e43daSPeter Zijlstra 
13391e43daSPeter Zijlstra struct rt_bandwidth def_rt_bandwidth;
14391e43daSPeter Zijlstra 
15d9ab0e63SZhen Ni /*
16d9ab0e63SZhen Ni  * period over which we measure -rt task CPU usage in us.
17d9ab0e63SZhen Ni  * default: 1s
18d9ab0e63SZhen Ni  */
19d9ab0e63SZhen Ni unsigned int sysctl_sched_rt_period = 1000000;
20d9ab0e63SZhen Ni 
21d9ab0e63SZhen Ni /*
22d9ab0e63SZhen Ni  * part of the period that we allow rt tasks to run in us.
23d9ab0e63SZhen Ni  * default: 0.95s
24d9ab0e63SZhen Ni  */
25d9ab0e63SZhen Ni int sysctl_sched_rt_runtime = 950000;
26d9ab0e63SZhen Ni 
2728f152cdSBaisong Zhong #ifdef CONFIG_SYSCTL
28*c7fcb998SCyril Hrubis static int sysctl_sched_rr_timeslice = (MSEC_PER_SEC * RR_TIMESLICE) / HZ;
29d9ab0e63SZhen Ni static int sched_rt_handler(struct ctl_table *table, int write, void *buffer,
30d9ab0e63SZhen Ni 		size_t *lenp, loff_t *ppos);
31dafd7a9dSZhen Ni static int sched_rr_handler(struct ctl_table *table, int write, void *buffer,
32dafd7a9dSZhen Ni 		size_t *lenp, loff_t *ppos);
33d9ab0e63SZhen Ni static struct ctl_table sched_rt_sysctls[] = {
34d9ab0e63SZhen Ni 	{
35d9ab0e63SZhen Ni 		.procname       = "sched_rt_period_us",
36d9ab0e63SZhen Ni 		.data           = &sysctl_sched_rt_period,
37d9ab0e63SZhen Ni 		.maxlen         = sizeof(unsigned int),
38d9ab0e63SZhen Ni 		.mode           = 0644,
39d9ab0e63SZhen Ni 		.proc_handler   = sched_rt_handler,
40d9ab0e63SZhen Ni 	},
41d9ab0e63SZhen Ni 	{
42d9ab0e63SZhen Ni 		.procname       = "sched_rt_runtime_us",
43d9ab0e63SZhen Ni 		.data           = &sysctl_sched_rt_runtime,
44d9ab0e63SZhen Ni 		.maxlen         = sizeof(int),
45d9ab0e63SZhen Ni 		.mode           = 0644,
46d9ab0e63SZhen Ni 		.proc_handler   = sched_rt_handler,
47d9ab0e63SZhen Ni 	},
48dafd7a9dSZhen Ni 	{
49dafd7a9dSZhen Ni 		.procname       = "sched_rr_timeslice_ms",
50dafd7a9dSZhen Ni 		.data           = &sysctl_sched_rr_timeslice,
51dafd7a9dSZhen Ni 		.maxlen         = sizeof(int),
52dafd7a9dSZhen Ni 		.mode           = 0644,
53dafd7a9dSZhen Ni 		.proc_handler   = sched_rr_handler,
54dafd7a9dSZhen Ni 	},
55d9ab0e63SZhen Ni 	{}
56d9ab0e63SZhen Ni };
57d9ab0e63SZhen Ni 
58d9ab0e63SZhen Ni static int __init sched_rt_sysctl_init(void)
59d9ab0e63SZhen Ni {
60d9ab0e63SZhen Ni 	register_sysctl_init("kernel", sched_rt_sysctls);
61d9ab0e63SZhen Ni 	return 0;
62d9ab0e63SZhen Ni }
63d9ab0e63SZhen Ni late_initcall(sched_rt_sysctl_init);
64d9ab0e63SZhen Ni #endif
65d9ab0e63SZhen Ni 
66391e43daSPeter Zijlstra static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer)
67391e43daSPeter Zijlstra {
68391e43daSPeter Zijlstra 	struct rt_bandwidth *rt_b =
69391e43daSPeter Zijlstra 		container_of(timer, struct rt_bandwidth, rt_period_timer);
70391e43daSPeter Zijlstra 	int idle = 0;
7177a4d1a1SPeter Zijlstra 	int overrun;
72391e43daSPeter Zijlstra 
7377a4d1a1SPeter Zijlstra 	raw_spin_lock(&rt_b->rt_runtime_lock);
74391e43daSPeter Zijlstra 	for (;;) {
7577a4d1a1SPeter Zijlstra 		overrun = hrtimer_forward_now(timer, rt_b->rt_period);
76391e43daSPeter Zijlstra 		if (!overrun)
77391e43daSPeter Zijlstra 			break;
78391e43daSPeter Zijlstra 
7977a4d1a1SPeter Zijlstra 		raw_spin_unlock(&rt_b->rt_runtime_lock);
80391e43daSPeter Zijlstra 		idle = do_sched_rt_period_timer(rt_b, overrun);
8177a4d1a1SPeter Zijlstra 		raw_spin_lock(&rt_b->rt_runtime_lock);
82391e43daSPeter Zijlstra 	}
834cfafd30SPeter Zijlstra 	if (idle)
844cfafd30SPeter Zijlstra 		rt_b->rt_period_active = 0;
8577a4d1a1SPeter Zijlstra 	raw_spin_unlock(&rt_b->rt_runtime_lock);
86391e43daSPeter Zijlstra 
87391e43daSPeter Zijlstra 	return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
88391e43daSPeter Zijlstra }
89391e43daSPeter Zijlstra 
90391e43daSPeter Zijlstra void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
91391e43daSPeter Zijlstra {
92391e43daSPeter Zijlstra 	rt_b->rt_period = ns_to_ktime(period);
93391e43daSPeter Zijlstra 	rt_b->rt_runtime = runtime;
94391e43daSPeter Zijlstra 
95391e43daSPeter Zijlstra 	raw_spin_lock_init(&rt_b->rt_runtime_lock);
96391e43daSPeter Zijlstra 
97d5096aa6SSebastian Andrzej Siewior 	hrtimer_init(&rt_b->rt_period_timer, CLOCK_MONOTONIC,
98d5096aa6SSebastian Andrzej Siewior 		     HRTIMER_MODE_REL_HARD);
99391e43daSPeter Zijlstra 	rt_b->rt_period_timer.function = sched_rt_period_timer;
100391e43daSPeter Zijlstra }
101391e43daSPeter Zijlstra 
1029b58e976SLi Hua static inline void do_start_rt_bandwidth(struct rt_bandwidth *rt_b)
103391e43daSPeter Zijlstra {
104391e43daSPeter Zijlstra 	raw_spin_lock(&rt_b->rt_runtime_lock);
1054cfafd30SPeter Zijlstra 	if (!rt_b->rt_period_active) {
1064cfafd30SPeter Zijlstra 		rt_b->rt_period_active = 1;
107c3a990dcSSteven Rostedt 		/*
108c3a990dcSSteven Rostedt 		 * SCHED_DEADLINE updates the bandwidth, as a run away
109c3a990dcSSteven Rostedt 		 * RT task with a DL task could hog a CPU. But DL does
110c3a990dcSSteven Rostedt 		 * not reset the period. If a deadline task was running
111c3a990dcSSteven Rostedt 		 * without an RT task running, it can cause RT tasks to
112c3a990dcSSteven Rostedt 		 * throttle when they start up. Kick the timer right away
113c3a990dcSSteven Rostedt 		 * to update the period.
114c3a990dcSSteven Rostedt 		 */
115c3a990dcSSteven Rostedt 		hrtimer_forward_now(&rt_b->rt_period_timer, ns_to_ktime(0));
116d5096aa6SSebastian Andrzej Siewior 		hrtimer_start_expires(&rt_b->rt_period_timer,
117d5096aa6SSebastian Andrzej Siewior 				      HRTIMER_MODE_ABS_PINNED_HARD);
1184cfafd30SPeter Zijlstra 	}
119391e43daSPeter Zijlstra 	raw_spin_unlock(&rt_b->rt_runtime_lock);
120391e43daSPeter Zijlstra }
121391e43daSPeter Zijlstra 
1229b58e976SLi Hua static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
1239b58e976SLi Hua {
1249b58e976SLi Hua 	if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)
1259b58e976SLi Hua 		return;
1269b58e976SLi Hua 
1279b58e976SLi Hua 	do_start_rt_bandwidth(rt_b);
1289b58e976SLi Hua }
1299b58e976SLi Hua 
13007c54f7aSAbel Vesa void init_rt_rq(struct rt_rq *rt_rq)
131391e43daSPeter Zijlstra {
132391e43daSPeter Zijlstra 	struct rt_prio_array *array;
133391e43daSPeter Zijlstra 	int i;
134391e43daSPeter Zijlstra 
135391e43daSPeter Zijlstra 	array = &rt_rq->active;
136391e43daSPeter Zijlstra 	for (i = 0; i < MAX_RT_PRIO; i++) {
137391e43daSPeter Zijlstra 		INIT_LIST_HEAD(array->queue + i);
138391e43daSPeter Zijlstra 		__clear_bit(i, array->bitmap);
139391e43daSPeter Zijlstra 	}
140391e43daSPeter Zijlstra 	/* delimiter for bitsearch: */
141391e43daSPeter Zijlstra 	__set_bit(MAX_RT_PRIO, array->bitmap);
142391e43daSPeter Zijlstra 
143391e43daSPeter Zijlstra #if defined CONFIG_SMP
144934fc331SPeter Zijlstra 	rt_rq->highest_prio.curr = MAX_RT_PRIO-1;
145934fc331SPeter Zijlstra 	rt_rq->highest_prio.next = MAX_RT_PRIO-1;
146391e43daSPeter Zijlstra 	rt_rq->rt_nr_migratory = 0;
147391e43daSPeter Zijlstra 	rt_rq->overloaded = 0;
148391e43daSPeter Zijlstra 	plist_head_init(&rt_rq->pushable_tasks);
149b6366f04SSteven Rostedt #endif /* CONFIG_SMP */
150f4ebcbc0SKirill Tkhai 	/* We start is dequeued state, because no RT tasks are queued */
151f4ebcbc0SKirill Tkhai 	rt_rq->rt_queued = 0;
152391e43daSPeter Zijlstra 
153391e43daSPeter Zijlstra 	rt_rq->rt_time = 0;
154391e43daSPeter Zijlstra 	rt_rq->rt_throttled = 0;
155391e43daSPeter Zijlstra 	rt_rq->rt_runtime = 0;
156391e43daSPeter Zijlstra 	raw_spin_lock_init(&rt_rq->rt_runtime_lock);
157391e43daSPeter Zijlstra }
158391e43daSPeter Zijlstra 
159391e43daSPeter Zijlstra #ifdef CONFIG_RT_GROUP_SCHED
160391e43daSPeter Zijlstra static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b)
161391e43daSPeter Zijlstra {
162391e43daSPeter Zijlstra 	hrtimer_cancel(&rt_b->rt_period_timer);
163391e43daSPeter Zijlstra }
164391e43daSPeter Zijlstra 
165391e43daSPeter Zijlstra #define rt_entity_is_task(rt_se) (!(rt_se)->my_q)
166391e43daSPeter Zijlstra 
167391e43daSPeter Zijlstra static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)
168391e43daSPeter Zijlstra {
169391e43daSPeter Zijlstra #ifdef CONFIG_SCHED_DEBUG
170391e43daSPeter Zijlstra 	WARN_ON_ONCE(!rt_entity_is_task(rt_se));
171391e43daSPeter Zijlstra #endif
172391e43daSPeter Zijlstra 	return container_of(rt_se, struct task_struct, rt);
173391e43daSPeter Zijlstra }
174391e43daSPeter Zijlstra 
175391e43daSPeter Zijlstra static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)
176391e43daSPeter Zijlstra {
177391e43daSPeter Zijlstra 	return rt_rq->rq;
178391e43daSPeter Zijlstra }
179391e43daSPeter Zijlstra 
180391e43daSPeter Zijlstra static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)
181391e43daSPeter Zijlstra {
182391e43daSPeter Zijlstra 	return rt_se->rt_rq;
183391e43daSPeter Zijlstra }
184391e43daSPeter Zijlstra 
185653d07a6SKirill Tkhai static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se)
186653d07a6SKirill Tkhai {
187653d07a6SKirill Tkhai 	struct rt_rq *rt_rq = rt_se->rt_rq;
188653d07a6SKirill Tkhai 
189653d07a6SKirill Tkhai 	return rt_rq->rq;
190653d07a6SKirill Tkhai }
191653d07a6SKirill Tkhai 
192b027789eSMathias Krause void unregister_rt_sched_group(struct task_group *tg)
193b027789eSMathias Krause {
194b027789eSMathias Krause 	if (tg->rt_se)
195b027789eSMathias Krause 		destroy_rt_bandwidth(&tg->rt_bandwidth);
196b027789eSMathias Krause 
197b027789eSMathias Krause }
198b027789eSMathias Krause 
199391e43daSPeter Zijlstra void free_rt_sched_group(struct task_group *tg)
200391e43daSPeter Zijlstra {
201391e43daSPeter Zijlstra 	int i;
202391e43daSPeter Zijlstra 
203391e43daSPeter Zijlstra 	for_each_possible_cpu(i) {
204391e43daSPeter Zijlstra 		if (tg->rt_rq)
205391e43daSPeter Zijlstra 			kfree(tg->rt_rq[i]);
206391e43daSPeter Zijlstra 		if (tg->rt_se)
207391e43daSPeter Zijlstra 			kfree(tg->rt_se[i]);
208391e43daSPeter Zijlstra 	}
209391e43daSPeter Zijlstra 
210391e43daSPeter Zijlstra 	kfree(tg->rt_rq);
211391e43daSPeter Zijlstra 	kfree(tg->rt_se);
212391e43daSPeter Zijlstra }
213391e43daSPeter Zijlstra 
214391e43daSPeter Zijlstra void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
215391e43daSPeter Zijlstra 		struct sched_rt_entity *rt_se, int cpu,
216391e43daSPeter Zijlstra 		struct sched_rt_entity *parent)
217391e43daSPeter Zijlstra {
218391e43daSPeter Zijlstra 	struct rq *rq = cpu_rq(cpu);
219391e43daSPeter Zijlstra 
220934fc331SPeter Zijlstra 	rt_rq->highest_prio.curr = MAX_RT_PRIO-1;
221391e43daSPeter Zijlstra 	rt_rq->rt_nr_boosted = 0;
222391e43daSPeter Zijlstra 	rt_rq->rq = rq;
223391e43daSPeter Zijlstra 	rt_rq->tg = tg;
224391e43daSPeter Zijlstra 
225391e43daSPeter Zijlstra 	tg->rt_rq[cpu] = rt_rq;
226391e43daSPeter Zijlstra 	tg->rt_se[cpu] = rt_se;
227391e43daSPeter Zijlstra 
228391e43daSPeter Zijlstra 	if (!rt_se)
229391e43daSPeter Zijlstra 		return;
230391e43daSPeter Zijlstra 
231391e43daSPeter Zijlstra 	if (!parent)
232391e43daSPeter Zijlstra 		rt_se->rt_rq = &rq->rt;
233391e43daSPeter Zijlstra 	else
234391e43daSPeter Zijlstra 		rt_se->rt_rq = parent->my_q;
235391e43daSPeter Zijlstra 
236391e43daSPeter Zijlstra 	rt_se->my_q = rt_rq;
237391e43daSPeter Zijlstra 	rt_se->parent = parent;
238391e43daSPeter Zijlstra 	INIT_LIST_HEAD(&rt_se->run_list);
239391e43daSPeter Zijlstra }
240391e43daSPeter Zijlstra 
241391e43daSPeter Zijlstra int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
242391e43daSPeter Zijlstra {
243391e43daSPeter Zijlstra 	struct rt_rq *rt_rq;
244391e43daSPeter Zijlstra 	struct sched_rt_entity *rt_se;
245391e43daSPeter Zijlstra 	int i;
246391e43daSPeter Zijlstra 
2476396bb22SKees Cook 	tg->rt_rq = kcalloc(nr_cpu_ids, sizeof(rt_rq), GFP_KERNEL);
248391e43daSPeter Zijlstra 	if (!tg->rt_rq)
249391e43daSPeter Zijlstra 		goto err;
2506396bb22SKees Cook 	tg->rt_se = kcalloc(nr_cpu_ids, sizeof(rt_se), GFP_KERNEL);
251391e43daSPeter Zijlstra 	if (!tg->rt_se)
252391e43daSPeter Zijlstra 		goto err;
253391e43daSPeter Zijlstra 
254391e43daSPeter Zijlstra 	init_rt_bandwidth(&tg->rt_bandwidth,
255391e43daSPeter Zijlstra 			ktime_to_ns(def_rt_bandwidth.rt_period), 0);
256391e43daSPeter Zijlstra 
257391e43daSPeter Zijlstra 	for_each_possible_cpu(i) {
258391e43daSPeter Zijlstra 		rt_rq = kzalloc_node(sizeof(struct rt_rq),
259391e43daSPeter Zijlstra 				     GFP_KERNEL, cpu_to_node(i));
260391e43daSPeter Zijlstra 		if (!rt_rq)
261391e43daSPeter Zijlstra 			goto err;
262391e43daSPeter Zijlstra 
263391e43daSPeter Zijlstra 		rt_se = kzalloc_node(sizeof(struct sched_rt_entity),
264391e43daSPeter Zijlstra 				     GFP_KERNEL, cpu_to_node(i));
265391e43daSPeter Zijlstra 		if (!rt_se)
266391e43daSPeter Zijlstra 			goto err_free_rq;
267391e43daSPeter Zijlstra 
26807c54f7aSAbel Vesa 		init_rt_rq(rt_rq);
269391e43daSPeter Zijlstra 		rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime;
270391e43daSPeter Zijlstra 		init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]);
271391e43daSPeter Zijlstra 	}
272391e43daSPeter Zijlstra 
273391e43daSPeter Zijlstra 	return 1;
274391e43daSPeter Zijlstra 
275391e43daSPeter Zijlstra err_free_rq:
276391e43daSPeter Zijlstra 	kfree(rt_rq);
277391e43daSPeter Zijlstra err:
278391e43daSPeter Zijlstra 	return 0;
279391e43daSPeter Zijlstra }
280391e43daSPeter Zijlstra 
281391e43daSPeter Zijlstra #else /* CONFIG_RT_GROUP_SCHED */
282391e43daSPeter Zijlstra 
283391e43daSPeter Zijlstra #define rt_entity_is_task(rt_se) (1)
284391e43daSPeter Zijlstra 
285391e43daSPeter Zijlstra static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)
286391e43daSPeter Zijlstra {
287391e43daSPeter Zijlstra 	return container_of(rt_se, struct task_struct, rt);
288391e43daSPeter Zijlstra }
289391e43daSPeter Zijlstra 
290391e43daSPeter Zijlstra static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)
291391e43daSPeter Zijlstra {
292391e43daSPeter Zijlstra 	return container_of(rt_rq, struct rq, rt);
293391e43daSPeter Zijlstra }
294391e43daSPeter Zijlstra 
295653d07a6SKirill Tkhai static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se)
296391e43daSPeter Zijlstra {
297391e43daSPeter Zijlstra 	struct task_struct *p = rt_task_of(rt_se);
298653d07a6SKirill Tkhai 
299653d07a6SKirill Tkhai 	return task_rq(p);
300653d07a6SKirill Tkhai }
301653d07a6SKirill Tkhai 
302653d07a6SKirill Tkhai static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)
303653d07a6SKirill Tkhai {
304653d07a6SKirill Tkhai 	struct rq *rq = rq_of_rt_se(rt_se);
305391e43daSPeter Zijlstra 
306391e43daSPeter Zijlstra 	return &rq->rt;
307391e43daSPeter Zijlstra }
308391e43daSPeter Zijlstra 
309b027789eSMathias Krause void unregister_rt_sched_group(struct task_group *tg) { }
310b027789eSMathias Krause 
311391e43daSPeter Zijlstra void free_rt_sched_group(struct task_group *tg) { }
312391e43daSPeter Zijlstra 
313391e43daSPeter Zijlstra int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
314391e43daSPeter Zijlstra {
315391e43daSPeter Zijlstra 	return 1;
316391e43daSPeter Zijlstra }
317391e43daSPeter Zijlstra #endif /* CONFIG_RT_GROUP_SCHED */
318391e43daSPeter Zijlstra 
319391e43daSPeter Zijlstra #ifdef CONFIG_SMP
320391e43daSPeter Zijlstra 
321dc877341SPeter Zijlstra static inline bool need_pull_rt_task(struct rq *rq, struct task_struct *prev)
322dc877341SPeter Zijlstra {
323dc877341SPeter Zijlstra 	/* Try to pull RT tasks here if we lower this rq's prio */
324120455c5SPeter Zijlstra 	return rq->online && rq->rt.highest_prio.curr > prev->prio;
325dc877341SPeter Zijlstra }
326dc877341SPeter Zijlstra 
327391e43daSPeter Zijlstra static inline int rt_overloaded(struct rq *rq)
328391e43daSPeter Zijlstra {
329391e43daSPeter Zijlstra 	return atomic_read(&rq->rd->rto_count);
330391e43daSPeter Zijlstra }
331391e43daSPeter Zijlstra 
332391e43daSPeter Zijlstra static inline void rt_set_overload(struct rq *rq)
333391e43daSPeter Zijlstra {
334391e43daSPeter Zijlstra 	if (!rq->online)
335391e43daSPeter Zijlstra 		return;
336391e43daSPeter Zijlstra 
337391e43daSPeter Zijlstra 	cpumask_set_cpu(rq->cpu, rq->rd->rto_mask);
338391e43daSPeter Zijlstra 	/*
339391e43daSPeter Zijlstra 	 * Make sure the mask is visible before we set
340391e43daSPeter Zijlstra 	 * the overload count. That is checked to determine
341391e43daSPeter Zijlstra 	 * if we should look at the mask. It would be a shame
342391e43daSPeter Zijlstra 	 * if we looked at the mask, but the mask was not
343391e43daSPeter Zijlstra 	 * updated yet.
3447c3f2ab7SPeter Zijlstra 	 *
3457c3f2ab7SPeter Zijlstra 	 * Matched by the barrier in pull_rt_task().
346391e43daSPeter Zijlstra 	 */
3477c3f2ab7SPeter Zijlstra 	smp_wmb();
348391e43daSPeter Zijlstra 	atomic_inc(&rq->rd->rto_count);
349391e43daSPeter Zijlstra }
350391e43daSPeter Zijlstra 
351391e43daSPeter Zijlstra static inline void rt_clear_overload(struct rq *rq)
352391e43daSPeter Zijlstra {
353391e43daSPeter Zijlstra 	if (!rq->online)
354391e43daSPeter Zijlstra 		return;
355391e43daSPeter Zijlstra 
356391e43daSPeter Zijlstra 	/* the order here really doesn't matter */
357391e43daSPeter Zijlstra 	atomic_dec(&rq->rd->rto_count);
358391e43daSPeter Zijlstra 	cpumask_clear_cpu(rq->cpu, rq->rd->rto_mask);
359391e43daSPeter Zijlstra }
360391e43daSPeter Zijlstra 
361391e43daSPeter Zijlstra static void update_rt_migration(struct rt_rq *rt_rq)
362391e43daSPeter Zijlstra {
363391e43daSPeter Zijlstra 	if (rt_rq->rt_nr_migratory && rt_rq->rt_nr_total > 1) {
364391e43daSPeter Zijlstra 		if (!rt_rq->overloaded) {
365391e43daSPeter Zijlstra 			rt_set_overload(rq_of_rt_rq(rt_rq));
366391e43daSPeter Zijlstra 			rt_rq->overloaded = 1;
367391e43daSPeter Zijlstra 		}
368391e43daSPeter Zijlstra 	} else if (rt_rq->overloaded) {
369391e43daSPeter Zijlstra 		rt_clear_overload(rq_of_rt_rq(rt_rq));
370391e43daSPeter Zijlstra 		rt_rq->overloaded = 0;
371391e43daSPeter Zijlstra 	}
372391e43daSPeter Zijlstra }
373391e43daSPeter Zijlstra 
374391e43daSPeter Zijlstra static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
375391e43daSPeter Zijlstra {
37629baa747SPeter Zijlstra 	struct task_struct *p;
37729baa747SPeter Zijlstra 
378391e43daSPeter Zijlstra 	if (!rt_entity_is_task(rt_se))
379391e43daSPeter Zijlstra 		return;
380391e43daSPeter Zijlstra 
38129baa747SPeter Zijlstra 	p = rt_task_of(rt_se);
382391e43daSPeter Zijlstra 	rt_rq = &rq_of_rt_rq(rt_rq)->rt;
383391e43daSPeter Zijlstra 
384391e43daSPeter Zijlstra 	rt_rq->rt_nr_total++;
3854b53a341SIngo Molnar 	if (p->nr_cpus_allowed > 1)
386391e43daSPeter Zijlstra 		rt_rq->rt_nr_migratory++;
387391e43daSPeter Zijlstra 
388391e43daSPeter Zijlstra 	update_rt_migration(rt_rq);
389391e43daSPeter Zijlstra }
390391e43daSPeter Zijlstra 
391391e43daSPeter Zijlstra static void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
392391e43daSPeter Zijlstra {
39329baa747SPeter Zijlstra 	struct task_struct *p;
39429baa747SPeter Zijlstra 
395391e43daSPeter Zijlstra 	if (!rt_entity_is_task(rt_se))
396391e43daSPeter Zijlstra 		return;
397391e43daSPeter Zijlstra 
39829baa747SPeter Zijlstra 	p = rt_task_of(rt_se);
399391e43daSPeter Zijlstra 	rt_rq = &rq_of_rt_rq(rt_rq)->rt;
400391e43daSPeter Zijlstra 
401391e43daSPeter Zijlstra 	rt_rq->rt_nr_total--;
4024b53a341SIngo Molnar 	if (p->nr_cpus_allowed > 1)
403391e43daSPeter Zijlstra 		rt_rq->rt_nr_migratory--;
404391e43daSPeter Zijlstra 
405391e43daSPeter Zijlstra 	update_rt_migration(rt_rq);
406391e43daSPeter Zijlstra }
407391e43daSPeter Zijlstra 
408391e43daSPeter Zijlstra static inline int has_pushable_tasks(struct rq *rq)
409391e43daSPeter Zijlstra {
410391e43daSPeter Zijlstra 	return !plist_head_empty(&rq->rt.pushable_tasks);
411391e43daSPeter Zijlstra }
412391e43daSPeter Zijlstra 
4138e5bad7dSKees Cook static DEFINE_PER_CPU(struct balance_callback, rt_push_head);
4148e5bad7dSKees Cook static DEFINE_PER_CPU(struct balance_callback, rt_pull_head);
415e3fca9e7SPeter Zijlstra 
416e3fca9e7SPeter Zijlstra static void push_rt_tasks(struct rq *);
417fd7a4bedSPeter Zijlstra static void pull_rt_task(struct rq *);
418e3fca9e7SPeter Zijlstra 
41902d8ec94SIngo Molnar static inline void rt_queue_push_tasks(struct rq *rq)
420dc877341SPeter Zijlstra {
421e3fca9e7SPeter Zijlstra 	if (!has_pushable_tasks(rq))
422e3fca9e7SPeter Zijlstra 		return;
423e3fca9e7SPeter Zijlstra 
424fd7a4bedSPeter Zijlstra 	queue_balance_callback(rq, &per_cpu(rt_push_head, rq->cpu), push_rt_tasks);
425fd7a4bedSPeter Zijlstra }
426fd7a4bedSPeter Zijlstra 
42702d8ec94SIngo Molnar static inline void rt_queue_pull_task(struct rq *rq)
428fd7a4bedSPeter Zijlstra {
429fd7a4bedSPeter Zijlstra 	queue_balance_callback(rq, &per_cpu(rt_pull_head, rq->cpu), pull_rt_task);
430dc877341SPeter Zijlstra }
431dc877341SPeter Zijlstra 
432391e43daSPeter Zijlstra static void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
433391e43daSPeter Zijlstra {
434391e43daSPeter Zijlstra 	plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);
435391e43daSPeter Zijlstra 	plist_node_init(&p->pushable_tasks, p->prio);
436391e43daSPeter Zijlstra 	plist_add(&p->pushable_tasks, &rq->rt.pushable_tasks);
437391e43daSPeter Zijlstra 
438391e43daSPeter Zijlstra 	/* Update the highest prio pushable task */
439391e43daSPeter Zijlstra 	if (p->prio < rq->rt.highest_prio.next)
440391e43daSPeter Zijlstra 		rq->rt.highest_prio.next = p->prio;
441391e43daSPeter Zijlstra }
442391e43daSPeter Zijlstra 
443391e43daSPeter Zijlstra static void dequeue_pushable_task(struct rq *rq, struct task_struct *p)
444391e43daSPeter Zijlstra {
445391e43daSPeter Zijlstra 	plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);
446391e43daSPeter Zijlstra 
447391e43daSPeter Zijlstra 	/* Update the new highest prio pushable task */
448391e43daSPeter Zijlstra 	if (has_pushable_tasks(rq)) {
449391e43daSPeter Zijlstra 		p = plist_first_entry(&rq->rt.pushable_tasks,
450391e43daSPeter Zijlstra 				      struct task_struct, pushable_tasks);
451391e43daSPeter Zijlstra 		rq->rt.highest_prio.next = p->prio;
452934fc331SPeter Zijlstra 	} else {
453934fc331SPeter Zijlstra 		rq->rt.highest_prio.next = MAX_RT_PRIO-1;
454934fc331SPeter Zijlstra 	}
455391e43daSPeter Zijlstra }
456391e43daSPeter Zijlstra 
457391e43daSPeter Zijlstra #else
458391e43daSPeter Zijlstra 
459391e43daSPeter Zijlstra static inline void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
460391e43daSPeter Zijlstra {
461391e43daSPeter Zijlstra }
462391e43daSPeter Zijlstra 
463391e43daSPeter Zijlstra static inline void dequeue_pushable_task(struct rq *rq, struct task_struct *p)
464391e43daSPeter Zijlstra {
465391e43daSPeter Zijlstra }
466391e43daSPeter Zijlstra 
467391e43daSPeter Zijlstra static inline
468391e43daSPeter Zijlstra void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
469391e43daSPeter Zijlstra {
470391e43daSPeter Zijlstra }
471391e43daSPeter Zijlstra 
472391e43daSPeter Zijlstra static inline
473391e43daSPeter Zijlstra void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
474391e43daSPeter Zijlstra {
475391e43daSPeter Zijlstra }
476391e43daSPeter Zijlstra 
47702d8ec94SIngo Molnar static inline void rt_queue_push_tasks(struct rq *rq)
478dc877341SPeter Zijlstra {
479dc877341SPeter Zijlstra }
480391e43daSPeter Zijlstra #endif /* CONFIG_SMP */
481391e43daSPeter Zijlstra 
482f4ebcbc0SKirill Tkhai static void enqueue_top_rt_rq(struct rt_rq *rt_rq);
4835c66d1b9SNicolas Saenz Julienne static void dequeue_top_rt_rq(struct rt_rq *rt_rq, unsigned int count);
484f4ebcbc0SKirill Tkhai 
485391e43daSPeter Zijlstra static inline int on_rt_rq(struct sched_rt_entity *rt_se)
486391e43daSPeter Zijlstra {
487ff77e468SPeter Zijlstra 	return rt_se->on_rq;
488391e43daSPeter Zijlstra }
489391e43daSPeter Zijlstra 
490804d402fSQais Yousef #ifdef CONFIG_UCLAMP_TASK
491804d402fSQais Yousef /*
492804d402fSQais Yousef  * Verify the fitness of task @p to run on @cpu taking into account the uclamp
493804d402fSQais Yousef  * settings.
494804d402fSQais Yousef  *
495804d402fSQais Yousef  * This check is only important for heterogeneous systems where uclamp_min value
496804d402fSQais Yousef  * is higher than the capacity of a @cpu. For non-heterogeneous system this
497804d402fSQais Yousef  * function will always return true.
498804d402fSQais Yousef  *
499804d402fSQais Yousef  * The function will return true if the capacity of the @cpu is >= the
500804d402fSQais Yousef  * uclamp_min and false otherwise.
501804d402fSQais Yousef  *
502804d402fSQais Yousef  * Note that uclamp_min will be clamped to uclamp_max if uclamp_min
503804d402fSQais Yousef  * > uclamp_max.
504804d402fSQais Yousef  */
505804d402fSQais Yousef static inline bool rt_task_fits_capacity(struct task_struct *p, int cpu)
506804d402fSQais Yousef {
507804d402fSQais Yousef 	unsigned int min_cap;
508804d402fSQais Yousef 	unsigned int max_cap;
509804d402fSQais Yousef 	unsigned int cpu_cap;
510804d402fSQais Yousef 
511804d402fSQais Yousef 	/* Only heterogeneous systems can benefit from this check */
512740cf8a7SDietmar Eggemann 	if (!sched_asym_cpucap_active())
513804d402fSQais Yousef 		return true;
514804d402fSQais Yousef 
515804d402fSQais Yousef 	min_cap = uclamp_eff_value(p, UCLAMP_MIN);
516804d402fSQais Yousef 	max_cap = uclamp_eff_value(p, UCLAMP_MAX);
517804d402fSQais Yousef 
518804d402fSQais Yousef 	cpu_cap = capacity_orig_of(cpu);
519804d402fSQais Yousef 
520804d402fSQais Yousef 	return cpu_cap >= min(min_cap, max_cap);
521804d402fSQais Yousef }
522804d402fSQais Yousef #else
523804d402fSQais Yousef static inline bool rt_task_fits_capacity(struct task_struct *p, int cpu)
524804d402fSQais Yousef {
525804d402fSQais Yousef 	return true;
526804d402fSQais Yousef }
527804d402fSQais Yousef #endif
528804d402fSQais Yousef 
529391e43daSPeter Zijlstra #ifdef CONFIG_RT_GROUP_SCHED
530391e43daSPeter Zijlstra 
531391e43daSPeter Zijlstra static inline u64 sched_rt_runtime(struct rt_rq *rt_rq)
532391e43daSPeter Zijlstra {
533391e43daSPeter Zijlstra 	if (!rt_rq->tg)
534391e43daSPeter Zijlstra 		return RUNTIME_INF;
535391e43daSPeter Zijlstra 
536391e43daSPeter Zijlstra 	return rt_rq->rt_runtime;
537391e43daSPeter Zijlstra }
538391e43daSPeter Zijlstra 
539391e43daSPeter Zijlstra static inline u64 sched_rt_period(struct rt_rq *rt_rq)
540391e43daSPeter Zijlstra {
541391e43daSPeter Zijlstra 	return ktime_to_ns(rt_rq->tg->rt_bandwidth.rt_period);
542391e43daSPeter Zijlstra }
543391e43daSPeter Zijlstra 
544391e43daSPeter Zijlstra typedef struct task_group *rt_rq_iter_t;
545391e43daSPeter Zijlstra 
546391e43daSPeter Zijlstra static inline struct task_group *next_task_group(struct task_group *tg)
547391e43daSPeter Zijlstra {
548391e43daSPeter Zijlstra 	do {
549391e43daSPeter Zijlstra 		tg = list_entry_rcu(tg->list.next,
550391e43daSPeter Zijlstra 			typeof(struct task_group), list);
551391e43daSPeter Zijlstra 	} while (&tg->list != &task_groups && task_group_is_autogroup(tg));
552391e43daSPeter Zijlstra 
553391e43daSPeter Zijlstra 	if (&tg->list == &task_groups)
554391e43daSPeter Zijlstra 		tg = NULL;
555391e43daSPeter Zijlstra 
556391e43daSPeter Zijlstra 	return tg;
557391e43daSPeter Zijlstra }
558391e43daSPeter Zijlstra 
559391e43daSPeter Zijlstra #define for_each_rt_rq(rt_rq, iter, rq)					\
560391e43daSPeter Zijlstra 	for (iter = container_of(&task_groups, typeof(*iter), list);	\
561391e43daSPeter Zijlstra 		(iter = next_task_group(iter)) &&			\
562391e43daSPeter Zijlstra 		(rt_rq = iter->rt_rq[cpu_of(rq)]);)
563391e43daSPeter Zijlstra 
564391e43daSPeter Zijlstra #define for_each_sched_rt_entity(rt_se) \
565391e43daSPeter Zijlstra 	for (; rt_se; rt_se = rt_se->parent)
566391e43daSPeter Zijlstra 
567391e43daSPeter Zijlstra static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se)
568391e43daSPeter Zijlstra {
569391e43daSPeter Zijlstra 	return rt_se->my_q;
570391e43daSPeter Zijlstra }
571391e43daSPeter Zijlstra 
572ff77e468SPeter Zijlstra static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags);
573ff77e468SPeter Zijlstra static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags);
574391e43daSPeter Zijlstra 
575391e43daSPeter Zijlstra static void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
576391e43daSPeter Zijlstra {
577391e43daSPeter Zijlstra 	struct task_struct *curr = rq_of_rt_rq(rt_rq)->curr;
5788875125eSKirill Tkhai 	struct rq *rq = rq_of_rt_rq(rt_rq);
579391e43daSPeter Zijlstra 	struct sched_rt_entity *rt_se;
580391e43daSPeter Zijlstra 
5818875125eSKirill Tkhai 	int cpu = cpu_of(rq);
582391e43daSPeter Zijlstra 
583391e43daSPeter Zijlstra 	rt_se = rt_rq->tg->rt_se[cpu];
584391e43daSPeter Zijlstra 
585391e43daSPeter Zijlstra 	if (rt_rq->rt_nr_running) {
586f4ebcbc0SKirill Tkhai 		if (!rt_se)
587f4ebcbc0SKirill Tkhai 			enqueue_top_rt_rq(rt_rq);
588f4ebcbc0SKirill Tkhai 		else if (!on_rt_rq(rt_se))
589ff77e468SPeter Zijlstra 			enqueue_rt_entity(rt_se, 0);
590f4ebcbc0SKirill Tkhai 
591391e43daSPeter Zijlstra 		if (rt_rq->highest_prio.curr < curr->prio)
5928875125eSKirill Tkhai 			resched_curr(rq);
593391e43daSPeter Zijlstra 	}
594391e43daSPeter Zijlstra }
595391e43daSPeter Zijlstra 
596391e43daSPeter Zijlstra static void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
597391e43daSPeter Zijlstra {
598391e43daSPeter Zijlstra 	struct sched_rt_entity *rt_se;
599391e43daSPeter Zijlstra 	int cpu = cpu_of(rq_of_rt_rq(rt_rq));
600391e43daSPeter Zijlstra 
601391e43daSPeter Zijlstra 	rt_se = rt_rq->tg->rt_se[cpu];
602391e43daSPeter Zijlstra 
603296b2ffeSVincent Guittot 	if (!rt_se) {
6045c66d1b9SNicolas Saenz Julienne 		dequeue_top_rt_rq(rt_rq, rt_rq->rt_nr_running);
605296b2ffeSVincent Guittot 		/* Kick cpufreq (see the comment in kernel/sched/sched.h). */
606296b2ffeSVincent Guittot 		cpufreq_update_util(rq_of_rt_rq(rt_rq), 0);
607296b2ffeSVincent Guittot 	}
608f4ebcbc0SKirill Tkhai 	else if (on_rt_rq(rt_se))
609ff77e468SPeter Zijlstra 		dequeue_rt_entity(rt_se, 0);
610391e43daSPeter Zijlstra }
611391e43daSPeter Zijlstra 
61246383648SKirill Tkhai static inline int rt_rq_throttled(struct rt_rq *rt_rq)
61346383648SKirill Tkhai {
61446383648SKirill Tkhai 	return rt_rq->rt_throttled && !rt_rq->rt_nr_boosted;
61546383648SKirill Tkhai }
61646383648SKirill Tkhai 
617391e43daSPeter Zijlstra static int rt_se_boosted(struct sched_rt_entity *rt_se)
618391e43daSPeter Zijlstra {
619391e43daSPeter Zijlstra 	struct rt_rq *rt_rq = group_rt_rq(rt_se);
620391e43daSPeter Zijlstra 	struct task_struct *p;
621391e43daSPeter Zijlstra 
622391e43daSPeter Zijlstra 	if (rt_rq)
623391e43daSPeter Zijlstra 		return !!rt_rq->rt_nr_boosted;
624391e43daSPeter Zijlstra 
625391e43daSPeter Zijlstra 	p = rt_task_of(rt_se);
626391e43daSPeter Zijlstra 	return p->prio != p->normal_prio;
627391e43daSPeter Zijlstra }
628391e43daSPeter Zijlstra 
629391e43daSPeter Zijlstra #ifdef CONFIG_SMP
630391e43daSPeter Zijlstra static inline const struct cpumask *sched_rt_period_mask(void)
631391e43daSPeter Zijlstra {
632424c93feSNathan Zimmer 	return this_rq()->rd->span;
633391e43daSPeter Zijlstra }
634391e43daSPeter Zijlstra #else
635391e43daSPeter Zijlstra static inline const struct cpumask *sched_rt_period_mask(void)
636391e43daSPeter Zijlstra {
637391e43daSPeter Zijlstra 	return cpu_online_mask;
638391e43daSPeter Zijlstra }
639391e43daSPeter Zijlstra #endif
640391e43daSPeter Zijlstra 
641391e43daSPeter Zijlstra static inline
642391e43daSPeter Zijlstra struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu)
643391e43daSPeter Zijlstra {
644391e43daSPeter Zijlstra 	return container_of(rt_b, struct task_group, rt_bandwidth)->rt_rq[cpu];
645391e43daSPeter Zijlstra }
646391e43daSPeter Zijlstra 
647391e43daSPeter Zijlstra static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq)
648391e43daSPeter Zijlstra {
649391e43daSPeter Zijlstra 	return &rt_rq->tg->rt_bandwidth;
650391e43daSPeter Zijlstra }
651391e43daSPeter Zijlstra 
652391e43daSPeter Zijlstra #else /* !CONFIG_RT_GROUP_SCHED */
653391e43daSPeter Zijlstra 
654391e43daSPeter Zijlstra static inline u64 sched_rt_runtime(struct rt_rq *rt_rq)
655391e43daSPeter Zijlstra {
656391e43daSPeter Zijlstra 	return rt_rq->rt_runtime;
657391e43daSPeter Zijlstra }
658391e43daSPeter Zijlstra 
659391e43daSPeter Zijlstra static inline u64 sched_rt_period(struct rt_rq *rt_rq)
660391e43daSPeter Zijlstra {
661391e43daSPeter Zijlstra 	return ktime_to_ns(def_rt_bandwidth.rt_period);
662391e43daSPeter Zijlstra }
663391e43daSPeter Zijlstra 
664391e43daSPeter Zijlstra typedef struct rt_rq *rt_rq_iter_t;
665391e43daSPeter Zijlstra 
666391e43daSPeter Zijlstra #define for_each_rt_rq(rt_rq, iter, rq) \
667391e43daSPeter Zijlstra 	for ((void) iter, rt_rq = &rq->rt; rt_rq; rt_rq = NULL)
668391e43daSPeter Zijlstra 
669391e43daSPeter Zijlstra #define for_each_sched_rt_entity(rt_se) \
670391e43daSPeter Zijlstra 	for (; rt_se; rt_se = NULL)
671391e43daSPeter Zijlstra 
672391e43daSPeter Zijlstra static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se)
673391e43daSPeter Zijlstra {
674391e43daSPeter Zijlstra 	return NULL;
675391e43daSPeter Zijlstra }
676391e43daSPeter Zijlstra 
677391e43daSPeter Zijlstra static inline void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
678391e43daSPeter Zijlstra {
679f4ebcbc0SKirill Tkhai 	struct rq *rq = rq_of_rt_rq(rt_rq);
680f4ebcbc0SKirill Tkhai 
681f4ebcbc0SKirill Tkhai 	if (!rt_rq->rt_nr_running)
682f4ebcbc0SKirill Tkhai 		return;
683f4ebcbc0SKirill Tkhai 
684f4ebcbc0SKirill Tkhai 	enqueue_top_rt_rq(rt_rq);
6858875125eSKirill Tkhai 	resched_curr(rq);
686391e43daSPeter Zijlstra }
687391e43daSPeter Zijlstra 
688391e43daSPeter Zijlstra static inline void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
689391e43daSPeter Zijlstra {
6905c66d1b9SNicolas Saenz Julienne 	dequeue_top_rt_rq(rt_rq, rt_rq->rt_nr_running);
691391e43daSPeter Zijlstra }
692391e43daSPeter Zijlstra 
69346383648SKirill Tkhai static inline int rt_rq_throttled(struct rt_rq *rt_rq)
69446383648SKirill Tkhai {
69546383648SKirill Tkhai 	return rt_rq->rt_throttled;
69646383648SKirill Tkhai }
69746383648SKirill Tkhai 
698391e43daSPeter Zijlstra static inline const struct cpumask *sched_rt_period_mask(void)
699391e43daSPeter Zijlstra {
700391e43daSPeter Zijlstra 	return cpu_online_mask;
701391e43daSPeter Zijlstra }
702391e43daSPeter Zijlstra 
703391e43daSPeter Zijlstra static inline
704391e43daSPeter Zijlstra struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu)
705391e43daSPeter Zijlstra {
706391e43daSPeter Zijlstra 	return &cpu_rq(cpu)->rt;
707391e43daSPeter Zijlstra }
708391e43daSPeter Zijlstra 
709391e43daSPeter Zijlstra static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq)
710391e43daSPeter Zijlstra {
711391e43daSPeter Zijlstra 	return &def_rt_bandwidth;
712391e43daSPeter Zijlstra }
713391e43daSPeter Zijlstra 
714391e43daSPeter Zijlstra #endif /* CONFIG_RT_GROUP_SCHED */
715391e43daSPeter Zijlstra 
716faa59937SJuri Lelli bool sched_rt_bandwidth_account(struct rt_rq *rt_rq)
717faa59937SJuri Lelli {
718faa59937SJuri Lelli 	struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
719faa59937SJuri Lelli 
720faa59937SJuri Lelli 	return (hrtimer_active(&rt_b->rt_period_timer) ||
721faa59937SJuri Lelli 		rt_rq->rt_time < rt_b->rt_runtime);
722faa59937SJuri Lelli }
723faa59937SJuri Lelli 
724391e43daSPeter Zijlstra #ifdef CONFIG_SMP
725391e43daSPeter Zijlstra /*
726391e43daSPeter Zijlstra  * We ran out of runtime, see if we can borrow some from our neighbours.
727391e43daSPeter Zijlstra  */
728269b26a5SJuri Lelli static void do_balance_runtime(struct rt_rq *rt_rq)
729391e43daSPeter Zijlstra {
730391e43daSPeter Zijlstra 	struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
731aa7f6730SShawn Bohrer 	struct root_domain *rd = rq_of_rt_rq(rt_rq)->rd;
732269b26a5SJuri Lelli 	int i, weight;
733391e43daSPeter Zijlstra 	u64 rt_period;
734391e43daSPeter Zijlstra 
735391e43daSPeter Zijlstra 	weight = cpumask_weight(rd->span);
736391e43daSPeter Zijlstra 
737391e43daSPeter Zijlstra 	raw_spin_lock(&rt_b->rt_runtime_lock);
738391e43daSPeter Zijlstra 	rt_period = ktime_to_ns(rt_b->rt_period);
739391e43daSPeter Zijlstra 	for_each_cpu(i, rd->span) {
740391e43daSPeter Zijlstra 		struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);
741391e43daSPeter Zijlstra 		s64 diff;
742391e43daSPeter Zijlstra 
743391e43daSPeter Zijlstra 		if (iter == rt_rq)
744391e43daSPeter Zijlstra 			continue;
745391e43daSPeter Zijlstra 
746391e43daSPeter Zijlstra 		raw_spin_lock(&iter->rt_runtime_lock);
747391e43daSPeter Zijlstra 		/*
748391e43daSPeter Zijlstra 		 * Either all rqs have inf runtime and there's nothing to steal
749391e43daSPeter Zijlstra 		 * or __disable_runtime() below sets a specific rq to inf to
7503b03706fSIngo Molnar 		 * indicate its been disabled and disallow stealing.
751391e43daSPeter Zijlstra 		 */
752391e43daSPeter Zijlstra 		if (iter->rt_runtime == RUNTIME_INF)
753391e43daSPeter Zijlstra 			goto next;
754391e43daSPeter Zijlstra 
755391e43daSPeter Zijlstra 		/*
756391e43daSPeter Zijlstra 		 * From runqueues with spare time, take 1/n part of their
757391e43daSPeter Zijlstra 		 * spare time, but no more than our period.
758391e43daSPeter Zijlstra 		 */
759391e43daSPeter Zijlstra 		diff = iter->rt_runtime - iter->rt_time;
760391e43daSPeter Zijlstra 		if (diff > 0) {
761391e43daSPeter Zijlstra 			diff = div_u64((u64)diff, weight);
762391e43daSPeter Zijlstra 			if (rt_rq->rt_runtime + diff > rt_period)
763391e43daSPeter Zijlstra 				diff = rt_period - rt_rq->rt_runtime;
764391e43daSPeter Zijlstra 			iter->rt_runtime -= diff;
765391e43daSPeter Zijlstra 			rt_rq->rt_runtime += diff;
766391e43daSPeter Zijlstra 			if (rt_rq->rt_runtime == rt_period) {
767391e43daSPeter Zijlstra 				raw_spin_unlock(&iter->rt_runtime_lock);
768391e43daSPeter Zijlstra 				break;
769391e43daSPeter Zijlstra 			}
770391e43daSPeter Zijlstra 		}
771391e43daSPeter Zijlstra next:
772391e43daSPeter Zijlstra 		raw_spin_unlock(&iter->rt_runtime_lock);
773391e43daSPeter Zijlstra 	}
774391e43daSPeter Zijlstra 	raw_spin_unlock(&rt_b->rt_runtime_lock);
775391e43daSPeter Zijlstra }
776391e43daSPeter Zijlstra 
777391e43daSPeter Zijlstra /*
778391e43daSPeter Zijlstra  * Ensure this RQ takes back all the runtime it lend to its neighbours.
779391e43daSPeter Zijlstra  */
780391e43daSPeter Zijlstra static void __disable_runtime(struct rq *rq)
781391e43daSPeter Zijlstra {
782391e43daSPeter Zijlstra 	struct root_domain *rd = rq->rd;
783391e43daSPeter Zijlstra 	rt_rq_iter_t iter;
784391e43daSPeter Zijlstra 	struct rt_rq *rt_rq;
785391e43daSPeter Zijlstra 
786391e43daSPeter Zijlstra 	if (unlikely(!scheduler_running))
787391e43daSPeter Zijlstra 		return;
788391e43daSPeter Zijlstra 
789391e43daSPeter Zijlstra 	for_each_rt_rq(rt_rq, iter, rq) {
790391e43daSPeter Zijlstra 		struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
791391e43daSPeter Zijlstra 		s64 want;
792391e43daSPeter Zijlstra 		int i;
793391e43daSPeter Zijlstra 
794391e43daSPeter Zijlstra 		raw_spin_lock(&rt_b->rt_runtime_lock);
795391e43daSPeter Zijlstra 		raw_spin_lock(&rt_rq->rt_runtime_lock);
796391e43daSPeter Zijlstra 		/*
797391e43daSPeter Zijlstra 		 * Either we're all inf and nobody needs to borrow, or we're
798391e43daSPeter Zijlstra 		 * already disabled and thus have nothing to do, or we have
799391e43daSPeter Zijlstra 		 * exactly the right amount of runtime to take out.
800391e43daSPeter Zijlstra 		 */
801391e43daSPeter Zijlstra 		if (rt_rq->rt_runtime == RUNTIME_INF ||
802391e43daSPeter Zijlstra 				rt_rq->rt_runtime == rt_b->rt_runtime)
803391e43daSPeter Zijlstra 			goto balanced;
804391e43daSPeter Zijlstra 		raw_spin_unlock(&rt_rq->rt_runtime_lock);
805391e43daSPeter Zijlstra 
806391e43daSPeter Zijlstra 		/*
807391e43daSPeter Zijlstra 		 * Calculate the difference between what we started out with
808391e43daSPeter Zijlstra 		 * and what we current have, that's the amount of runtime
809391e43daSPeter Zijlstra 		 * we lend and now have to reclaim.
810391e43daSPeter Zijlstra 		 */
811391e43daSPeter Zijlstra 		want = rt_b->rt_runtime - rt_rq->rt_runtime;
812391e43daSPeter Zijlstra 
813391e43daSPeter Zijlstra 		/*
814391e43daSPeter Zijlstra 		 * Greedy reclaim, take back as much as we can.
815391e43daSPeter Zijlstra 		 */
816391e43daSPeter Zijlstra 		for_each_cpu(i, rd->span) {
817391e43daSPeter Zijlstra 			struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);
818391e43daSPeter Zijlstra 			s64 diff;
819391e43daSPeter Zijlstra 
820391e43daSPeter Zijlstra 			/*
821391e43daSPeter Zijlstra 			 * Can't reclaim from ourselves or disabled runqueues.
822391e43daSPeter Zijlstra 			 */
823391e43daSPeter Zijlstra 			if (iter == rt_rq || iter->rt_runtime == RUNTIME_INF)
824391e43daSPeter Zijlstra 				continue;
825391e43daSPeter Zijlstra 
826391e43daSPeter Zijlstra 			raw_spin_lock(&iter->rt_runtime_lock);
827391e43daSPeter Zijlstra 			if (want > 0) {
828391e43daSPeter Zijlstra 				diff = min_t(s64, iter->rt_runtime, want);
829391e43daSPeter Zijlstra 				iter->rt_runtime -= diff;
830391e43daSPeter Zijlstra 				want -= diff;
831391e43daSPeter Zijlstra 			} else {
832391e43daSPeter Zijlstra 				iter->rt_runtime -= want;
833391e43daSPeter Zijlstra 				want -= want;
834391e43daSPeter Zijlstra 			}
835391e43daSPeter Zijlstra 			raw_spin_unlock(&iter->rt_runtime_lock);
836391e43daSPeter Zijlstra 
837391e43daSPeter Zijlstra 			if (!want)
838391e43daSPeter Zijlstra 				break;
839391e43daSPeter Zijlstra 		}
840391e43daSPeter Zijlstra 
841391e43daSPeter Zijlstra 		raw_spin_lock(&rt_rq->rt_runtime_lock);
842391e43daSPeter Zijlstra 		/*
843391e43daSPeter Zijlstra 		 * We cannot be left wanting - that would mean some runtime
844391e43daSPeter Zijlstra 		 * leaked out of the system.
845391e43daSPeter Zijlstra 		 */
84609348d75SIngo Molnar 		WARN_ON_ONCE(want);
847391e43daSPeter Zijlstra balanced:
848391e43daSPeter Zijlstra 		/*
849391e43daSPeter Zijlstra 		 * Disable all the borrow logic by pretending we have inf
850391e43daSPeter Zijlstra 		 * runtime - in which case borrowing doesn't make sense.
851391e43daSPeter Zijlstra 		 */
852391e43daSPeter Zijlstra 		rt_rq->rt_runtime = RUNTIME_INF;
853a4c96ae3SPeter Boonstoppel 		rt_rq->rt_throttled = 0;
854391e43daSPeter Zijlstra 		raw_spin_unlock(&rt_rq->rt_runtime_lock);
855391e43daSPeter Zijlstra 		raw_spin_unlock(&rt_b->rt_runtime_lock);
85699b62567SKirill Tkhai 
85799b62567SKirill Tkhai 		/* Make rt_rq available for pick_next_task() */
85899b62567SKirill Tkhai 		sched_rt_rq_enqueue(rt_rq);
859391e43daSPeter Zijlstra 	}
860391e43daSPeter Zijlstra }
861391e43daSPeter Zijlstra 
862391e43daSPeter Zijlstra static void __enable_runtime(struct rq *rq)
863391e43daSPeter Zijlstra {
864391e43daSPeter Zijlstra 	rt_rq_iter_t iter;
865391e43daSPeter Zijlstra 	struct rt_rq *rt_rq;
866391e43daSPeter Zijlstra 
867391e43daSPeter Zijlstra 	if (unlikely(!scheduler_running))
868391e43daSPeter Zijlstra 		return;
869391e43daSPeter Zijlstra 
870391e43daSPeter Zijlstra 	/*
871391e43daSPeter Zijlstra 	 * Reset each runqueue's bandwidth settings
872391e43daSPeter Zijlstra 	 */
873391e43daSPeter Zijlstra 	for_each_rt_rq(rt_rq, iter, rq) {
874391e43daSPeter Zijlstra 		struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
875391e43daSPeter Zijlstra 
876391e43daSPeter Zijlstra 		raw_spin_lock(&rt_b->rt_runtime_lock);
877391e43daSPeter Zijlstra 		raw_spin_lock(&rt_rq->rt_runtime_lock);
878391e43daSPeter Zijlstra 		rt_rq->rt_runtime = rt_b->rt_runtime;
879391e43daSPeter Zijlstra 		rt_rq->rt_time = 0;
880391e43daSPeter Zijlstra 		rt_rq->rt_throttled = 0;
881391e43daSPeter Zijlstra 		raw_spin_unlock(&rt_rq->rt_runtime_lock);
882391e43daSPeter Zijlstra 		raw_spin_unlock(&rt_b->rt_runtime_lock);
883391e43daSPeter Zijlstra 	}
884391e43daSPeter Zijlstra }
885391e43daSPeter Zijlstra 
886269b26a5SJuri Lelli static void balance_runtime(struct rt_rq *rt_rq)
887391e43daSPeter Zijlstra {
888391e43daSPeter Zijlstra 	if (!sched_feat(RT_RUNTIME_SHARE))
889269b26a5SJuri Lelli 		return;
890391e43daSPeter Zijlstra 
891391e43daSPeter Zijlstra 	if (rt_rq->rt_time > rt_rq->rt_runtime) {
892391e43daSPeter Zijlstra 		raw_spin_unlock(&rt_rq->rt_runtime_lock);
893269b26a5SJuri Lelli 		do_balance_runtime(rt_rq);
894391e43daSPeter Zijlstra 		raw_spin_lock(&rt_rq->rt_runtime_lock);
895391e43daSPeter Zijlstra 	}
896391e43daSPeter Zijlstra }
897391e43daSPeter Zijlstra #else /* !CONFIG_SMP */
898269b26a5SJuri Lelli static inline void balance_runtime(struct rt_rq *rt_rq) {}
899391e43daSPeter Zijlstra #endif /* CONFIG_SMP */
900391e43daSPeter Zijlstra 
901391e43daSPeter Zijlstra static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun)
902391e43daSPeter Zijlstra {
90342c62a58SPeter Zijlstra 	int i, idle = 1, throttled = 0;
904391e43daSPeter Zijlstra 	const struct cpumask *span;
905391e43daSPeter Zijlstra 
906391e43daSPeter Zijlstra 	span = sched_rt_period_mask();
907e221d028SMike Galbraith #ifdef CONFIG_RT_GROUP_SCHED
908e221d028SMike Galbraith 	/*
909e221d028SMike Galbraith 	 * FIXME: isolated CPUs should really leave the root task group,
910e221d028SMike Galbraith 	 * whether they are isolcpus or were isolated via cpusets, lest
911e221d028SMike Galbraith 	 * the timer run on a CPU which does not service all runqueues,
912e221d028SMike Galbraith 	 * potentially leaving other CPUs indefinitely throttled.  If
913e221d028SMike Galbraith 	 * isolation is really required, the user will turn the throttle
914e221d028SMike Galbraith 	 * off to kill the perturbations it causes anyway.  Meanwhile,
915e221d028SMike Galbraith 	 * this maintains functionality for boot and/or troubleshooting.
916e221d028SMike Galbraith 	 */
917e221d028SMike Galbraith 	if (rt_b == &root_task_group.rt_bandwidth)
918e221d028SMike Galbraith 		span = cpu_online_mask;
919e221d028SMike Galbraith #endif
920391e43daSPeter Zijlstra 	for_each_cpu(i, span) {
921391e43daSPeter Zijlstra 		int enqueue = 0;
922391e43daSPeter Zijlstra 		struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i);
923391e43daSPeter Zijlstra 		struct rq *rq = rq_of_rt_rq(rt_rq);
9242679a837SHao Jia 		struct rq_flags rf;
925c249f255SDave Kleikamp 		int skip;
926c249f255SDave Kleikamp 
927c249f255SDave Kleikamp 		/*
928c249f255SDave Kleikamp 		 * When span == cpu_online_mask, taking each rq->lock
929c249f255SDave Kleikamp 		 * can be time-consuming. Try to avoid it when possible.
930c249f255SDave Kleikamp 		 */
931c249f255SDave Kleikamp 		raw_spin_lock(&rt_rq->rt_runtime_lock);
932f3d133eeSHailong Liu 		if (!sched_feat(RT_RUNTIME_SHARE) && rt_rq->rt_runtime != RUNTIME_INF)
933f3d133eeSHailong Liu 			rt_rq->rt_runtime = rt_b->rt_runtime;
934c249f255SDave Kleikamp 		skip = !rt_rq->rt_time && !rt_rq->rt_nr_running;
935c249f255SDave Kleikamp 		raw_spin_unlock(&rt_rq->rt_runtime_lock);
936c249f255SDave Kleikamp 		if (skip)
937c249f255SDave Kleikamp 			continue;
938391e43daSPeter Zijlstra 
9392679a837SHao Jia 		rq_lock(rq, &rf);
940d29a2064SDavidlohr Bueso 		update_rq_clock(rq);
941d29a2064SDavidlohr Bueso 
942391e43daSPeter Zijlstra 		if (rt_rq->rt_time) {
943391e43daSPeter Zijlstra 			u64 runtime;
944391e43daSPeter Zijlstra 
945391e43daSPeter Zijlstra 			raw_spin_lock(&rt_rq->rt_runtime_lock);
946391e43daSPeter Zijlstra 			if (rt_rq->rt_throttled)
947391e43daSPeter Zijlstra 				balance_runtime(rt_rq);
948391e43daSPeter Zijlstra 			runtime = rt_rq->rt_runtime;
949391e43daSPeter Zijlstra 			rt_rq->rt_time -= min(rt_rq->rt_time, overrun*runtime);
950391e43daSPeter Zijlstra 			if (rt_rq->rt_throttled && rt_rq->rt_time < runtime) {
951391e43daSPeter Zijlstra 				rt_rq->rt_throttled = 0;
952391e43daSPeter Zijlstra 				enqueue = 1;
953391e43daSPeter Zijlstra 
954391e43daSPeter Zijlstra 				/*
9559edfbfedSPeter Zijlstra 				 * When we're idle and a woken (rt) task is
9569edfbfedSPeter Zijlstra 				 * throttled check_preempt_curr() will set
9579edfbfedSPeter Zijlstra 				 * skip_update and the time between the wakeup
9589edfbfedSPeter Zijlstra 				 * and this unthrottle will get accounted as
9599edfbfedSPeter Zijlstra 				 * 'runtime'.
960391e43daSPeter Zijlstra 				 */
961391e43daSPeter Zijlstra 				if (rt_rq->rt_nr_running && rq->curr == rq->idle)
962adcc8da8SDavidlohr Bueso 					rq_clock_cancel_skipupdate(rq);
963391e43daSPeter Zijlstra 			}
964391e43daSPeter Zijlstra 			if (rt_rq->rt_time || rt_rq->rt_nr_running)
965391e43daSPeter Zijlstra 				idle = 0;
966391e43daSPeter Zijlstra 			raw_spin_unlock(&rt_rq->rt_runtime_lock);
967391e43daSPeter Zijlstra 		} else if (rt_rq->rt_nr_running) {
968391e43daSPeter Zijlstra 			idle = 0;
969391e43daSPeter Zijlstra 			if (!rt_rq_throttled(rt_rq))
970391e43daSPeter Zijlstra 				enqueue = 1;
971391e43daSPeter Zijlstra 		}
97242c62a58SPeter Zijlstra 		if (rt_rq->rt_throttled)
97342c62a58SPeter Zijlstra 			throttled = 1;
974391e43daSPeter Zijlstra 
975391e43daSPeter Zijlstra 		if (enqueue)
976391e43daSPeter Zijlstra 			sched_rt_rq_enqueue(rt_rq);
9772679a837SHao Jia 		rq_unlock(rq, &rf);
978391e43daSPeter Zijlstra 	}
979391e43daSPeter Zijlstra 
98042c62a58SPeter Zijlstra 	if (!throttled && (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF))
98142c62a58SPeter Zijlstra 		return 1;
98242c62a58SPeter Zijlstra 
983391e43daSPeter Zijlstra 	return idle;
984391e43daSPeter Zijlstra }
985391e43daSPeter Zijlstra 
986391e43daSPeter Zijlstra static inline int rt_se_prio(struct sched_rt_entity *rt_se)
987391e43daSPeter Zijlstra {
988391e43daSPeter Zijlstra #ifdef CONFIG_RT_GROUP_SCHED
989391e43daSPeter Zijlstra 	struct rt_rq *rt_rq = group_rt_rq(rt_se);
990391e43daSPeter Zijlstra 
991391e43daSPeter Zijlstra 	if (rt_rq)
992391e43daSPeter Zijlstra 		return rt_rq->highest_prio.curr;
993391e43daSPeter Zijlstra #endif
994391e43daSPeter Zijlstra 
995391e43daSPeter Zijlstra 	return rt_task_of(rt_se)->prio;
996391e43daSPeter Zijlstra }
997391e43daSPeter Zijlstra 
998391e43daSPeter Zijlstra static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq)
999391e43daSPeter Zijlstra {
1000391e43daSPeter Zijlstra 	u64 runtime = sched_rt_runtime(rt_rq);
1001391e43daSPeter Zijlstra 
1002391e43daSPeter Zijlstra 	if (rt_rq->rt_throttled)
1003391e43daSPeter Zijlstra 		return rt_rq_throttled(rt_rq);
1004391e43daSPeter Zijlstra 
10055b680fd6SShan Hai 	if (runtime >= sched_rt_period(rt_rq))
1006391e43daSPeter Zijlstra 		return 0;
1007391e43daSPeter Zijlstra 
1008391e43daSPeter Zijlstra 	balance_runtime(rt_rq);
1009391e43daSPeter Zijlstra 	runtime = sched_rt_runtime(rt_rq);
1010391e43daSPeter Zijlstra 	if (runtime == RUNTIME_INF)
1011391e43daSPeter Zijlstra 		return 0;
1012391e43daSPeter Zijlstra 
1013391e43daSPeter Zijlstra 	if (rt_rq->rt_time > runtime) {
10147abc63b1SPeter Zijlstra 		struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
10157abc63b1SPeter Zijlstra 
10167abc63b1SPeter Zijlstra 		/*
10177abc63b1SPeter Zijlstra 		 * Don't actually throttle groups that have no runtime assigned
10187abc63b1SPeter Zijlstra 		 * but accrue some time due to boosting.
10197abc63b1SPeter Zijlstra 		 */
10207abc63b1SPeter Zijlstra 		if (likely(rt_b->rt_runtime)) {
1021391e43daSPeter Zijlstra 			rt_rq->rt_throttled = 1;
1022c224815dSJohn Stultz 			printk_deferred_once("sched: RT throttling activated\n");
10237abc63b1SPeter Zijlstra 		} else {
10247abc63b1SPeter Zijlstra 			/*
10257abc63b1SPeter Zijlstra 			 * In case we did anyway, make it go away,
10267abc63b1SPeter Zijlstra 			 * replenishment is a joke, since it will replenish us
10277abc63b1SPeter Zijlstra 			 * with exactly 0 ns.
10287abc63b1SPeter Zijlstra 			 */
10297abc63b1SPeter Zijlstra 			rt_rq->rt_time = 0;
10307abc63b1SPeter Zijlstra 		}
10317abc63b1SPeter Zijlstra 
1032391e43daSPeter Zijlstra 		if (rt_rq_throttled(rt_rq)) {
1033391e43daSPeter Zijlstra 			sched_rt_rq_dequeue(rt_rq);
1034391e43daSPeter Zijlstra 			return 1;
1035391e43daSPeter Zijlstra 		}
1036391e43daSPeter Zijlstra 	}
1037391e43daSPeter Zijlstra 
1038391e43daSPeter Zijlstra 	return 0;
1039391e43daSPeter Zijlstra }
1040391e43daSPeter Zijlstra 
1041391e43daSPeter Zijlstra /*
1042391e43daSPeter Zijlstra  * Update the current task's runtime statistics. Skip current tasks that
1043391e43daSPeter Zijlstra  * are not in our scheduling class.
1044391e43daSPeter Zijlstra  */
1045391e43daSPeter Zijlstra static void update_curr_rt(struct rq *rq)
1046391e43daSPeter Zijlstra {
1047391e43daSPeter Zijlstra 	struct task_struct *curr = rq->curr;
1048391e43daSPeter Zijlstra 	struct sched_rt_entity *rt_se = &curr->rt;
1049391e43daSPeter Zijlstra 	u64 delta_exec;
1050a7711602SWen Yang 	u64 now;
1051391e43daSPeter Zijlstra 
1052391e43daSPeter Zijlstra 	if (curr->sched_class != &rt_sched_class)
1053391e43daSPeter Zijlstra 		return;
1054391e43daSPeter Zijlstra 
1055a7711602SWen Yang 	now = rq_clock_task(rq);
1056e7ad2031SWen Yang 	delta_exec = now - curr->se.exec_start;
1057fc79e240SKirill Tkhai 	if (unlikely((s64)delta_exec <= 0))
1058fc79e240SKirill Tkhai 		return;
1059391e43daSPeter Zijlstra 
1060ceeadb83SYafang Shao 	schedstat_set(curr->stats.exec_max,
1061ceeadb83SYafang Shao 		      max(curr->stats.exec_max, delta_exec));
1062391e43daSPeter Zijlstra 
1063ed7b564cSYafang Shao 	trace_sched_stat_runtime(curr, delta_exec, 0);
1064ed7b564cSYafang Shao 
10655531ecffSShang XiaoJing 	update_current_exec_runtime(curr, now, delta_exec);
1066391e43daSPeter Zijlstra 
1067391e43daSPeter Zijlstra 	if (!rt_bandwidth_enabled())
1068391e43daSPeter Zijlstra 		return;
1069391e43daSPeter Zijlstra 
1070391e43daSPeter Zijlstra 	for_each_sched_rt_entity(rt_se) {
10710b07939cSGiedrius Rekasius 		struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
10729b58e976SLi Hua 		int exceeded;
1073391e43daSPeter Zijlstra 
1074391e43daSPeter Zijlstra 		if (sched_rt_runtime(rt_rq) != RUNTIME_INF) {
1075391e43daSPeter Zijlstra 			raw_spin_lock(&rt_rq->rt_runtime_lock);
1076391e43daSPeter Zijlstra 			rt_rq->rt_time += delta_exec;
10779b58e976SLi Hua 			exceeded = sched_rt_runtime_exceeded(rt_rq);
10789b58e976SLi Hua 			if (exceeded)
10798875125eSKirill Tkhai 				resched_curr(rq);
1080391e43daSPeter Zijlstra 			raw_spin_unlock(&rt_rq->rt_runtime_lock);
10819b58e976SLi Hua 			if (exceeded)
10829b58e976SLi Hua 				do_start_rt_bandwidth(sched_rt_bandwidth(rt_rq));
1083391e43daSPeter Zijlstra 		}
1084391e43daSPeter Zijlstra 	}
1085391e43daSPeter Zijlstra }
1086391e43daSPeter Zijlstra 
1087f4ebcbc0SKirill Tkhai static void
10885c66d1b9SNicolas Saenz Julienne dequeue_top_rt_rq(struct rt_rq *rt_rq, unsigned int count)
1089f4ebcbc0SKirill Tkhai {
1090f4ebcbc0SKirill Tkhai 	struct rq *rq = rq_of_rt_rq(rt_rq);
1091f4ebcbc0SKirill Tkhai 
1092f4ebcbc0SKirill Tkhai 	BUG_ON(&rq->rt != rt_rq);
1093f4ebcbc0SKirill Tkhai 
1094f4ebcbc0SKirill Tkhai 	if (!rt_rq->rt_queued)
1095f4ebcbc0SKirill Tkhai 		return;
1096f4ebcbc0SKirill Tkhai 
1097f4ebcbc0SKirill Tkhai 	BUG_ON(!rq->nr_running);
1098f4ebcbc0SKirill Tkhai 
10995c66d1b9SNicolas Saenz Julienne 	sub_nr_running(rq, count);
1100f4ebcbc0SKirill Tkhai 	rt_rq->rt_queued = 0;
11018f111bc3SPeter Zijlstra 
1102f4ebcbc0SKirill Tkhai }
1103f4ebcbc0SKirill Tkhai 
1104f4ebcbc0SKirill Tkhai static void
1105f4ebcbc0SKirill Tkhai enqueue_top_rt_rq(struct rt_rq *rt_rq)
1106f4ebcbc0SKirill Tkhai {
1107f4ebcbc0SKirill Tkhai 	struct rq *rq = rq_of_rt_rq(rt_rq);
1108f4ebcbc0SKirill Tkhai 
1109f4ebcbc0SKirill Tkhai 	BUG_ON(&rq->rt != rt_rq);
1110f4ebcbc0SKirill Tkhai 
1111f4ebcbc0SKirill Tkhai 	if (rt_rq->rt_queued)
1112f4ebcbc0SKirill Tkhai 		return;
1113296b2ffeSVincent Guittot 
1114296b2ffeSVincent Guittot 	if (rt_rq_throttled(rt_rq))
1115f4ebcbc0SKirill Tkhai 		return;
1116f4ebcbc0SKirill Tkhai 
1117296b2ffeSVincent Guittot 	if (rt_rq->rt_nr_running) {
111872465447SKirill Tkhai 		add_nr_running(rq, rt_rq->rt_nr_running);
1119f4ebcbc0SKirill Tkhai 		rt_rq->rt_queued = 1;
1120296b2ffeSVincent Guittot 	}
11218f111bc3SPeter Zijlstra 
11228f111bc3SPeter Zijlstra 	/* Kick cpufreq (see the comment in kernel/sched/sched.h). */
11238f111bc3SPeter Zijlstra 	cpufreq_update_util(rq, 0);
1124f4ebcbc0SKirill Tkhai }
1125f4ebcbc0SKirill Tkhai 
1126391e43daSPeter Zijlstra #if defined CONFIG_SMP
1127391e43daSPeter Zijlstra 
1128391e43daSPeter Zijlstra static void
1129391e43daSPeter Zijlstra inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio)
1130391e43daSPeter Zijlstra {
1131391e43daSPeter Zijlstra 	struct rq *rq = rq_of_rt_rq(rt_rq);
1132391e43daSPeter Zijlstra 
1133757dfcaaSKirill Tkhai #ifdef CONFIG_RT_GROUP_SCHED
1134757dfcaaSKirill Tkhai 	/*
1135757dfcaaSKirill Tkhai 	 * Change rq's cpupri only if rt_rq is the top queue.
1136757dfcaaSKirill Tkhai 	 */
1137757dfcaaSKirill Tkhai 	if (&rq->rt != rt_rq)
1138757dfcaaSKirill Tkhai 		return;
1139757dfcaaSKirill Tkhai #endif
1140391e43daSPeter Zijlstra 	if (rq->online && prio < prev_prio)
1141391e43daSPeter Zijlstra 		cpupri_set(&rq->rd->cpupri, rq->cpu, prio);
1142391e43daSPeter Zijlstra }
1143391e43daSPeter Zijlstra 
1144391e43daSPeter Zijlstra static void
1145391e43daSPeter Zijlstra dec_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio)
1146391e43daSPeter Zijlstra {
1147391e43daSPeter Zijlstra 	struct rq *rq = rq_of_rt_rq(rt_rq);
1148391e43daSPeter Zijlstra 
1149757dfcaaSKirill Tkhai #ifdef CONFIG_RT_GROUP_SCHED
1150757dfcaaSKirill Tkhai 	/*
1151757dfcaaSKirill Tkhai 	 * Change rq's cpupri only if rt_rq is the top queue.
1152757dfcaaSKirill Tkhai 	 */
1153757dfcaaSKirill Tkhai 	if (&rq->rt != rt_rq)
1154757dfcaaSKirill Tkhai 		return;
1155757dfcaaSKirill Tkhai #endif
1156391e43daSPeter Zijlstra 	if (rq->online && rt_rq->highest_prio.curr != prev_prio)
1157391e43daSPeter Zijlstra 		cpupri_set(&rq->rd->cpupri, rq->cpu, rt_rq->highest_prio.curr);
1158391e43daSPeter Zijlstra }
1159391e43daSPeter Zijlstra 
1160391e43daSPeter Zijlstra #else /* CONFIG_SMP */
1161391e43daSPeter Zijlstra 
1162391e43daSPeter Zijlstra static inline
1163391e43daSPeter Zijlstra void inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) {}
1164391e43daSPeter Zijlstra static inline
1165391e43daSPeter Zijlstra void dec_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) {}
1166391e43daSPeter Zijlstra 
1167391e43daSPeter Zijlstra #endif /* CONFIG_SMP */
1168391e43daSPeter Zijlstra 
1169391e43daSPeter Zijlstra #if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
1170391e43daSPeter Zijlstra static void
1171391e43daSPeter Zijlstra inc_rt_prio(struct rt_rq *rt_rq, int prio)
1172391e43daSPeter Zijlstra {
1173391e43daSPeter Zijlstra 	int prev_prio = rt_rq->highest_prio.curr;
1174391e43daSPeter Zijlstra 
1175391e43daSPeter Zijlstra 	if (prio < prev_prio)
1176391e43daSPeter Zijlstra 		rt_rq->highest_prio.curr = prio;
1177391e43daSPeter Zijlstra 
1178391e43daSPeter Zijlstra 	inc_rt_prio_smp(rt_rq, prio, prev_prio);
1179391e43daSPeter Zijlstra }
1180391e43daSPeter Zijlstra 
1181391e43daSPeter Zijlstra static void
1182391e43daSPeter Zijlstra dec_rt_prio(struct rt_rq *rt_rq, int prio)
1183391e43daSPeter Zijlstra {
1184391e43daSPeter Zijlstra 	int prev_prio = rt_rq->highest_prio.curr;
1185391e43daSPeter Zijlstra 
1186391e43daSPeter Zijlstra 	if (rt_rq->rt_nr_running) {
1187391e43daSPeter Zijlstra 
1188391e43daSPeter Zijlstra 		WARN_ON(prio < prev_prio);
1189391e43daSPeter Zijlstra 
1190391e43daSPeter Zijlstra 		/*
1191391e43daSPeter Zijlstra 		 * This may have been our highest task, and therefore
1192391e43daSPeter Zijlstra 		 * we may have some recomputation to do
1193391e43daSPeter Zijlstra 		 */
1194391e43daSPeter Zijlstra 		if (prio == prev_prio) {
1195391e43daSPeter Zijlstra 			struct rt_prio_array *array = &rt_rq->active;
1196391e43daSPeter Zijlstra 
1197391e43daSPeter Zijlstra 			rt_rq->highest_prio.curr =
1198391e43daSPeter Zijlstra 				sched_find_first_bit(array->bitmap);
1199391e43daSPeter Zijlstra 		}
1200391e43daSPeter Zijlstra 
1201934fc331SPeter Zijlstra 	} else {
1202934fc331SPeter Zijlstra 		rt_rq->highest_prio.curr = MAX_RT_PRIO-1;
1203934fc331SPeter Zijlstra 	}
1204391e43daSPeter Zijlstra 
1205391e43daSPeter Zijlstra 	dec_rt_prio_smp(rt_rq, prio, prev_prio);
1206391e43daSPeter Zijlstra }
1207391e43daSPeter Zijlstra 
1208391e43daSPeter Zijlstra #else
1209391e43daSPeter Zijlstra 
1210391e43daSPeter Zijlstra static inline void inc_rt_prio(struct rt_rq *rt_rq, int prio) {}
1211391e43daSPeter Zijlstra static inline void dec_rt_prio(struct rt_rq *rt_rq, int prio) {}
1212391e43daSPeter Zijlstra 
1213391e43daSPeter Zijlstra #endif /* CONFIG_SMP || CONFIG_RT_GROUP_SCHED */
1214391e43daSPeter Zijlstra 
1215391e43daSPeter Zijlstra #ifdef CONFIG_RT_GROUP_SCHED
1216391e43daSPeter Zijlstra 
1217391e43daSPeter Zijlstra static void
1218391e43daSPeter Zijlstra inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
1219391e43daSPeter Zijlstra {
1220391e43daSPeter Zijlstra 	if (rt_se_boosted(rt_se))
1221391e43daSPeter Zijlstra 		rt_rq->rt_nr_boosted++;
1222391e43daSPeter Zijlstra 
1223391e43daSPeter Zijlstra 	if (rt_rq->tg)
1224391e43daSPeter Zijlstra 		start_rt_bandwidth(&rt_rq->tg->rt_bandwidth);
1225391e43daSPeter Zijlstra }
1226391e43daSPeter Zijlstra 
1227391e43daSPeter Zijlstra static void
1228391e43daSPeter Zijlstra dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
1229391e43daSPeter Zijlstra {
1230391e43daSPeter Zijlstra 	if (rt_se_boosted(rt_se))
1231391e43daSPeter Zijlstra 		rt_rq->rt_nr_boosted--;
1232391e43daSPeter Zijlstra 
1233391e43daSPeter Zijlstra 	WARN_ON(!rt_rq->rt_nr_running && rt_rq->rt_nr_boosted);
1234391e43daSPeter Zijlstra }
1235391e43daSPeter Zijlstra 
1236391e43daSPeter Zijlstra #else /* CONFIG_RT_GROUP_SCHED */
1237391e43daSPeter Zijlstra 
1238391e43daSPeter Zijlstra static void
1239391e43daSPeter Zijlstra inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
1240391e43daSPeter Zijlstra {
1241391e43daSPeter Zijlstra 	start_rt_bandwidth(&def_rt_bandwidth);
1242391e43daSPeter Zijlstra }
1243391e43daSPeter Zijlstra 
1244391e43daSPeter Zijlstra static inline
1245391e43daSPeter Zijlstra void dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) {}
1246391e43daSPeter Zijlstra 
1247391e43daSPeter Zijlstra #endif /* CONFIG_RT_GROUP_SCHED */
1248391e43daSPeter Zijlstra 
1249391e43daSPeter Zijlstra static inline
125022abdef3SKirill Tkhai unsigned int rt_se_nr_running(struct sched_rt_entity *rt_se)
125122abdef3SKirill Tkhai {
125222abdef3SKirill Tkhai 	struct rt_rq *group_rq = group_rt_rq(rt_se);
125322abdef3SKirill Tkhai 
125422abdef3SKirill Tkhai 	if (group_rq)
125522abdef3SKirill Tkhai 		return group_rq->rt_nr_running;
125622abdef3SKirill Tkhai 	else
125722abdef3SKirill Tkhai 		return 1;
125822abdef3SKirill Tkhai }
125922abdef3SKirill Tkhai 
126022abdef3SKirill Tkhai static inline
126101d36d0aSFrederic Weisbecker unsigned int rt_se_rr_nr_running(struct sched_rt_entity *rt_se)
126201d36d0aSFrederic Weisbecker {
126301d36d0aSFrederic Weisbecker 	struct rt_rq *group_rq = group_rt_rq(rt_se);
126401d36d0aSFrederic Weisbecker 	struct task_struct *tsk;
126501d36d0aSFrederic Weisbecker 
126601d36d0aSFrederic Weisbecker 	if (group_rq)
126701d36d0aSFrederic Weisbecker 		return group_rq->rr_nr_running;
126801d36d0aSFrederic Weisbecker 
126901d36d0aSFrederic Weisbecker 	tsk = rt_task_of(rt_se);
127001d36d0aSFrederic Weisbecker 
127101d36d0aSFrederic Weisbecker 	return (tsk->policy == SCHED_RR) ? 1 : 0;
127201d36d0aSFrederic Weisbecker }
127301d36d0aSFrederic Weisbecker 
127401d36d0aSFrederic Weisbecker static inline
1275391e43daSPeter Zijlstra void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
1276391e43daSPeter Zijlstra {
1277391e43daSPeter Zijlstra 	int prio = rt_se_prio(rt_se);
1278391e43daSPeter Zijlstra 
1279391e43daSPeter Zijlstra 	WARN_ON(!rt_prio(prio));
128022abdef3SKirill Tkhai 	rt_rq->rt_nr_running += rt_se_nr_running(rt_se);
128101d36d0aSFrederic Weisbecker 	rt_rq->rr_nr_running += rt_se_rr_nr_running(rt_se);
1282391e43daSPeter Zijlstra 
1283391e43daSPeter Zijlstra 	inc_rt_prio(rt_rq, prio);
1284391e43daSPeter Zijlstra 	inc_rt_migration(rt_se, rt_rq);
1285391e43daSPeter Zijlstra 	inc_rt_group(rt_se, rt_rq);
1286391e43daSPeter Zijlstra }
1287391e43daSPeter Zijlstra 
1288391e43daSPeter Zijlstra static inline
1289391e43daSPeter Zijlstra void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
1290391e43daSPeter Zijlstra {
1291391e43daSPeter Zijlstra 	WARN_ON(!rt_prio(rt_se_prio(rt_se)));
1292391e43daSPeter Zijlstra 	WARN_ON(!rt_rq->rt_nr_running);
129322abdef3SKirill Tkhai 	rt_rq->rt_nr_running -= rt_se_nr_running(rt_se);
129401d36d0aSFrederic Weisbecker 	rt_rq->rr_nr_running -= rt_se_rr_nr_running(rt_se);
1295391e43daSPeter Zijlstra 
1296391e43daSPeter Zijlstra 	dec_rt_prio(rt_rq, rt_se_prio(rt_se));
1297391e43daSPeter Zijlstra 	dec_rt_migration(rt_se, rt_rq);
1298391e43daSPeter Zijlstra 	dec_rt_group(rt_se, rt_rq);
1299391e43daSPeter Zijlstra }
1300391e43daSPeter Zijlstra 
1301ff77e468SPeter Zijlstra /*
1302ff77e468SPeter Zijlstra  * Change rt_se->run_list location unless SAVE && !MOVE
1303ff77e468SPeter Zijlstra  *
1304ff77e468SPeter Zijlstra  * assumes ENQUEUE/DEQUEUE flags match
1305ff77e468SPeter Zijlstra  */
1306ff77e468SPeter Zijlstra static inline bool move_entity(unsigned int flags)
1307ff77e468SPeter Zijlstra {
1308ff77e468SPeter Zijlstra 	if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) == DEQUEUE_SAVE)
1309ff77e468SPeter Zijlstra 		return false;
1310ff77e468SPeter Zijlstra 
1311ff77e468SPeter Zijlstra 	return true;
1312ff77e468SPeter Zijlstra }
1313ff77e468SPeter Zijlstra 
1314ff77e468SPeter Zijlstra static void __delist_rt_entity(struct sched_rt_entity *rt_se, struct rt_prio_array *array)
1315ff77e468SPeter Zijlstra {
1316ff77e468SPeter Zijlstra 	list_del_init(&rt_se->run_list);
1317ff77e468SPeter Zijlstra 
1318ff77e468SPeter Zijlstra 	if (list_empty(array->queue + rt_se_prio(rt_se)))
1319ff77e468SPeter Zijlstra 		__clear_bit(rt_se_prio(rt_se), array->bitmap);
1320ff77e468SPeter Zijlstra 
1321ff77e468SPeter Zijlstra 	rt_se->on_list = 0;
1322ff77e468SPeter Zijlstra }
1323ff77e468SPeter Zijlstra 
132457a5c2daSYafang Shao static inline struct sched_statistics *
132557a5c2daSYafang Shao __schedstats_from_rt_se(struct sched_rt_entity *rt_se)
132657a5c2daSYafang Shao {
132757a5c2daSYafang Shao #ifdef CONFIG_RT_GROUP_SCHED
132857a5c2daSYafang Shao 	/* schedstats is not supported for rt group. */
132957a5c2daSYafang Shao 	if (!rt_entity_is_task(rt_se))
133057a5c2daSYafang Shao 		return NULL;
133157a5c2daSYafang Shao #endif
133257a5c2daSYafang Shao 
133357a5c2daSYafang Shao 	return &rt_task_of(rt_se)->stats;
133457a5c2daSYafang Shao }
133557a5c2daSYafang Shao 
133657a5c2daSYafang Shao static inline void
133757a5c2daSYafang Shao update_stats_wait_start_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se)
133857a5c2daSYafang Shao {
133957a5c2daSYafang Shao 	struct sched_statistics *stats;
134057a5c2daSYafang Shao 	struct task_struct *p = NULL;
134157a5c2daSYafang Shao 
134257a5c2daSYafang Shao 	if (!schedstat_enabled())
134357a5c2daSYafang Shao 		return;
134457a5c2daSYafang Shao 
134557a5c2daSYafang Shao 	if (rt_entity_is_task(rt_se))
134657a5c2daSYafang Shao 		p = rt_task_of(rt_se);
134757a5c2daSYafang Shao 
134857a5c2daSYafang Shao 	stats = __schedstats_from_rt_se(rt_se);
134957a5c2daSYafang Shao 	if (!stats)
135057a5c2daSYafang Shao 		return;
135157a5c2daSYafang Shao 
135257a5c2daSYafang Shao 	__update_stats_wait_start(rq_of_rt_rq(rt_rq), p, stats);
135357a5c2daSYafang Shao }
135457a5c2daSYafang Shao 
135557a5c2daSYafang Shao static inline void
135657a5c2daSYafang Shao update_stats_enqueue_sleeper_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se)
135757a5c2daSYafang Shao {
135857a5c2daSYafang Shao 	struct sched_statistics *stats;
135957a5c2daSYafang Shao 	struct task_struct *p = NULL;
136057a5c2daSYafang Shao 
136157a5c2daSYafang Shao 	if (!schedstat_enabled())
136257a5c2daSYafang Shao 		return;
136357a5c2daSYafang Shao 
136457a5c2daSYafang Shao 	if (rt_entity_is_task(rt_se))
136557a5c2daSYafang Shao 		p = rt_task_of(rt_se);
136657a5c2daSYafang Shao 
136757a5c2daSYafang Shao 	stats = __schedstats_from_rt_se(rt_se);
136857a5c2daSYafang Shao 	if (!stats)
136957a5c2daSYafang Shao 		return;
137057a5c2daSYafang Shao 
137157a5c2daSYafang Shao 	__update_stats_enqueue_sleeper(rq_of_rt_rq(rt_rq), p, stats);
137257a5c2daSYafang Shao }
137357a5c2daSYafang Shao 
137457a5c2daSYafang Shao static inline void
137557a5c2daSYafang Shao update_stats_enqueue_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se,
137657a5c2daSYafang Shao 			int flags)
137757a5c2daSYafang Shao {
137857a5c2daSYafang Shao 	if (!schedstat_enabled())
137957a5c2daSYafang Shao 		return;
138057a5c2daSYafang Shao 
138157a5c2daSYafang Shao 	if (flags & ENQUEUE_WAKEUP)
138257a5c2daSYafang Shao 		update_stats_enqueue_sleeper_rt(rt_rq, rt_se);
138357a5c2daSYafang Shao }
138457a5c2daSYafang Shao 
138557a5c2daSYafang Shao static inline void
138657a5c2daSYafang Shao update_stats_wait_end_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se)
138757a5c2daSYafang Shao {
138857a5c2daSYafang Shao 	struct sched_statistics *stats;
138957a5c2daSYafang Shao 	struct task_struct *p = NULL;
139057a5c2daSYafang Shao 
139157a5c2daSYafang Shao 	if (!schedstat_enabled())
139257a5c2daSYafang Shao 		return;
139357a5c2daSYafang Shao 
139457a5c2daSYafang Shao 	if (rt_entity_is_task(rt_se))
139557a5c2daSYafang Shao 		p = rt_task_of(rt_se);
139657a5c2daSYafang Shao 
139757a5c2daSYafang Shao 	stats = __schedstats_from_rt_se(rt_se);
139857a5c2daSYafang Shao 	if (!stats)
139957a5c2daSYafang Shao 		return;
140057a5c2daSYafang Shao 
140157a5c2daSYafang Shao 	__update_stats_wait_end(rq_of_rt_rq(rt_rq), p, stats);
140257a5c2daSYafang Shao }
140357a5c2daSYafang Shao 
140457a5c2daSYafang Shao static inline void
140557a5c2daSYafang Shao update_stats_dequeue_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se,
140657a5c2daSYafang Shao 			int flags)
140757a5c2daSYafang Shao {
140857a5c2daSYafang Shao 	struct task_struct *p = NULL;
140957a5c2daSYafang Shao 
141057a5c2daSYafang Shao 	if (!schedstat_enabled())
141157a5c2daSYafang Shao 		return;
141257a5c2daSYafang Shao 
141357a5c2daSYafang Shao 	if (rt_entity_is_task(rt_se))
141457a5c2daSYafang Shao 		p = rt_task_of(rt_se);
141557a5c2daSYafang Shao 
141657a5c2daSYafang Shao 	if ((flags & DEQUEUE_SLEEP) && p) {
141757a5c2daSYafang Shao 		unsigned int state;
141857a5c2daSYafang Shao 
141957a5c2daSYafang Shao 		state = READ_ONCE(p->__state);
142057a5c2daSYafang Shao 		if (state & TASK_INTERRUPTIBLE)
142157a5c2daSYafang Shao 			__schedstat_set(p->stats.sleep_start,
142257a5c2daSYafang Shao 					rq_clock(rq_of_rt_rq(rt_rq)));
142357a5c2daSYafang Shao 
142457a5c2daSYafang Shao 		if (state & TASK_UNINTERRUPTIBLE)
142557a5c2daSYafang Shao 			__schedstat_set(p->stats.block_start,
142657a5c2daSYafang Shao 					rq_clock(rq_of_rt_rq(rt_rq)));
142757a5c2daSYafang Shao 	}
142857a5c2daSYafang Shao }
142957a5c2daSYafang Shao 
1430ff77e468SPeter Zijlstra static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
1431391e43daSPeter Zijlstra {
1432391e43daSPeter Zijlstra 	struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
1433391e43daSPeter Zijlstra 	struct rt_prio_array *array = &rt_rq->active;
1434391e43daSPeter Zijlstra 	struct rt_rq *group_rq = group_rt_rq(rt_se);
1435391e43daSPeter Zijlstra 	struct list_head *queue = array->queue + rt_se_prio(rt_se);
1436391e43daSPeter Zijlstra 
1437391e43daSPeter Zijlstra 	/*
1438391e43daSPeter Zijlstra 	 * Don't enqueue the group if its throttled, or when empty.
1439391e43daSPeter Zijlstra 	 * The latter is a consequence of the former when a child group
1440391e43daSPeter Zijlstra 	 * get throttled and the current group doesn't have any other
1441391e43daSPeter Zijlstra 	 * active members.
1442391e43daSPeter Zijlstra 	 */
1443ff77e468SPeter Zijlstra 	if (group_rq && (rt_rq_throttled(group_rq) || !group_rq->rt_nr_running)) {
1444ff77e468SPeter Zijlstra 		if (rt_se->on_list)
1445ff77e468SPeter Zijlstra 			__delist_rt_entity(rt_se, array);
1446391e43daSPeter Zijlstra 		return;
1447ff77e468SPeter Zijlstra 	}
1448391e43daSPeter Zijlstra 
1449ff77e468SPeter Zijlstra 	if (move_entity(flags)) {
1450ff77e468SPeter Zijlstra 		WARN_ON_ONCE(rt_se->on_list);
1451ff77e468SPeter Zijlstra 		if (flags & ENQUEUE_HEAD)
1452391e43daSPeter Zijlstra 			list_add(&rt_se->run_list, queue);
1453391e43daSPeter Zijlstra 		else
1454391e43daSPeter Zijlstra 			list_add_tail(&rt_se->run_list, queue);
1455ff77e468SPeter Zijlstra 
1456391e43daSPeter Zijlstra 		__set_bit(rt_se_prio(rt_se), array->bitmap);
1457ff77e468SPeter Zijlstra 		rt_se->on_list = 1;
1458ff77e468SPeter Zijlstra 	}
1459ff77e468SPeter Zijlstra 	rt_se->on_rq = 1;
1460391e43daSPeter Zijlstra 
1461391e43daSPeter Zijlstra 	inc_rt_tasks(rt_se, rt_rq);
1462391e43daSPeter Zijlstra }
1463391e43daSPeter Zijlstra 
1464ff77e468SPeter Zijlstra static void __dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
1465391e43daSPeter Zijlstra {
1466391e43daSPeter Zijlstra 	struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
1467391e43daSPeter Zijlstra 	struct rt_prio_array *array = &rt_rq->active;
1468391e43daSPeter Zijlstra 
1469ff77e468SPeter Zijlstra 	if (move_entity(flags)) {
1470ff77e468SPeter Zijlstra 		WARN_ON_ONCE(!rt_se->on_list);
1471ff77e468SPeter Zijlstra 		__delist_rt_entity(rt_se, array);
1472ff77e468SPeter Zijlstra 	}
1473ff77e468SPeter Zijlstra 	rt_se->on_rq = 0;
1474391e43daSPeter Zijlstra 
1475391e43daSPeter Zijlstra 	dec_rt_tasks(rt_se, rt_rq);
1476391e43daSPeter Zijlstra }
1477391e43daSPeter Zijlstra 
1478391e43daSPeter Zijlstra /*
1479391e43daSPeter Zijlstra  * Because the prio of an upper entry depends on the lower
1480391e43daSPeter Zijlstra  * entries, we must remove entries top - down.
1481391e43daSPeter Zijlstra  */
1482ff77e468SPeter Zijlstra static void dequeue_rt_stack(struct sched_rt_entity *rt_se, unsigned int flags)
1483391e43daSPeter Zijlstra {
1484391e43daSPeter Zijlstra 	struct sched_rt_entity *back = NULL;
14855c66d1b9SNicolas Saenz Julienne 	unsigned int rt_nr_running;
1486391e43daSPeter Zijlstra 
1487391e43daSPeter Zijlstra 	for_each_sched_rt_entity(rt_se) {
1488391e43daSPeter Zijlstra 		rt_se->back = back;
1489391e43daSPeter Zijlstra 		back = rt_se;
1490391e43daSPeter Zijlstra 	}
1491391e43daSPeter Zijlstra 
14925c66d1b9SNicolas Saenz Julienne 	rt_nr_running = rt_rq_of_se(back)->rt_nr_running;
1493f4ebcbc0SKirill Tkhai 
1494391e43daSPeter Zijlstra 	for (rt_se = back; rt_se; rt_se = rt_se->back) {
1495391e43daSPeter Zijlstra 		if (on_rt_rq(rt_se))
1496ff77e468SPeter Zijlstra 			__dequeue_rt_entity(rt_se, flags);
1497391e43daSPeter Zijlstra 	}
14985c66d1b9SNicolas Saenz Julienne 
14995c66d1b9SNicolas Saenz Julienne 	dequeue_top_rt_rq(rt_rq_of_se(back), rt_nr_running);
1500391e43daSPeter Zijlstra }
1501391e43daSPeter Zijlstra 
1502ff77e468SPeter Zijlstra static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
1503391e43daSPeter Zijlstra {
1504f4ebcbc0SKirill Tkhai 	struct rq *rq = rq_of_rt_se(rt_se);
1505f4ebcbc0SKirill Tkhai 
150657a5c2daSYafang Shao 	update_stats_enqueue_rt(rt_rq_of_se(rt_se), rt_se, flags);
150757a5c2daSYafang Shao 
1508ff77e468SPeter Zijlstra 	dequeue_rt_stack(rt_se, flags);
1509391e43daSPeter Zijlstra 	for_each_sched_rt_entity(rt_se)
1510ff77e468SPeter Zijlstra 		__enqueue_rt_entity(rt_se, flags);
1511f4ebcbc0SKirill Tkhai 	enqueue_top_rt_rq(&rq->rt);
1512391e43daSPeter Zijlstra }
1513391e43daSPeter Zijlstra 
1514ff77e468SPeter Zijlstra static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
1515391e43daSPeter Zijlstra {
1516f4ebcbc0SKirill Tkhai 	struct rq *rq = rq_of_rt_se(rt_se);
1517f4ebcbc0SKirill Tkhai 
151857a5c2daSYafang Shao 	update_stats_dequeue_rt(rt_rq_of_se(rt_se), rt_se, flags);
151957a5c2daSYafang Shao 
1520ff77e468SPeter Zijlstra 	dequeue_rt_stack(rt_se, flags);
1521391e43daSPeter Zijlstra 
1522391e43daSPeter Zijlstra 	for_each_sched_rt_entity(rt_se) {
1523391e43daSPeter Zijlstra 		struct rt_rq *rt_rq = group_rt_rq(rt_se);
1524391e43daSPeter Zijlstra 
1525391e43daSPeter Zijlstra 		if (rt_rq && rt_rq->rt_nr_running)
1526ff77e468SPeter Zijlstra 			__enqueue_rt_entity(rt_se, flags);
1527391e43daSPeter Zijlstra 	}
1528f4ebcbc0SKirill Tkhai 	enqueue_top_rt_rq(&rq->rt);
1529391e43daSPeter Zijlstra }
1530391e43daSPeter Zijlstra 
1531391e43daSPeter Zijlstra /*
1532391e43daSPeter Zijlstra  * Adding/removing a task to/from a priority array:
1533391e43daSPeter Zijlstra  */
1534391e43daSPeter Zijlstra static void
1535391e43daSPeter Zijlstra enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags)
1536391e43daSPeter Zijlstra {
1537391e43daSPeter Zijlstra 	struct sched_rt_entity *rt_se = &p->rt;
1538391e43daSPeter Zijlstra 
1539391e43daSPeter Zijlstra 	if (flags & ENQUEUE_WAKEUP)
1540391e43daSPeter Zijlstra 		rt_se->timeout = 0;
1541391e43daSPeter Zijlstra 
154257a5c2daSYafang Shao 	check_schedstat_required();
154357a5c2daSYafang Shao 	update_stats_wait_start_rt(rt_rq_of_se(rt_se), rt_se);
154457a5c2daSYafang Shao 
1545ff77e468SPeter Zijlstra 	enqueue_rt_entity(rt_se, flags);
1546391e43daSPeter Zijlstra 
15474b53a341SIngo Molnar 	if (!task_current(rq, p) && p->nr_cpus_allowed > 1)
1548391e43daSPeter Zijlstra 		enqueue_pushable_task(rq, p);
1549391e43daSPeter Zijlstra }
1550391e43daSPeter Zijlstra 
1551391e43daSPeter Zijlstra static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags)
1552391e43daSPeter Zijlstra {
1553391e43daSPeter Zijlstra 	struct sched_rt_entity *rt_se = &p->rt;
1554391e43daSPeter Zijlstra 
1555391e43daSPeter Zijlstra 	update_curr_rt(rq);
1556ff77e468SPeter Zijlstra 	dequeue_rt_entity(rt_se, flags);
1557391e43daSPeter Zijlstra 
1558391e43daSPeter Zijlstra 	dequeue_pushable_task(rq, p);
1559391e43daSPeter Zijlstra }
1560391e43daSPeter Zijlstra 
1561391e43daSPeter Zijlstra /*
1562391e43daSPeter Zijlstra  * Put task to the head or the end of the run list without the overhead of
1563391e43daSPeter Zijlstra  * dequeue followed by enqueue.
1564391e43daSPeter Zijlstra  */
1565391e43daSPeter Zijlstra static void
1566391e43daSPeter Zijlstra requeue_rt_entity(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, int head)
1567391e43daSPeter Zijlstra {
1568391e43daSPeter Zijlstra 	if (on_rt_rq(rt_se)) {
1569391e43daSPeter Zijlstra 		struct rt_prio_array *array = &rt_rq->active;
1570391e43daSPeter Zijlstra 		struct list_head *queue = array->queue + rt_se_prio(rt_se);
1571391e43daSPeter Zijlstra 
1572391e43daSPeter Zijlstra 		if (head)
1573391e43daSPeter Zijlstra 			list_move(&rt_se->run_list, queue);
1574391e43daSPeter Zijlstra 		else
1575391e43daSPeter Zijlstra 			list_move_tail(&rt_se->run_list, queue);
1576391e43daSPeter Zijlstra 	}
1577391e43daSPeter Zijlstra }
1578391e43daSPeter Zijlstra 
1579391e43daSPeter Zijlstra static void requeue_task_rt(struct rq *rq, struct task_struct *p, int head)
1580391e43daSPeter Zijlstra {
1581391e43daSPeter Zijlstra 	struct sched_rt_entity *rt_se = &p->rt;
1582391e43daSPeter Zijlstra 	struct rt_rq *rt_rq;
1583391e43daSPeter Zijlstra 
1584391e43daSPeter Zijlstra 	for_each_sched_rt_entity(rt_se) {
1585391e43daSPeter Zijlstra 		rt_rq = rt_rq_of_se(rt_se);
1586391e43daSPeter Zijlstra 		requeue_rt_entity(rt_rq, rt_se, head);
1587391e43daSPeter Zijlstra 	}
1588391e43daSPeter Zijlstra }
1589391e43daSPeter Zijlstra 
1590391e43daSPeter Zijlstra static void yield_task_rt(struct rq *rq)
1591391e43daSPeter Zijlstra {
1592391e43daSPeter Zijlstra 	requeue_task_rt(rq, rq->curr, 0);
1593391e43daSPeter Zijlstra }
1594391e43daSPeter Zijlstra 
1595391e43daSPeter Zijlstra #ifdef CONFIG_SMP
1596391e43daSPeter Zijlstra static int find_lowest_rq(struct task_struct *task);
1597391e43daSPeter Zijlstra 
1598391e43daSPeter Zijlstra static int
15993aef1551SValentin Schneider select_task_rq_rt(struct task_struct *p, int cpu, int flags)
1600391e43daSPeter Zijlstra {
1601391e43daSPeter Zijlstra 	struct task_struct *curr;
1602391e43daSPeter Zijlstra 	struct rq *rq;
1603804d402fSQais Yousef 	bool test;
1604391e43daSPeter Zijlstra 
1605391e43daSPeter Zijlstra 	/* For anything but wake ups, just return the task_cpu */
16063aef1551SValentin Schneider 	if (!(flags & (WF_TTWU | WF_FORK)))
1607391e43daSPeter Zijlstra 		goto out;
1608391e43daSPeter Zijlstra 
1609391e43daSPeter Zijlstra 	rq = cpu_rq(cpu);
1610391e43daSPeter Zijlstra 
1611391e43daSPeter Zijlstra 	rcu_read_lock();
1612316c1608SJason Low 	curr = READ_ONCE(rq->curr); /* unlocked access */
1613391e43daSPeter Zijlstra 
1614391e43daSPeter Zijlstra 	/*
1615391e43daSPeter Zijlstra 	 * If the current task on @p's runqueue is an RT task, then
1616391e43daSPeter Zijlstra 	 * try to see if we can wake this RT task up on another
1617391e43daSPeter Zijlstra 	 * runqueue. Otherwise simply start this RT task
1618391e43daSPeter Zijlstra 	 * on its current runqueue.
1619391e43daSPeter Zijlstra 	 *
1620391e43daSPeter Zijlstra 	 * We want to avoid overloading runqueues. If the woken
1621391e43daSPeter Zijlstra 	 * task is a higher priority, then it will stay on this CPU
1622391e43daSPeter Zijlstra 	 * and the lower prio task should be moved to another CPU.
1623391e43daSPeter Zijlstra 	 * Even though this will probably make the lower prio task
1624391e43daSPeter Zijlstra 	 * lose its cache, we do not want to bounce a higher task
1625391e43daSPeter Zijlstra 	 * around just because it gave up its CPU, perhaps for a
1626391e43daSPeter Zijlstra 	 * lock?
1627391e43daSPeter Zijlstra 	 *
1628391e43daSPeter Zijlstra 	 * For equal prio tasks, we just let the scheduler sort it out.
1629391e43daSPeter Zijlstra 	 *
1630391e43daSPeter Zijlstra 	 * Otherwise, just let it ride on the affined RQ and the
1631391e43daSPeter Zijlstra 	 * post-schedule router will push the preempted task away
1632391e43daSPeter Zijlstra 	 *
1633391e43daSPeter Zijlstra 	 * This test is optimistic, if we get it wrong the load-balancer
1634391e43daSPeter Zijlstra 	 * will have to sort it out.
1635804d402fSQais Yousef 	 *
1636804d402fSQais Yousef 	 * We take into account the capacity of the CPU to ensure it fits the
1637804d402fSQais Yousef 	 * requirement of the task - which is only important on heterogeneous
1638804d402fSQais Yousef 	 * systems like big.LITTLE.
1639391e43daSPeter Zijlstra 	 */
1640804d402fSQais Yousef 	test = curr &&
1641804d402fSQais Yousef 	       unlikely(rt_task(curr)) &&
1642804d402fSQais Yousef 	       (curr->nr_cpus_allowed < 2 || curr->prio <= p->prio);
1643804d402fSQais Yousef 
1644804d402fSQais Yousef 	if (test || !rt_task_fits_capacity(p, cpu)) {
1645391e43daSPeter Zijlstra 		int target = find_lowest_rq(p);
1646391e43daSPeter Zijlstra 
164780e3d87bSTim Chen 		/*
1648b28bc1e0SQais Yousef 		 * Bail out if we were forcing a migration to find a better
1649b28bc1e0SQais Yousef 		 * fitting CPU but our search failed.
1650b28bc1e0SQais Yousef 		 */
1651b28bc1e0SQais Yousef 		if (!test && target != -1 && !rt_task_fits_capacity(p, target))
1652b28bc1e0SQais Yousef 			goto out_unlock;
1653b28bc1e0SQais Yousef 
1654b28bc1e0SQais Yousef 		/*
165580e3d87bSTim Chen 		 * Don't bother moving it if the destination CPU is
165680e3d87bSTim Chen 		 * not running a lower priority task.
165780e3d87bSTim Chen 		 */
165880e3d87bSTim Chen 		if (target != -1 &&
165980e3d87bSTim Chen 		    p->prio < cpu_rq(target)->rt.highest_prio.curr)
1660391e43daSPeter Zijlstra 			cpu = target;
1661391e43daSPeter Zijlstra 	}
1662b28bc1e0SQais Yousef 
1663b28bc1e0SQais Yousef out_unlock:
1664391e43daSPeter Zijlstra 	rcu_read_unlock();
1665391e43daSPeter Zijlstra 
1666391e43daSPeter Zijlstra out:
1667391e43daSPeter Zijlstra 	return cpu;
1668391e43daSPeter Zijlstra }
1669391e43daSPeter Zijlstra 
1670391e43daSPeter Zijlstra static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)
1671391e43daSPeter Zijlstra {
1672308a623aSWanpeng Li 	/*
1673308a623aSWanpeng Li 	 * Current can't be migrated, useless to reschedule,
1674308a623aSWanpeng Li 	 * let's hope p can move out.
1675308a623aSWanpeng Li 	 */
16764b53a341SIngo Molnar 	if (rq->curr->nr_cpus_allowed == 1 ||
1677a1bd02e1SQais Yousef 	    !cpupri_find(&rq->rd->cpupri, rq->curr, NULL))
1678391e43daSPeter Zijlstra 		return;
1679391e43daSPeter Zijlstra 
1680308a623aSWanpeng Li 	/*
1681308a623aSWanpeng Li 	 * p is migratable, so let's not schedule it and
1682308a623aSWanpeng Li 	 * see if it is pushed or pulled somewhere else.
1683308a623aSWanpeng Li 	 */
1684804d402fSQais Yousef 	if (p->nr_cpus_allowed != 1 &&
1685a1bd02e1SQais Yousef 	    cpupri_find(&rq->rd->cpupri, p, NULL))
1686391e43daSPeter Zijlstra 		return;
1687391e43daSPeter Zijlstra 
1688391e43daSPeter Zijlstra 	/*
168997fb7a0aSIngo Molnar 	 * There appear to be other CPUs that can accept
169097fb7a0aSIngo Molnar 	 * the current task but none can run 'p', so lets reschedule
169197fb7a0aSIngo Molnar 	 * to try and push the current task away:
1692391e43daSPeter Zijlstra 	 */
1693391e43daSPeter Zijlstra 	requeue_task_rt(rq, p, 1);
16948875125eSKirill Tkhai 	resched_curr(rq);
1695391e43daSPeter Zijlstra }
1696391e43daSPeter Zijlstra 
16976e2df058SPeter Zijlstra static int balance_rt(struct rq *rq, struct task_struct *p, struct rq_flags *rf)
16986e2df058SPeter Zijlstra {
16996e2df058SPeter Zijlstra 	if (!on_rt_rq(&p->rt) && need_pull_rt_task(rq, p)) {
17006e2df058SPeter Zijlstra 		/*
17016e2df058SPeter Zijlstra 		 * This is OK, because current is on_cpu, which avoids it being
17026e2df058SPeter Zijlstra 		 * picked for load-balance and preemption/IRQs are still
17036e2df058SPeter Zijlstra 		 * disabled avoiding further scheduler activity on it and we've
17046e2df058SPeter Zijlstra 		 * not yet started the picking loop.
17056e2df058SPeter Zijlstra 		 */
17066e2df058SPeter Zijlstra 		rq_unpin_lock(rq, rf);
17076e2df058SPeter Zijlstra 		pull_rt_task(rq);
17086e2df058SPeter Zijlstra 		rq_repin_lock(rq, rf);
17096e2df058SPeter Zijlstra 	}
17106e2df058SPeter Zijlstra 
17116e2df058SPeter Zijlstra 	return sched_stop_runnable(rq) || sched_dl_runnable(rq) || sched_rt_runnable(rq);
17126e2df058SPeter Zijlstra }
1713391e43daSPeter Zijlstra #endif /* CONFIG_SMP */
1714391e43daSPeter Zijlstra 
1715391e43daSPeter Zijlstra /*
1716391e43daSPeter Zijlstra  * Preempt the current task with a newly woken task if needed:
1717391e43daSPeter Zijlstra  */
1718391e43daSPeter Zijlstra static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p, int flags)
1719391e43daSPeter Zijlstra {
1720391e43daSPeter Zijlstra 	if (p->prio < rq->curr->prio) {
17218875125eSKirill Tkhai 		resched_curr(rq);
1722391e43daSPeter Zijlstra 		return;
1723391e43daSPeter Zijlstra 	}
1724391e43daSPeter Zijlstra 
1725391e43daSPeter Zijlstra #ifdef CONFIG_SMP
1726391e43daSPeter Zijlstra 	/*
1727391e43daSPeter Zijlstra 	 * If:
1728391e43daSPeter Zijlstra 	 *
1729391e43daSPeter Zijlstra 	 * - the newly woken task is of equal priority to the current task
1730391e43daSPeter Zijlstra 	 * - the newly woken task is non-migratable while current is migratable
1731391e43daSPeter Zijlstra 	 * - current will be preempted on the next reschedule
1732391e43daSPeter Zijlstra 	 *
1733391e43daSPeter Zijlstra 	 * we should check to see if current can readily move to a different
1734391e43daSPeter Zijlstra 	 * cpu.  If so, we will reschedule to allow the push logic to try
1735391e43daSPeter Zijlstra 	 * to move current somewhere else, making room for our non-migratable
1736391e43daSPeter Zijlstra 	 * task.
1737391e43daSPeter Zijlstra 	 */
1738391e43daSPeter Zijlstra 	if (p->prio == rq->curr->prio && !test_tsk_need_resched(rq->curr))
1739391e43daSPeter Zijlstra 		check_preempt_equal_prio(rq, p);
1740391e43daSPeter Zijlstra #endif
1741391e43daSPeter Zijlstra }
1742391e43daSPeter Zijlstra 
1743a0e813f2SPeter Zijlstra static inline void set_next_task_rt(struct rq *rq, struct task_struct *p, bool first)
1744ff1cdc94SMuchun Song {
174557a5c2daSYafang Shao 	struct sched_rt_entity *rt_se = &p->rt;
174657a5c2daSYafang Shao 	struct rt_rq *rt_rq = &rq->rt;
174757a5c2daSYafang Shao 
1748ff1cdc94SMuchun Song 	p->se.exec_start = rq_clock_task(rq);
174957a5c2daSYafang Shao 	if (on_rt_rq(&p->rt))
175057a5c2daSYafang Shao 		update_stats_wait_end_rt(rt_rq, rt_se);
1751ff1cdc94SMuchun Song 
1752ff1cdc94SMuchun Song 	/* The running task is never eligible for pushing */
1753ff1cdc94SMuchun Song 	dequeue_pushable_task(rq, p);
1754f95d4eaeSPeter Zijlstra 
1755a0e813f2SPeter Zijlstra 	if (!first)
1756a0e813f2SPeter Zijlstra 		return;
1757a0e813f2SPeter Zijlstra 
1758f95d4eaeSPeter Zijlstra 	/*
1759f95d4eaeSPeter Zijlstra 	 * If prev task was rt, put_prev_task() has already updated the
1760f95d4eaeSPeter Zijlstra 	 * utilization. We only care of the case where we start to schedule a
1761f95d4eaeSPeter Zijlstra 	 * rt task
1762f95d4eaeSPeter Zijlstra 	 */
1763f95d4eaeSPeter Zijlstra 	if (rq->curr->sched_class != &rt_sched_class)
1764f95d4eaeSPeter Zijlstra 		update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 0);
1765f95d4eaeSPeter Zijlstra 
1766f95d4eaeSPeter Zijlstra 	rt_queue_push_tasks(rq);
1767ff1cdc94SMuchun Song }
1768ff1cdc94SMuchun Song 
1769821aecd0SDietmar Eggemann static struct sched_rt_entity *pick_next_rt_entity(struct rt_rq *rt_rq)
1770391e43daSPeter Zijlstra {
1771391e43daSPeter Zijlstra 	struct rt_prio_array *array = &rt_rq->active;
1772391e43daSPeter Zijlstra 	struct sched_rt_entity *next = NULL;
1773391e43daSPeter Zijlstra 	struct list_head *queue;
1774391e43daSPeter Zijlstra 	int idx;
1775391e43daSPeter Zijlstra 
1776391e43daSPeter Zijlstra 	idx = sched_find_first_bit(array->bitmap);
1777391e43daSPeter Zijlstra 	BUG_ON(idx >= MAX_RT_PRIO);
1778391e43daSPeter Zijlstra 
1779391e43daSPeter Zijlstra 	queue = array->queue + idx;
17807c4a5b89SPietro Borrello 	if (SCHED_WARN_ON(list_empty(queue)))
17817c4a5b89SPietro Borrello 		return NULL;
1782391e43daSPeter Zijlstra 	next = list_entry(queue->next, struct sched_rt_entity, run_list);
1783391e43daSPeter Zijlstra 
1784391e43daSPeter Zijlstra 	return next;
1785391e43daSPeter Zijlstra }
1786391e43daSPeter Zijlstra 
1787391e43daSPeter Zijlstra static struct task_struct *_pick_next_task_rt(struct rq *rq)
1788391e43daSPeter Zijlstra {
1789391e43daSPeter Zijlstra 	struct sched_rt_entity *rt_se;
1790606dba2eSPeter Zijlstra 	struct rt_rq *rt_rq  = &rq->rt;
1791391e43daSPeter Zijlstra 
1792391e43daSPeter Zijlstra 	do {
1793821aecd0SDietmar Eggemann 		rt_se = pick_next_rt_entity(rt_rq);
17947c4a5b89SPietro Borrello 		if (unlikely(!rt_se))
17957c4a5b89SPietro Borrello 			return NULL;
1796391e43daSPeter Zijlstra 		rt_rq = group_rt_rq(rt_se);
1797391e43daSPeter Zijlstra 	} while (rt_rq);
1798391e43daSPeter Zijlstra 
1799ff1cdc94SMuchun Song 	return rt_task_of(rt_se);
1800391e43daSPeter Zijlstra }
1801391e43daSPeter Zijlstra 
180221f56ffeSPeter Zijlstra static struct task_struct *pick_task_rt(struct rq *rq)
1803391e43daSPeter Zijlstra {
1804606dba2eSPeter Zijlstra 	struct task_struct *p;
1805606dba2eSPeter Zijlstra 
18066e2df058SPeter Zijlstra 	if (!sched_rt_runnable(rq))
1807606dba2eSPeter Zijlstra 		return NULL;
1808606dba2eSPeter Zijlstra 
1809606dba2eSPeter Zijlstra 	p = _pick_next_task_rt(rq);
181021f56ffeSPeter Zijlstra 
181121f56ffeSPeter Zijlstra 	return p;
181221f56ffeSPeter Zijlstra }
181321f56ffeSPeter Zijlstra 
181421f56ffeSPeter Zijlstra static struct task_struct *pick_next_task_rt(struct rq *rq)
181521f56ffeSPeter Zijlstra {
181621f56ffeSPeter Zijlstra 	struct task_struct *p = pick_task_rt(rq);
181721f56ffeSPeter Zijlstra 
181821f56ffeSPeter Zijlstra 	if (p)
1819a0e813f2SPeter Zijlstra 		set_next_task_rt(rq, p, true);
182021f56ffeSPeter Zijlstra 
1821391e43daSPeter Zijlstra 	return p;
1822391e43daSPeter Zijlstra }
1823391e43daSPeter Zijlstra 
18246e2df058SPeter Zijlstra static void put_prev_task_rt(struct rq *rq, struct task_struct *p)
1825391e43daSPeter Zijlstra {
182657a5c2daSYafang Shao 	struct sched_rt_entity *rt_se = &p->rt;
182757a5c2daSYafang Shao 	struct rt_rq *rt_rq = &rq->rt;
182857a5c2daSYafang Shao 
182957a5c2daSYafang Shao 	if (on_rt_rq(&p->rt))
183057a5c2daSYafang Shao 		update_stats_wait_start_rt(rt_rq, rt_se);
183157a5c2daSYafang Shao 
1832391e43daSPeter Zijlstra 	update_curr_rt(rq);
1833391e43daSPeter Zijlstra 
183423127296SVincent Guittot 	update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 1);
1835371bf427SVincent Guittot 
1836391e43daSPeter Zijlstra 	/*
1837391e43daSPeter Zijlstra 	 * The previous task needs to be made eligible for pushing
1838391e43daSPeter Zijlstra 	 * if it is still active
1839391e43daSPeter Zijlstra 	 */
18404b53a341SIngo Molnar 	if (on_rt_rq(&p->rt) && p->nr_cpus_allowed > 1)
1841391e43daSPeter Zijlstra 		enqueue_pushable_task(rq, p);
1842391e43daSPeter Zijlstra }
1843391e43daSPeter Zijlstra 
1844391e43daSPeter Zijlstra #ifdef CONFIG_SMP
1845391e43daSPeter Zijlstra 
1846391e43daSPeter Zijlstra /* Only try algorithms three times */
1847391e43daSPeter Zijlstra #define RT_MAX_TRIES 3
1848391e43daSPeter Zijlstra 
1849391e43daSPeter Zijlstra static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu)
1850391e43daSPeter Zijlstra {
18510b9d46fcSPeter Zijlstra 	if (!task_on_cpu(rq, p) &&
185295158a89SPeter Zijlstra 	    cpumask_test_cpu(cpu, &p->cpus_mask))
1853391e43daSPeter Zijlstra 		return 1;
185497fb7a0aSIngo Molnar 
1855391e43daSPeter Zijlstra 	return 0;
1856391e43daSPeter Zijlstra }
1857391e43daSPeter Zijlstra 
1858e23ee747SKirill Tkhai /*
1859e23ee747SKirill Tkhai  * Return the highest pushable rq's task, which is suitable to be executed
186097fb7a0aSIngo Molnar  * on the CPU, NULL otherwise
1861e23ee747SKirill Tkhai  */
1862e23ee747SKirill Tkhai static struct task_struct *pick_highest_pushable_task(struct rq *rq, int cpu)
1863391e43daSPeter Zijlstra {
1864e23ee747SKirill Tkhai 	struct plist_head *head = &rq->rt.pushable_tasks;
1865391e43daSPeter Zijlstra 	struct task_struct *p;
1866391e43daSPeter Zijlstra 
1867e23ee747SKirill Tkhai 	if (!has_pushable_tasks(rq))
1868e23ee747SKirill Tkhai 		return NULL;
1869391e43daSPeter Zijlstra 
1870e23ee747SKirill Tkhai 	plist_for_each_entry(p, head, pushable_tasks) {
1871e23ee747SKirill Tkhai 		if (pick_rt_task(rq, p, cpu))
1872e23ee747SKirill Tkhai 			return p;
1873391e43daSPeter Zijlstra 	}
1874391e43daSPeter Zijlstra 
1875e23ee747SKirill Tkhai 	return NULL;
1876391e43daSPeter Zijlstra }
1877391e43daSPeter Zijlstra 
1878391e43daSPeter Zijlstra static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask);
1879391e43daSPeter Zijlstra 
1880391e43daSPeter Zijlstra static int find_lowest_rq(struct task_struct *task)
1881391e43daSPeter Zijlstra {
1882391e43daSPeter Zijlstra 	struct sched_domain *sd;
18834ba29684SChristoph Lameter 	struct cpumask *lowest_mask = this_cpu_cpumask_var_ptr(local_cpu_mask);
1884391e43daSPeter Zijlstra 	int this_cpu = smp_processor_id();
1885391e43daSPeter Zijlstra 	int cpu      = task_cpu(task);
1886a1bd02e1SQais Yousef 	int ret;
1887391e43daSPeter Zijlstra 
1888391e43daSPeter Zijlstra 	/* Make sure the mask is initialized first */
1889391e43daSPeter Zijlstra 	if (unlikely(!lowest_mask))
1890391e43daSPeter Zijlstra 		return -1;
1891391e43daSPeter Zijlstra 
18924b53a341SIngo Molnar 	if (task->nr_cpus_allowed == 1)
1893391e43daSPeter Zijlstra 		return -1; /* No other targets possible */
1894391e43daSPeter Zijlstra 
1895a1bd02e1SQais Yousef 	/*
1896a1bd02e1SQais Yousef 	 * If we're on asym system ensure we consider the different capacities
1897a1bd02e1SQais Yousef 	 * of the CPUs when searching for the lowest_mask.
1898a1bd02e1SQais Yousef 	 */
1899740cf8a7SDietmar Eggemann 	if (sched_asym_cpucap_active()) {
1900a1bd02e1SQais Yousef 
1901a1bd02e1SQais Yousef 		ret = cpupri_find_fitness(&task_rq(task)->rd->cpupri,
1902a1bd02e1SQais Yousef 					  task, lowest_mask,
1903a1bd02e1SQais Yousef 					  rt_task_fits_capacity);
1904a1bd02e1SQais Yousef 	} else {
1905a1bd02e1SQais Yousef 
1906a1bd02e1SQais Yousef 		ret = cpupri_find(&task_rq(task)->rd->cpupri,
1907a1bd02e1SQais Yousef 				  task, lowest_mask);
1908a1bd02e1SQais Yousef 	}
1909a1bd02e1SQais Yousef 
1910a1bd02e1SQais Yousef 	if (!ret)
1911391e43daSPeter Zijlstra 		return -1; /* No targets found */
1912391e43daSPeter Zijlstra 
1913391e43daSPeter Zijlstra 	/*
191497fb7a0aSIngo Molnar 	 * At this point we have built a mask of CPUs representing the
1915391e43daSPeter Zijlstra 	 * lowest priority tasks in the system.  Now we want to elect
1916391e43daSPeter Zijlstra 	 * the best one based on our affinity and topology.
1917391e43daSPeter Zijlstra 	 *
191897fb7a0aSIngo Molnar 	 * We prioritize the last CPU that the task executed on since
1919391e43daSPeter Zijlstra 	 * it is most likely cache-hot in that location.
1920391e43daSPeter Zijlstra 	 */
1921391e43daSPeter Zijlstra 	if (cpumask_test_cpu(cpu, lowest_mask))
1922391e43daSPeter Zijlstra 		return cpu;
1923391e43daSPeter Zijlstra 
1924391e43daSPeter Zijlstra 	/*
1925391e43daSPeter Zijlstra 	 * Otherwise, we consult the sched_domains span maps to figure
192697fb7a0aSIngo Molnar 	 * out which CPU is logically closest to our hot cache data.
1927391e43daSPeter Zijlstra 	 */
1928391e43daSPeter Zijlstra 	if (!cpumask_test_cpu(this_cpu, lowest_mask))
1929391e43daSPeter Zijlstra 		this_cpu = -1; /* Skip this_cpu opt if not among lowest */
1930391e43daSPeter Zijlstra 
1931391e43daSPeter Zijlstra 	rcu_read_lock();
1932391e43daSPeter Zijlstra 	for_each_domain(cpu, sd) {
1933391e43daSPeter Zijlstra 		if (sd->flags & SD_WAKE_AFFINE) {
1934391e43daSPeter Zijlstra 			int best_cpu;
1935391e43daSPeter Zijlstra 
1936391e43daSPeter Zijlstra 			/*
1937391e43daSPeter Zijlstra 			 * "this_cpu" is cheaper to preempt than a
1938391e43daSPeter Zijlstra 			 * remote processor.
1939391e43daSPeter Zijlstra 			 */
1940391e43daSPeter Zijlstra 			if (this_cpu != -1 &&
1941391e43daSPeter Zijlstra 			    cpumask_test_cpu(this_cpu, sched_domain_span(sd))) {
1942391e43daSPeter Zijlstra 				rcu_read_unlock();
1943391e43daSPeter Zijlstra 				return this_cpu;
1944391e43daSPeter Zijlstra 			}
1945391e43daSPeter Zijlstra 
194614e292f8SPeter Zijlstra 			best_cpu = cpumask_any_and_distribute(lowest_mask,
1947391e43daSPeter Zijlstra 							      sched_domain_span(sd));
1948391e43daSPeter Zijlstra 			if (best_cpu < nr_cpu_ids) {
1949391e43daSPeter Zijlstra 				rcu_read_unlock();
1950391e43daSPeter Zijlstra 				return best_cpu;
1951391e43daSPeter Zijlstra 			}
1952391e43daSPeter Zijlstra 		}
1953391e43daSPeter Zijlstra 	}
1954391e43daSPeter Zijlstra 	rcu_read_unlock();
1955391e43daSPeter Zijlstra 
1956391e43daSPeter Zijlstra 	/*
1957391e43daSPeter Zijlstra 	 * And finally, if there were no matches within the domains
1958391e43daSPeter Zijlstra 	 * just give the caller *something* to work with from the compatible
1959391e43daSPeter Zijlstra 	 * locations.
1960391e43daSPeter Zijlstra 	 */
1961391e43daSPeter Zijlstra 	if (this_cpu != -1)
1962391e43daSPeter Zijlstra 		return this_cpu;
1963391e43daSPeter Zijlstra 
196414e292f8SPeter Zijlstra 	cpu = cpumask_any_distribute(lowest_mask);
1965391e43daSPeter Zijlstra 	if (cpu < nr_cpu_ids)
1966391e43daSPeter Zijlstra 		return cpu;
196797fb7a0aSIngo Molnar 
1968391e43daSPeter Zijlstra 	return -1;
1969391e43daSPeter Zijlstra }
1970391e43daSPeter Zijlstra 
1971391e43daSPeter Zijlstra /* Will lock the rq it finds */
1972391e43daSPeter Zijlstra static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq)
1973391e43daSPeter Zijlstra {
1974391e43daSPeter Zijlstra 	struct rq *lowest_rq = NULL;
1975391e43daSPeter Zijlstra 	int tries;
1976391e43daSPeter Zijlstra 	int cpu;
1977391e43daSPeter Zijlstra 
1978391e43daSPeter Zijlstra 	for (tries = 0; tries < RT_MAX_TRIES; tries++) {
1979391e43daSPeter Zijlstra 		cpu = find_lowest_rq(task);
1980391e43daSPeter Zijlstra 
1981391e43daSPeter Zijlstra 		if ((cpu == -1) || (cpu == rq->cpu))
1982391e43daSPeter Zijlstra 			break;
1983391e43daSPeter Zijlstra 
1984391e43daSPeter Zijlstra 		lowest_rq = cpu_rq(cpu);
1985391e43daSPeter Zijlstra 
198680e3d87bSTim Chen 		if (lowest_rq->rt.highest_prio.curr <= task->prio) {
198780e3d87bSTim Chen 			/*
198880e3d87bSTim Chen 			 * Target rq has tasks of equal or higher priority,
198980e3d87bSTim Chen 			 * retrying does not release any lock and is unlikely
199080e3d87bSTim Chen 			 * to yield a different result.
199180e3d87bSTim Chen 			 */
199280e3d87bSTim Chen 			lowest_rq = NULL;
199380e3d87bSTim Chen 			break;
199480e3d87bSTim Chen 		}
199580e3d87bSTim Chen 
1996391e43daSPeter Zijlstra 		/* if the prio of this runqueue changed, try again */
1997391e43daSPeter Zijlstra 		if (double_lock_balance(rq, lowest_rq)) {
1998391e43daSPeter Zijlstra 			/*
1999391e43daSPeter Zijlstra 			 * We had to unlock the run queue. In
2000391e43daSPeter Zijlstra 			 * the mean time, task could have
2001391e43daSPeter Zijlstra 			 * migrated already or had its affinity changed.
2002391e43daSPeter Zijlstra 			 * Also make sure that it wasn't scheduled on its rq.
2003feffe5bbSSchspa Shi 			 * It is possible the task was scheduled, set
2004feffe5bbSSchspa Shi 			 * "migrate_disabled" and then got preempted, so we must
2005feffe5bbSSchspa Shi 			 * check the task migration disable flag here too.
2006391e43daSPeter Zijlstra 			 */
2007391e43daSPeter Zijlstra 			if (unlikely(task_rq(task) != rq ||
200895158a89SPeter Zijlstra 				     !cpumask_test_cpu(lowest_rq->cpu, &task->cpus_mask) ||
20090b9d46fcSPeter Zijlstra 				     task_on_cpu(rq, task) ||
201013b5ab02SXunlei Pang 				     !rt_task(task) ||
2011feffe5bbSSchspa Shi 				     is_migration_disabled(task) ||
2012da0c1e65SKirill Tkhai 				     !task_on_rq_queued(task))) {
2013391e43daSPeter Zijlstra 
20147f1b4393SPeter Zijlstra 				double_unlock_balance(rq, lowest_rq);
2015391e43daSPeter Zijlstra 				lowest_rq = NULL;
2016391e43daSPeter Zijlstra 				break;
2017391e43daSPeter Zijlstra 			}
2018391e43daSPeter Zijlstra 		}
2019391e43daSPeter Zijlstra 
2020391e43daSPeter Zijlstra 		/* If this rq is still suitable use it. */
2021391e43daSPeter Zijlstra 		if (lowest_rq->rt.highest_prio.curr > task->prio)
2022391e43daSPeter Zijlstra 			break;
2023391e43daSPeter Zijlstra 
2024391e43daSPeter Zijlstra 		/* try again */
2025391e43daSPeter Zijlstra 		double_unlock_balance(rq, lowest_rq);
2026391e43daSPeter Zijlstra 		lowest_rq = NULL;
2027391e43daSPeter Zijlstra 	}
2028391e43daSPeter Zijlstra 
2029391e43daSPeter Zijlstra 	return lowest_rq;
2030391e43daSPeter Zijlstra }
2031391e43daSPeter Zijlstra 
2032391e43daSPeter Zijlstra static struct task_struct *pick_next_pushable_task(struct rq *rq)
2033391e43daSPeter Zijlstra {
2034391e43daSPeter Zijlstra 	struct task_struct *p;
2035391e43daSPeter Zijlstra 
2036391e43daSPeter Zijlstra 	if (!has_pushable_tasks(rq))
2037391e43daSPeter Zijlstra 		return NULL;
2038391e43daSPeter Zijlstra 
2039391e43daSPeter Zijlstra 	p = plist_first_entry(&rq->rt.pushable_tasks,
2040391e43daSPeter Zijlstra 			      struct task_struct, pushable_tasks);
2041391e43daSPeter Zijlstra 
2042391e43daSPeter Zijlstra 	BUG_ON(rq->cpu != task_cpu(p));
2043391e43daSPeter Zijlstra 	BUG_ON(task_current(rq, p));
20444b53a341SIngo Molnar 	BUG_ON(p->nr_cpus_allowed <= 1);
2045391e43daSPeter Zijlstra 
2046da0c1e65SKirill Tkhai 	BUG_ON(!task_on_rq_queued(p));
2047391e43daSPeter Zijlstra 	BUG_ON(!rt_task(p));
2048391e43daSPeter Zijlstra 
2049391e43daSPeter Zijlstra 	return p;
2050391e43daSPeter Zijlstra }
2051391e43daSPeter Zijlstra 
2052391e43daSPeter Zijlstra /*
2053391e43daSPeter Zijlstra  * If the current CPU has more than one RT task, see if the non
2054391e43daSPeter Zijlstra  * running task can migrate over to a CPU that is running a task
2055391e43daSPeter Zijlstra  * of lesser priority.
2056391e43daSPeter Zijlstra  */
2057a7c81556SPeter Zijlstra static int push_rt_task(struct rq *rq, bool pull)
2058391e43daSPeter Zijlstra {
2059391e43daSPeter Zijlstra 	struct task_struct *next_task;
2060391e43daSPeter Zijlstra 	struct rq *lowest_rq;
2061391e43daSPeter Zijlstra 	int ret = 0;
2062391e43daSPeter Zijlstra 
2063391e43daSPeter Zijlstra 	if (!rq->rt.overloaded)
2064391e43daSPeter Zijlstra 		return 0;
2065391e43daSPeter Zijlstra 
2066391e43daSPeter Zijlstra 	next_task = pick_next_pushable_task(rq);
2067391e43daSPeter Zijlstra 	if (!next_task)
2068391e43daSPeter Zijlstra 		return 0;
2069391e43daSPeter Zijlstra 
2070391e43daSPeter Zijlstra retry:
207149bef33eSValentin Schneider 	/*
207249bef33eSValentin Schneider 	 * It's possible that the next_task slipped in of
207349bef33eSValentin Schneider 	 * higher priority than current. If that's the case
207449bef33eSValentin Schneider 	 * just reschedule current.
207549bef33eSValentin Schneider 	 */
207649bef33eSValentin Schneider 	if (unlikely(next_task->prio < rq->curr->prio)) {
207749bef33eSValentin Schneider 		resched_curr(rq);
207849bef33eSValentin Schneider 		return 0;
207949bef33eSValentin Schneider 	}
208049bef33eSValentin Schneider 
2081a7c81556SPeter Zijlstra 	if (is_migration_disabled(next_task)) {
2082a7c81556SPeter Zijlstra 		struct task_struct *push_task = NULL;
2083a7c81556SPeter Zijlstra 		int cpu;
2084a7c81556SPeter Zijlstra 
2085a7c81556SPeter Zijlstra 		if (!pull || rq->push_busy)
2086a7c81556SPeter Zijlstra 			return 0;
2087a7c81556SPeter Zijlstra 
208849bef33eSValentin Schneider 		/*
208949bef33eSValentin Schneider 		 * Invoking find_lowest_rq() on anything but an RT task doesn't
209049bef33eSValentin Schneider 		 * make sense. Per the above priority check, curr has to
209149bef33eSValentin Schneider 		 * be of higher priority than next_task, so no need to
209249bef33eSValentin Schneider 		 * reschedule when bailing out.
209349bef33eSValentin Schneider 		 *
209449bef33eSValentin Schneider 		 * Note that the stoppers are masqueraded as SCHED_FIFO
209549bef33eSValentin Schneider 		 * (cf. sched_set_stop_task()), so we can't rely on rt_task().
209649bef33eSValentin Schneider 		 */
209749bef33eSValentin Schneider 		if (rq->curr->sched_class != &rt_sched_class)
209849bef33eSValentin Schneider 			return 0;
209949bef33eSValentin Schneider 
2100a7c81556SPeter Zijlstra 		cpu = find_lowest_rq(rq->curr);
2101a7c81556SPeter Zijlstra 		if (cpu == -1 || cpu == rq->cpu)
2102a7c81556SPeter Zijlstra 			return 0;
2103a7c81556SPeter Zijlstra 
2104a7c81556SPeter Zijlstra 		/*
2105a7c81556SPeter Zijlstra 		 * Given we found a CPU with lower priority than @next_task,
2106a7c81556SPeter Zijlstra 		 * therefore it should be running. However we cannot migrate it
2107a7c81556SPeter Zijlstra 		 * to this other CPU, instead attempt to push the current
2108a7c81556SPeter Zijlstra 		 * running task on this CPU away.
2109a7c81556SPeter Zijlstra 		 */
2110a7c81556SPeter Zijlstra 		push_task = get_push_task(rq);
2111a7c81556SPeter Zijlstra 		if (push_task) {
21125cb9eaa3SPeter Zijlstra 			raw_spin_rq_unlock(rq);
2113a7c81556SPeter Zijlstra 			stop_one_cpu_nowait(rq->cpu, push_cpu_stop,
2114a7c81556SPeter Zijlstra 					    push_task, &rq->push_work);
21155cb9eaa3SPeter Zijlstra 			raw_spin_rq_lock(rq);
2116a7c81556SPeter Zijlstra 		}
2117a7c81556SPeter Zijlstra 
2118a7c81556SPeter Zijlstra 		return 0;
2119a7c81556SPeter Zijlstra 	}
2120a7c81556SPeter Zijlstra 
21219ebc6053SYangtao Li 	if (WARN_ON(next_task == rq->curr))
2122391e43daSPeter Zijlstra 		return 0;
2123391e43daSPeter Zijlstra 
2124391e43daSPeter Zijlstra 	/* We might release rq lock */
2125391e43daSPeter Zijlstra 	get_task_struct(next_task);
2126391e43daSPeter Zijlstra 
2127391e43daSPeter Zijlstra 	/* find_lock_lowest_rq locks the rq if found */
2128391e43daSPeter Zijlstra 	lowest_rq = find_lock_lowest_rq(next_task, rq);
2129391e43daSPeter Zijlstra 	if (!lowest_rq) {
2130391e43daSPeter Zijlstra 		struct task_struct *task;
2131391e43daSPeter Zijlstra 		/*
2132391e43daSPeter Zijlstra 		 * find_lock_lowest_rq releases rq->lock
2133391e43daSPeter Zijlstra 		 * so it is possible that next_task has migrated.
2134391e43daSPeter Zijlstra 		 *
2135391e43daSPeter Zijlstra 		 * We need to make sure that the task is still on the same
2136391e43daSPeter Zijlstra 		 * run-queue and is also still the next task eligible for
2137391e43daSPeter Zijlstra 		 * pushing.
2138391e43daSPeter Zijlstra 		 */
2139391e43daSPeter Zijlstra 		task = pick_next_pushable_task(rq);
2140de16b91eSByungchul Park 		if (task == next_task) {
2141391e43daSPeter Zijlstra 			/*
2142391e43daSPeter Zijlstra 			 * The task hasn't migrated, and is still the next
2143391e43daSPeter Zijlstra 			 * eligible task, but we failed to find a run-queue
2144391e43daSPeter Zijlstra 			 * to push it to.  Do not retry in this case, since
214597fb7a0aSIngo Molnar 			 * other CPUs will pull from us when ready.
2146391e43daSPeter Zijlstra 			 */
2147391e43daSPeter Zijlstra 			goto out;
2148391e43daSPeter Zijlstra 		}
2149391e43daSPeter Zijlstra 
2150391e43daSPeter Zijlstra 		if (!task)
2151391e43daSPeter Zijlstra 			/* No more tasks, just exit */
2152391e43daSPeter Zijlstra 			goto out;
2153391e43daSPeter Zijlstra 
2154391e43daSPeter Zijlstra 		/*
2155391e43daSPeter Zijlstra 		 * Something has shifted, try again.
2156391e43daSPeter Zijlstra 		 */
2157391e43daSPeter Zijlstra 		put_task_struct(next_task);
2158391e43daSPeter Zijlstra 		next_task = task;
2159391e43daSPeter Zijlstra 		goto retry;
2160391e43daSPeter Zijlstra 	}
2161391e43daSPeter Zijlstra 
2162391e43daSPeter Zijlstra 	deactivate_task(rq, next_task, 0);
2163391e43daSPeter Zijlstra 	set_task_cpu(next_task, lowest_rq->cpu);
2164391e43daSPeter Zijlstra 	activate_task(lowest_rq, next_task, 0);
2165a7c81556SPeter Zijlstra 	resched_curr(lowest_rq);
2166391e43daSPeter Zijlstra 	ret = 1;
2167391e43daSPeter Zijlstra 
2168391e43daSPeter Zijlstra 	double_unlock_balance(rq, lowest_rq);
2169391e43daSPeter Zijlstra out:
2170391e43daSPeter Zijlstra 	put_task_struct(next_task);
2171391e43daSPeter Zijlstra 
2172391e43daSPeter Zijlstra 	return ret;
2173391e43daSPeter Zijlstra }
2174391e43daSPeter Zijlstra 
2175391e43daSPeter Zijlstra static void push_rt_tasks(struct rq *rq)
2176391e43daSPeter Zijlstra {
2177391e43daSPeter Zijlstra 	/* push_rt_task will return true if it moved an RT */
2178a7c81556SPeter Zijlstra 	while (push_rt_task(rq, false))
2179391e43daSPeter Zijlstra 		;
2180391e43daSPeter Zijlstra }
2181391e43daSPeter Zijlstra 
2182b6366f04SSteven Rostedt #ifdef HAVE_RT_PUSH_IPI
2183b6366f04SSteven Rostedt 
21843e777f99SSteven Rostedt (VMware) /*
21853e777f99SSteven Rostedt (VMware)  * When a high priority task schedules out from a CPU and a lower priority
21863e777f99SSteven Rostedt (VMware)  * task is scheduled in, a check is made to see if there's any RT tasks
21873e777f99SSteven Rostedt (VMware)  * on other CPUs that are waiting to run because a higher priority RT task
21883e777f99SSteven Rostedt (VMware)  * is currently running on its CPU. In this case, the CPU with multiple RT
21893e777f99SSteven Rostedt (VMware)  * tasks queued on it (overloaded) needs to be notified that a CPU has opened
21903e777f99SSteven Rostedt (VMware)  * up that may be able to run one of its non-running queued RT tasks.
21913e777f99SSteven Rostedt (VMware)  *
21924bdced5cSSteven Rostedt (Red Hat)  * All CPUs with overloaded RT tasks need to be notified as there is currently
21934bdced5cSSteven Rostedt (Red Hat)  * no way to know which of these CPUs have the highest priority task waiting
21944bdced5cSSteven Rostedt (Red Hat)  * to run. Instead of trying to take a spinlock on each of these CPUs,
21954bdced5cSSteven Rostedt (Red Hat)  * which has shown to cause large latency when done on machines with many
21964bdced5cSSteven Rostedt (Red Hat)  * CPUs, sending an IPI to the CPUs to have them push off the overloaded
21974bdced5cSSteven Rostedt (Red Hat)  * RT tasks waiting to run.
21983e777f99SSteven Rostedt (VMware)  *
21994bdced5cSSteven Rostedt (Red Hat)  * Just sending an IPI to each of the CPUs is also an issue, as on large
22004bdced5cSSteven Rostedt (Red Hat)  * count CPU machines, this can cause an IPI storm on a CPU, especially
22014bdced5cSSteven Rostedt (Red Hat)  * if its the only CPU with multiple RT tasks queued, and a large number
22024bdced5cSSteven Rostedt (Red Hat)  * of CPUs scheduling a lower priority task at the same time.
22033e777f99SSteven Rostedt (VMware)  *
22044bdced5cSSteven Rostedt (Red Hat)  * Each root domain has its own irq work function that can iterate over
22054bdced5cSSteven Rostedt (Red Hat)  * all CPUs with RT overloaded tasks. Since all CPUs with overloaded RT
22063b03706fSIngo Molnar  * task must be checked if there's one or many CPUs that are lowering
22074bdced5cSSteven Rostedt (Red Hat)  * their priority, there's a single irq work iterator that will try to
22084bdced5cSSteven Rostedt (Red Hat)  * push off RT tasks that are waiting to run.
22093e777f99SSteven Rostedt (VMware)  *
22104bdced5cSSteven Rostedt (Red Hat)  * When a CPU schedules a lower priority task, it will kick off the
22114bdced5cSSteven Rostedt (Red Hat)  * irq work iterator that will jump to each CPU with overloaded RT tasks.
22124bdced5cSSteven Rostedt (Red Hat)  * As it only takes the first CPU that schedules a lower priority task
22134bdced5cSSteven Rostedt (Red Hat)  * to start the process, the rto_start variable is incremented and if
22144bdced5cSSteven Rostedt (Red Hat)  * the atomic result is one, then that CPU will try to take the rto_lock.
22154bdced5cSSteven Rostedt (Red Hat)  * This prevents high contention on the lock as the process handles all
22164bdced5cSSteven Rostedt (Red Hat)  * CPUs scheduling lower priority tasks.
22173e777f99SSteven Rostedt (VMware)  *
22184bdced5cSSteven Rostedt (Red Hat)  * All CPUs that are scheduling a lower priority task will increment the
22194bdced5cSSteven Rostedt (Red Hat)  * rt_loop_next variable. This will make sure that the irq work iterator
22204bdced5cSSteven Rostedt (Red Hat)  * checks all RT overloaded CPUs whenever a CPU schedules a new lower
22214bdced5cSSteven Rostedt (Red Hat)  * priority task, even if the iterator is in the middle of a scan. Incrementing
22224bdced5cSSteven Rostedt (Red Hat)  * the rt_loop_next will cause the iterator to perform another scan.
22233e777f99SSteven Rostedt (VMware)  *
22243e777f99SSteven Rostedt (VMware)  */
2225ad0f1d9dSSteven Rostedt (VMware) static int rto_next_cpu(struct root_domain *rd)
2226b6366f04SSteven Rostedt {
22274bdced5cSSteven Rostedt (Red Hat) 	int next;
2228b6366f04SSteven Rostedt 	int cpu;
2229b6366f04SSteven Rostedt 
2230b6366f04SSteven Rostedt 	/*
22314bdced5cSSteven Rostedt (Red Hat) 	 * When starting the IPI RT pushing, the rto_cpu is set to -1,
22324bdced5cSSteven Rostedt (Red Hat) 	 * rt_next_cpu() will simply return the first CPU found in
22334bdced5cSSteven Rostedt (Red Hat) 	 * the rto_mask.
22344bdced5cSSteven Rostedt (Red Hat) 	 *
223597fb7a0aSIngo Molnar 	 * If rto_next_cpu() is called with rto_cpu is a valid CPU, it
22364bdced5cSSteven Rostedt (Red Hat) 	 * will return the next CPU found in the rto_mask.
22374bdced5cSSteven Rostedt (Red Hat) 	 *
22384bdced5cSSteven Rostedt (Red Hat) 	 * If there are no more CPUs left in the rto_mask, then a check is made
22394bdced5cSSteven Rostedt (Red Hat) 	 * against rto_loop and rto_loop_next. rto_loop is only updated with
22404bdced5cSSteven Rostedt (Red Hat) 	 * the rto_lock held, but any CPU may increment the rto_loop_next
22414bdced5cSSteven Rostedt (Red Hat) 	 * without any locking.
2242b6366f04SSteven Rostedt 	 */
22434bdced5cSSteven Rostedt (Red Hat) 	for (;;) {
22444bdced5cSSteven Rostedt (Red Hat) 
22454bdced5cSSteven Rostedt (Red Hat) 		/* When rto_cpu is -1 this acts like cpumask_first() */
22464bdced5cSSteven Rostedt (Red Hat) 		cpu = cpumask_next(rd->rto_cpu, rd->rto_mask);
22474bdced5cSSteven Rostedt (Red Hat) 
22484bdced5cSSteven Rostedt (Red Hat) 		rd->rto_cpu = cpu;
22494bdced5cSSteven Rostedt (Red Hat) 
22504bdced5cSSteven Rostedt (Red Hat) 		if (cpu < nr_cpu_ids)
22514bdced5cSSteven Rostedt (Red Hat) 			return cpu;
22524bdced5cSSteven Rostedt (Red Hat) 
22534bdced5cSSteven Rostedt (Red Hat) 		rd->rto_cpu = -1;
22544bdced5cSSteven Rostedt (Red Hat) 
22554bdced5cSSteven Rostedt (Red Hat) 		/*
22564bdced5cSSteven Rostedt (Red Hat) 		 * ACQUIRE ensures we see the @rto_mask changes
22574bdced5cSSteven Rostedt (Red Hat) 		 * made prior to the @next value observed.
22584bdced5cSSteven Rostedt (Red Hat) 		 *
22594bdced5cSSteven Rostedt (Red Hat) 		 * Matches WMB in rt_set_overload().
22604bdced5cSSteven Rostedt (Red Hat) 		 */
22614bdced5cSSteven Rostedt (Red Hat) 		next = atomic_read_acquire(&rd->rto_loop_next);
22624bdced5cSSteven Rostedt (Red Hat) 
22634bdced5cSSteven Rostedt (Red Hat) 		if (rd->rto_loop == next)
22644bdced5cSSteven Rostedt (Red Hat) 			break;
22654bdced5cSSteven Rostedt (Red Hat) 
22664bdced5cSSteven Rostedt (Red Hat) 		rd->rto_loop = next;
2267b6366f04SSteven Rostedt 	}
2268b6366f04SSteven Rostedt 
22694bdced5cSSteven Rostedt (Red Hat) 	return -1;
22704bdced5cSSteven Rostedt (Red Hat) }
2271b6366f04SSteven Rostedt 
22724bdced5cSSteven Rostedt (Red Hat) static inline bool rto_start_trylock(atomic_t *v)
22734bdced5cSSteven Rostedt (Red Hat) {
22744bdced5cSSteven Rostedt (Red Hat) 	return !atomic_cmpxchg_acquire(v, 0, 1);
22754bdced5cSSteven Rostedt (Red Hat) }
22764bdced5cSSteven Rostedt (Red Hat) 
22774bdced5cSSteven Rostedt (Red Hat) static inline void rto_start_unlock(atomic_t *v)
22784bdced5cSSteven Rostedt (Red Hat) {
22794bdced5cSSteven Rostedt (Red Hat) 	atomic_set_release(v, 0);
22804bdced5cSSteven Rostedt (Red Hat) }
22814bdced5cSSteven Rostedt (Red Hat) 
22824bdced5cSSteven Rostedt (Red Hat) static void tell_cpu_to_push(struct rq *rq)
22834bdced5cSSteven Rostedt (Red Hat) {
22844bdced5cSSteven Rostedt (Red Hat) 	int cpu = -1;
22854bdced5cSSteven Rostedt (Red Hat) 
22864bdced5cSSteven Rostedt (Red Hat) 	/* Keep the loop going if the IPI is currently active */
22874bdced5cSSteven Rostedt (Red Hat) 	atomic_inc(&rq->rd->rto_loop_next);
22884bdced5cSSteven Rostedt (Red Hat) 
22894bdced5cSSteven Rostedt (Red Hat) 	/* Only one CPU can initiate a loop at a time */
22904bdced5cSSteven Rostedt (Red Hat) 	if (!rto_start_trylock(&rq->rd->rto_loop_start))
2291b6366f04SSteven Rostedt 		return;
2292b6366f04SSteven Rostedt 
22934bdced5cSSteven Rostedt (Red Hat) 	raw_spin_lock(&rq->rd->rto_lock);
2294b6366f04SSteven Rostedt 
22954bdced5cSSteven Rostedt (Red Hat) 	/*
229697fb7a0aSIngo Molnar 	 * The rto_cpu is updated under the lock, if it has a valid CPU
22974bdced5cSSteven Rostedt (Red Hat) 	 * then the IPI is still running and will continue due to the
22984bdced5cSSteven Rostedt (Red Hat) 	 * update to loop_next, and nothing needs to be done here.
22994bdced5cSSteven Rostedt (Red Hat) 	 * Otherwise it is finishing up and an ipi needs to be sent.
23004bdced5cSSteven Rostedt (Red Hat) 	 */
23014bdced5cSSteven Rostedt (Red Hat) 	if (rq->rd->rto_cpu < 0)
2302ad0f1d9dSSteven Rostedt (VMware) 		cpu = rto_next_cpu(rq->rd);
23034bdced5cSSteven Rostedt (Red Hat) 
23044bdced5cSSteven Rostedt (Red Hat) 	raw_spin_unlock(&rq->rd->rto_lock);
23054bdced5cSSteven Rostedt (Red Hat) 
23064bdced5cSSteven Rostedt (Red Hat) 	rto_start_unlock(&rq->rd->rto_loop_start);
23074bdced5cSSteven Rostedt (Red Hat) 
2308364f5665SSteven Rostedt (VMware) 	if (cpu >= 0) {
2309364f5665SSteven Rostedt (VMware) 		/* Make sure the rd does not get freed while pushing */
2310364f5665SSteven Rostedt (VMware) 		sched_get_rd(rq->rd);
23114bdced5cSSteven Rostedt (Red Hat) 		irq_work_queue_on(&rq->rd->rto_push_work, cpu);
2312b6366f04SSteven Rostedt 	}
2313364f5665SSteven Rostedt (VMware) }
2314b6366f04SSteven Rostedt 
2315b6366f04SSteven Rostedt /* Called from hardirq context */
23164bdced5cSSteven Rostedt (Red Hat) void rto_push_irq_work_func(struct irq_work *work)
2317b6366f04SSteven Rostedt {
2318ad0f1d9dSSteven Rostedt (VMware) 	struct root_domain *rd =
2319ad0f1d9dSSteven Rostedt (VMware) 		container_of(work, struct root_domain, rto_push_work);
23204bdced5cSSteven Rostedt (Red Hat) 	struct rq *rq;
2321b6366f04SSteven Rostedt 	int cpu;
2322b6366f04SSteven Rostedt 
23234bdced5cSSteven Rostedt (Red Hat) 	rq = this_rq();
2324b6366f04SSteven Rostedt 
23254bdced5cSSteven Rostedt (Red Hat) 	/*
23264bdced5cSSteven Rostedt (Red Hat) 	 * We do not need to grab the lock to check for has_pushable_tasks.
23274bdced5cSSteven Rostedt (Red Hat) 	 * When it gets updated, a check is made if a push is possible.
23284bdced5cSSteven Rostedt (Red Hat) 	 */
2329b6366f04SSteven Rostedt 	if (has_pushable_tasks(rq)) {
23305cb9eaa3SPeter Zijlstra 		raw_spin_rq_lock(rq);
2331a7c81556SPeter Zijlstra 		while (push_rt_task(rq, true))
2332a7c81556SPeter Zijlstra 			;
23335cb9eaa3SPeter Zijlstra 		raw_spin_rq_unlock(rq);
2334b6366f04SSteven Rostedt 	}
2335b6366f04SSteven Rostedt 
2336ad0f1d9dSSteven Rostedt (VMware) 	raw_spin_lock(&rd->rto_lock);
23374bdced5cSSteven Rostedt (Red Hat) 
2338b6366f04SSteven Rostedt 	/* Pass the IPI to the next rt overloaded queue */
2339ad0f1d9dSSteven Rostedt (VMware) 	cpu = rto_next_cpu(rd);
2340b6366f04SSteven Rostedt 
2341ad0f1d9dSSteven Rostedt (VMware) 	raw_spin_unlock(&rd->rto_lock);
2342b6366f04SSteven Rostedt 
2343364f5665SSteven Rostedt (VMware) 	if (cpu < 0) {
2344364f5665SSteven Rostedt (VMware) 		sched_put_rd(rd);
2345b6366f04SSteven Rostedt 		return;
2346364f5665SSteven Rostedt (VMware) 	}
2347b6366f04SSteven Rostedt 
2348b6366f04SSteven Rostedt 	/* Try the next RT overloaded CPU */
2349ad0f1d9dSSteven Rostedt (VMware) 	irq_work_queue_on(&rd->rto_push_work, cpu);
2350b6366f04SSteven Rostedt }
2351b6366f04SSteven Rostedt #endif /* HAVE_RT_PUSH_IPI */
2352b6366f04SSteven Rostedt 
23538046d680SPeter Zijlstra static void pull_rt_task(struct rq *this_rq)
2354391e43daSPeter Zijlstra {
23558046d680SPeter Zijlstra 	int this_cpu = this_rq->cpu, cpu;
23568046d680SPeter Zijlstra 	bool resched = false;
2357a7c81556SPeter Zijlstra 	struct task_struct *p, *push_task;
2358391e43daSPeter Zijlstra 	struct rq *src_rq;
2359f73c52a5SSteven Rostedt 	int rt_overload_count = rt_overloaded(this_rq);
2360391e43daSPeter Zijlstra 
2361f73c52a5SSteven Rostedt 	if (likely(!rt_overload_count))
23628046d680SPeter Zijlstra 		return;
2363391e43daSPeter Zijlstra 
23647c3f2ab7SPeter Zijlstra 	/*
23657c3f2ab7SPeter Zijlstra 	 * Match the barrier from rt_set_overloaded; this guarantees that if we
23667c3f2ab7SPeter Zijlstra 	 * see overloaded we must also see the rto_mask bit.
23677c3f2ab7SPeter Zijlstra 	 */
23687c3f2ab7SPeter Zijlstra 	smp_rmb();
23697c3f2ab7SPeter Zijlstra 
2370f73c52a5SSteven Rostedt 	/* If we are the only overloaded CPU do nothing */
2371f73c52a5SSteven Rostedt 	if (rt_overload_count == 1 &&
2372f73c52a5SSteven Rostedt 	    cpumask_test_cpu(this_rq->cpu, this_rq->rd->rto_mask))
2373f73c52a5SSteven Rostedt 		return;
2374f73c52a5SSteven Rostedt 
2375b6366f04SSteven Rostedt #ifdef HAVE_RT_PUSH_IPI
2376b6366f04SSteven Rostedt 	if (sched_feat(RT_PUSH_IPI)) {
2377b6366f04SSteven Rostedt 		tell_cpu_to_push(this_rq);
23788046d680SPeter Zijlstra 		return;
2379b6366f04SSteven Rostedt 	}
2380b6366f04SSteven Rostedt #endif
2381b6366f04SSteven Rostedt 
2382391e43daSPeter Zijlstra 	for_each_cpu(cpu, this_rq->rd->rto_mask) {
2383391e43daSPeter Zijlstra 		if (this_cpu == cpu)
2384391e43daSPeter Zijlstra 			continue;
2385391e43daSPeter Zijlstra 
2386391e43daSPeter Zijlstra 		src_rq = cpu_rq(cpu);
2387391e43daSPeter Zijlstra 
2388391e43daSPeter Zijlstra 		/*
2389391e43daSPeter Zijlstra 		 * Don't bother taking the src_rq->lock if the next highest
2390391e43daSPeter Zijlstra 		 * task is known to be lower-priority than our current task.
2391391e43daSPeter Zijlstra 		 * This may look racy, but if this value is about to go
2392391e43daSPeter Zijlstra 		 * logically higher, the src_rq will push this task away.
2393391e43daSPeter Zijlstra 		 * And if its going logically lower, we do not care
2394391e43daSPeter Zijlstra 		 */
2395391e43daSPeter Zijlstra 		if (src_rq->rt.highest_prio.next >=
2396391e43daSPeter Zijlstra 		    this_rq->rt.highest_prio.curr)
2397391e43daSPeter Zijlstra 			continue;
2398391e43daSPeter Zijlstra 
2399391e43daSPeter Zijlstra 		/*
2400391e43daSPeter Zijlstra 		 * We can potentially drop this_rq's lock in
2401391e43daSPeter Zijlstra 		 * double_lock_balance, and another CPU could
2402391e43daSPeter Zijlstra 		 * alter this_rq
2403391e43daSPeter Zijlstra 		 */
2404a7c81556SPeter Zijlstra 		push_task = NULL;
2405391e43daSPeter Zijlstra 		double_lock_balance(this_rq, src_rq);
2406391e43daSPeter Zijlstra 
2407391e43daSPeter Zijlstra 		/*
2408e23ee747SKirill Tkhai 		 * We can pull only a task, which is pushable
2409e23ee747SKirill Tkhai 		 * on its rq, and no others.
2410391e43daSPeter Zijlstra 		 */
2411e23ee747SKirill Tkhai 		p = pick_highest_pushable_task(src_rq, this_cpu);
2412391e43daSPeter Zijlstra 
2413391e43daSPeter Zijlstra 		/*
2414391e43daSPeter Zijlstra 		 * Do we have an RT task that preempts
2415391e43daSPeter Zijlstra 		 * the to-be-scheduled task?
2416391e43daSPeter Zijlstra 		 */
2417391e43daSPeter Zijlstra 		if (p && (p->prio < this_rq->rt.highest_prio.curr)) {
2418391e43daSPeter Zijlstra 			WARN_ON(p == src_rq->curr);
2419da0c1e65SKirill Tkhai 			WARN_ON(!task_on_rq_queued(p));
2420391e43daSPeter Zijlstra 
2421391e43daSPeter Zijlstra 			/*
2422391e43daSPeter Zijlstra 			 * There's a chance that p is higher in priority
242397fb7a0aSIngo Molnar 			 * than what's currently running on its CPU.
24243b03706fSIngo Molnar 			 * This is just that p is waking up and hasn't
2425391e43daSPeter Zijlstra 			 * had a chance to schedule. We only pull
2426391e43daSPeter Zijlstra 			 * p if it is lower in priority than the
2427391e43daSPeter Zijlstra 			 * current task on the run queue
2428391e43daSPeter Zijlstra 			 */
2429391e43daSPeter Zijlstra 			if (p->prio < src_rq->curr->prio)
2430391e43daSPeter Zijlstra 				goto skip;
2431391e43daSPeter Zijlstra 
2432a7c81556SPeter Zijlstra 			if (is_migration_disabled(p)) {
2433a7c81556SPeter Zijlstra 				push_task = get_push_task(src_rq);
2434a7c81556SPeter Zijlstra 			} else {
2435391e43daSPeter Zijlstra 				deactivate_task(src_rq, p, 0);
2436391e43daSPeter Zijlstra 				set_task_cpu(p, this_cpu);
2437391e43daSPeter Zijlstra 				activate_task(this_rq, p, 0);
2438a7c81556SPeter Zijlstra 				resched = true;
2439a7c81556SPeter Zijlstra 			}
2440391e43daSPeter Zijlstra 			/*
2441391e43daSPeter Zijlstra 			 * We continue with the search, just in
2442391e43daSPeter Zijlstra 			 * case there's an even higher prio task
2443391e43daSPeter Zijlstra 			 * in another runqueue. (low likelihood
2444391e43daSPeter Zijlstra 			 * but possible)
2445391e43daSPeter Zijlstra 			 */
2446391e43daSPeter Zijlstra 		}
2447391e43daSPeter Zijlstra skip:
2448391e43daSPeter Zijlstra 		double_unlock_balance(this_rq, src_rq);
2449a7c81556SPeter Zijlstra 
2450a7c81556SPeter Zijlstra 		if (push_task) {
24515cb9eaa3SPeter Zijlstra 			raw_spin_rq_unlock(this_rq);
2452a7c81556SPeter Zijlstra 			stop_one_cpu_nowait(src_rq->cpu, push_cpu_stop,
2453a7c81556SPeter Zijlstra 					    push_task, &src_rq->push_work);
24545cb9eaa3SPeter Zijlstra 			raw_spin_rq_lock(this_rq);
2455a7c81556SPeter Zijlstra 		}
2456391e43daSPeter Zijlstra 	}
2457391e43daSPeter Zijlstra 
24588046d680SPeter Zijlstra 	if (resched)
24598046d680SPeter Zijlstra 		resched_curr(this_rq);
2460391e43daSPeter Zijlstra }
2461391e43daSPeter Zijlstra 
2462391e43daSPeter Zijlstra /*
2463391e43daSPeter Zijlstra  * If we are not running and we are not going to reschedule soon, we should
2464391e43daSPeter Zijlstra  * try to push tasks away now
2465391e43daSPeter Zijlstra  */
2466391e43daSPeter Zijlstra static void task_woken_rt(struct rq *rq, struct task_struct *p)
2467391e43daSPeter Zijlstra {
24680b9d46fcSPeter Zijlstra 	bool need_to_push = !task_on_cpu(rq, p) &&
2469391e43daSPeter Zijlstra 			    !test_tsk_need_resched(rq->curr) &&
24704b53a341SIngo Molnar 			    p->nr_cpus_allowed > 1 &&
24711baca4ceSJuri Lelli 			    (dl_task(rq->curr) || rt_task(rq->curr)) &&
24724b53a341SIngo Molnar 			    (rq->curr->nr_cpus_allowed < 2 ||
2473804d402fSQais Yousef 			     rq->curr->prio <= p->prio);
2474804d402fSQais Yousef 
2475d94a9df4SQais Yousef 	if (need_to_push)
2476391e43daSPeter Zijlstra 		push_rt_tasks(rq);
2477391e43daSPeter Zijlstra }
2478391e43daSPeter Zijlstra 
2479391e43daSPeter Zijlstra /* Assumes rq->lock is held */
2480391e43daSPeter Zijlstra static void rq_online_rt(struct rq *rq)
2481391e43daSPeter Zijlstra {
2482391e43daSPeter Zijlstra 	if (rq->rt.overloaded)
2483391e43daSPeter Zijlstra 		rt_set_overload(rq);
2484391e43daSPeter Zijlstra 
2485391e43daSPeter Zijlstra 	__enable_runtime(rq);
2486391e43daSPeter Zijlstra 
2487391e43daSPeter Zijlstra 	cpupri_set(&rq->rd->cpupri, rq->cpu, rq->rt.highest_prio.curr);
2488391e43daSPeter Zijlstra }
2489391e43daSPeter Zijlstra 
2490391e43daSPeter Zijlstra /* Assumes rq->lock is held */
2491391e43daSPeter Zijlstra static void rq_offline_rt(struct rq *rq)
2492391e43daSPeter Zijlstra {
2493391e43daSPeter Zijlstra 	if (rq->rt.overloaded)
2494391e43daSPeter Zijlstra 		rt_clear_overload(rq);
2495391e43daSPeter Zijlstra 
2496391e43daSPeter Zijlstra 	__disable_runtime(rq);
2497391e43daSPeter Zijlstra 
2498391e43daSPeter Zijlstra 	cpupri_set(&rq->rd->cpupri, rq->cpu, CPUPRI_INVALID);
2499391e43daSPeter Zijlstra }
2500391e43daSPeter Zijlstra 
2501391e43daSPeter Zijlstra /*
2502391e43daSPeter Zijlstra  * When switch from the rt queue, we bring ourselves to a position
2503391e43daSPeter Zijlstra  * that we might want to pull RT tasks from other runqueues.
2504391e43daSPeter Zijlstra  */
2505391e43daSPeter Zijlstra static void switched_from_rt(struct rq *rq, struct task_struct *p)
2506391e43daSPeter Zijlstra {
2507391e43daSPeter Zijlstra 	/*
2508391e43daSPeter Zijlstra 	 * If there are other RT tasks then we will reschedule
2509391e43daSPeter Zijlstra 	 * and the scheduling of the other RT tasks will handle
2510391e43daSPeter Zijlstra 	 * the balancing. But if we are the last RT task
2511391e43daSPeter Zijlstra 	 * we may need to handle the pulling of RT tasks
2512391e43daSPeter Zijlstra 	 * now.
2513391e43daSPeter Zijlstra 	 */
2514da0c1e65SKirill Tkhai 	if (!task_on_rq_queued(p) || rq->rt.rt_nr_running)
25151158ddb5SKirill Tkhai 		return;
25161158ddb5SKirill Tkhai 
251702d8ec94SIngo Molnar 	rt_queue_pull_task(rq);
2518391e43daSPeter Zijlstra }
2519391e43daSPeter Zijlstra 
252011c785b7SLi Zefan void __init init_sched_rt_class(void)
2521391e43daSPeter Zijlstra {
2522391e43daSPeter Zijlstra 	unsigned int i;
2523391e43daSPeter Zijlstra 
2524391e43daSPeter Zijlstra 	for_each_possible_cpu(i) {
2525391e43daSPeter Zijlstra 		zalloc_cpumask_var_node(&per_cpu(local_cpu_mask, i),
2526391e43daSPeter Zijlstra 					GFP_KERNEL, cpu_to_node(i));
2527391e43daSPeter Zijlstra 	}
2528391e43daSPeter Zijlstra }
2529391e43daSPeter Zijlstra #endif /* CONFIG_SMP */
2530391e43daSPeter Zijlstra 
2531391e43daSPeter Zijlstra /*
2532391e43daSPeter Zijlstra  * When switching a task to RT, we may overload the runqueue
2533391e43daSPeter Zijlstra  * with RT tasks. In this case we try to push them off to
2534391e43daSPeter Zijlstra  * other runqueues.
2535391e43daSPeter Zijlstra  */
2536391e43daSPeter Zijlstra static void switched_to_rt(struct rq *rq, struct task_struct *p)
2537391e43daSPeter Zijlstra {
2538391e43daSPeter Zijlstra 	/*
2539fecfcbc2SVincent Donnefort 	 * If we are running, update the avg_rt tracking, as the running time
2540fecfcbc2SVincent Donnefort 	 * will now on be accounted into the latter.
2541fecfcbc2SVincent Donnefort 	 */
2542fecfcbc2SVincent Donnefort 	if (task_current(rq, p)) {
2543fecfcbc2SVincent Donnefort 		update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 0);
2544fecfcbc2SVincent Donnefort 		return;
2545fecfcbc2SVincent Donnefort 	}
2546fecfcbc2SVincent Donnefort 
2547fecfcbc2SVincent Donnefort 	/*
2548fecfcbc2SVincent Donnefort 	 * If we are not running we may need to preempt the current
2549fecfcbc2SVincent Donnefort 	 * running task. If that current running task is also an RT task
2550391e43daSPeter Zijlstra 	 * then see if we can move to another run queue.
2551391e43daSPeter Zijlstra 	 */
2552fecfcbc2SVincent Donnefort 	if (task_on_rq_queued(p)) {
2553391e43daSPeter Zijlstra #ifdef CONFIG_SMP
2554d94a9df4SQais Yousef 		if (p->nr_cpus_allowed > 1 && rq->rt.overloaded)
255502d8ec94SIngo Molnar 			rt_queue_push_tasks(rq);
2556619bd4a7SSebastian Andrzej Siewior #endif /* CONFIG_SMP */
25572fe25826SPaul E. McKenney 		if (p->prio < rq->curr->prio && cpu_online(cpu_of(rq)))
25588875125eSKirill Tkhai 			resched_curr(rq);
2559391e43daSPeter Zijlstra 	}
2560391e43daSPeter Zijlstra }
2561391e43daSPeter Zijlstra 
2562391e43daSPeter Zijlstra /*
2563391e43daSPeter Zijlstra  * Priority of the task has changed. This may cause
2564391e43daSPeter Zijlstra  * us to initiate a push or pull.
2565391e43daSPeter Zijlstra  */
2566391e43daSPeter Zijlstra static void
2567391e43daSPeter Zijlstra prio_changed_rt(struct rq *rq, struct task_struct *p, int oldprio)
2568391e43daSPeter Zijlstra {
2569da0c1e65SKirill Tkhai 	if (!task_on_rq_queued(p))
2570391e43daSPeter Zijlstra 		return;
2571391e43daSPeter Zijlstra 
257265bcf072SHui Su 	if (task_current(rq, p)) {
2573391e43daSPeter Zijlstra #ifdef CONFIG_SMP
2574391e43daSPeter Zijlstra 		/*
2575391e43daSPeter Zijlstra 		 * If our priority decreases while running, we
2576391e43daSPeter Zijlstra 		 * may need to pull tasks to this runqueue.
2577391e43daSPeter Zijlstra 		 */
2578391e43daSPeter Zijlstra 		if (oldprio < p->prio)
257902d8ec94SIngo Molnar 			rt_queue_pull_task(rq);
2580fd7a4bedSPeter Zijlstra 
2581391e43daSPeter Zijlstra 		/*
2582391e43daSPeter Zijlstra 		 * If there's a higher priority task waiting to run
2583fd7a4bedSPeter Zijlstra 		 * then reschedule.
2584391e43daSPeter Zijlstra 		 */
2585fd7a4bedSPeter Zijlstra 		if (p->prio > rq->rt.highest_prio.curr)
25868875125eSKirill Tkhai 			resched_curr(rq);
2587391e43daSPeter Zijlstra #else
2588391e43daSPeter Zijlstra 		/* For UP simply resched on drop of prio */
2589391e43daSPeter Zijlstra 		if (oldprio < p->prio)
25908875125eSKirill Tkhai 			resched_curr(rq);
2591391e43daSPeter Zijlstra #endif /* CONFIG_SMP */
2592391e43daSPeter Zijlstra 	} else {
2593391e43daSPeter Zijlstra 		/*
2594391e43daSPeter Zijlstra 		 * This task is not running, but if it is
2595391e43daSPeter Zijlstra 		 * greater than the current running task
2596391e43daSPeter Zijlstra 		 * then reschedule.
2597391e43daSPeter Zijlstra 		 */
2598391e43daSPeter Zijlstra 		if (p->prio < rq->curr->prio)
25998875125eSKirill Tkhai 			resched_curr(rq);
2600391e43daSPeter Zijlstra 	}
2601391e43daSPeter Zijlstra }
2602391e43daSPeter Zijlstra 
2603b18b6a9cSNicolas Pitre #ifdef CONFIG_POSIX_TIMERS
2604391e43daSPeter Zijlstra static void watchdog(struct rq *rq, struct task_struct *p)
2605391e43daSPeter Zijlstra {
2606391e43daSPeter Zijlstra 	unsigned long soft, hard;
2607391e43daSPeter Zijlstra 
2608391e43daSPeter Zijlstra 	/* max may change after cur was read, this will be fixed next tick */
2609391e43daSPeter Zijlstra 	soft = task_rlimit(p, RLIMIT_RTTIME);
2610391e43daSPeter Zijlstra 	hard = task_rlimit_max(p, RLIMIT_RTTIME);
2611391e43daSPeter Zijlstra 
2612391e43daSPeter Zijlstra 	if (soft != RLIM_INFINITY) {
2613391e43daSPeter Zijlstra 		unsigned long next;
2614391e43daSPeter Zijlstra 
261557d2aa00SYing Xue 		if (p->rt.watchdog_stamp != jiffies) {
2616391e43daSPeter Zijlstra 			p->rt.timeout++;
261757d2aa00SYing Xue 			p->rt.watchdog_stamp = jiffies;
261857d2aa00SYing Xue 		}
261957d2aa00SYing Xue 
2620391e43daSPeter Zijlstra 		next = DIV_ROUND_UP(min(soft, hard), USEC_PER_SEC/HZ);
26213a245c0fSThomas Gleixner 		if (p->rt.timeout > next) {
26223a245c0fSThomas Gleixner 			posix_cputimers_rt_watchdog(&p->posix_cputimers,
26233a245c0fSThomas Gleixner 						    p->se.sum_exec_runtime);
26243a245c0fSThomas Gleixner 		}
2625391e43daSPeter Zijlstra 	}
2626391e43daSPeter Zijlstra }
2627b18b6a9cSNicolas Pitre #else
2628b18b6a9cSNicolas Pitre static inline void watchdog(struct rq *rq, struct task_struct *p) { }
2629b18b6a9cSNicolas Pitre #endif
2630391e43daSPeter Zijlstra 
2631d84b3131SFrederic Weisbecker /*
2632d84b3131SFrederic Weisbecker  * scheduler tick hitting a task of our scheduling class.
2633d84b3131SFrederic Weisbecker  *
2634d84b3131SFrederic Weisbecker  * NOTE: This function can be called remotely by the tick offload that
2635d84b3131SFrederic Weisbecker  * goes along full dynticks. Therefore no local assumption can be made
2636d84b3131SFrederic Weisbecker  * and everything must be accessed through the @rq and @curr passed in
2637d84b3131SFrederic Weisbecker  * parameters.
2638d84b3131SFrederic Weisbecker  */
2639391e43daSPeter Zijlstra static void task_tick_rt(struct rq *rq, struct task_struct *p, int queued)
2640391e43daSPeter Zijlstra {
2641454c7999SColin Cross 	struct sched_rt_entity *rt_se = &p->rt;
2642454c7999SColin Cross 
2643391e43daSPeter Zijlstra 	update_curr_rt(rq);
264423127296SVincent Guittot 	update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 1);
2645391e43daSPeter Zijlstra 
2646391e43daSPeter Zijlstra 	watchdog(rq, p);
2647391e43daSPeter Zijlstra 
2648391e43daSPeter Zijlstra 	/*
2649391e43daSPeter Zijlstra 	 * RR tasks need a special form of timeslice management.
2650391e43daSPeter Zijlstra 	 * FIFO tasks have no timeslices.
2651391e43daSPeter Zijlstra 	 */
2652391e43daSPeter Zijlstra 	if (p->policy != SCHED_RR)
2653391e43daSPeter Zijlstra 		return;
2654391e43daSPeter Zijlstra 
2655391e43daSPeter Zijlstra 	if (--p->rt.time_slice)
2656391e43daSPeter Zijlstra 		return;
2657391e43daSPeter Zijlstra 
2658ce0dbbbbSClark Williams 	p->rt.time_slice = sched_rr_timeslice;
2659391e43daSPeter Zijlstra 
2660391e43daSPeter Zijlstra 	/*
2661e9aa39bbSLi Bin 	 * Requeue to the end of queue if we (and all of our ancestors) are not
2662e9aa39bbSLi Bin 	 * the only element on the queue
2663391e43daSPeter Zijlstra 	 */
2664454c7999SColin Cross 	for_each_sched_rt_entity(rt_se) {
2665454c7999SColin Cross 		if (rt_se->run_list.prev != rt_se->run_list.next) {
2666391e43daSPeter Zijlstra 			requeue_task_rt(rq, p, 0);
26678aa6f0ebSKirill Tkhai 			resched_curr(rq);
2668454c7999SColin Cross 			return;
2669454c7999SColin Cross 		}
2670391e43daSPeter Zijlstra 	}
2671391e43daSPeter Zijlstra }
2672391e43daSPeter Zijlstra 
2673391e43daSPeter Zijlstra static unsigned int get_rr_interval_rt(struct rq *rq, struct task_struct *task)
2674391e43daSPeter Zijlstra {
2675391e43daSPeter Zijlstra 	/*
2676391e43daSPeter Zijlstra 	 * Time slice is 0 for SCHED_FIFO tasks
2677391e43daSPeter Zijlstra 	 */
2678391e43daSPeter Zijlstra 	if (task->policy == SCHED_RR)
2679ce0dbbbbSClark Williams 		return sched_rr_timeslice;
2680391e43daSPeter Zijlstra 	else
2681391e43daSPeter Zijlstra 		return 0;
2682391e43daSPeter Zijlstra }
2683391e43daSPeter Zijlstra 
2684530bfad1SHao Jia #ifdef CONFIG_SCHED_CORE
2685530bfad1SHao Jia static int task_is_throttled_rt(struct task_struct *p, int cpu)
2686530bfad1SHao Jia {
2687530bfad1SHao Jia 	struct rt_rq *rt_rq;
2688530bfad1SHao Jia 
2689530bfad1SHao Jia #ifdef CONFIG_RT_GROUP_SCHED
2690530bfad1SHao Jia 	rt_rq = task_group(p)->rt_rq[cpu];
2691530bfad1SHao Jia #else
2692530bfad1SHao Jia 	rt_rq = &cpu_rq(cpu)->rt;
2693530bfad1SHao Jia #endif
2694530bfad1SHao Jia 
2695530bfad1SHao Jia 	return rt_rq_throttled(rt_rq);
2696530bfad1SHao Jia }
2697530bfad1SHao Jia #endif
2698530bfad1SHao Jia 
269943c31ac0SPeter Zijlstra DEFINE_SCHED_CLASS(rt) = {
270043c31ac0SPeter Zijlstra 
2701391e43daSPeter Zijlstra 	.enqueue_task		= enqueue_task_rt,
2702391e43daSPeter Zijlstra 	.dequeue_task		= dequeue_task_rt,
2703391e43daSPeter Zijlstra 	.yield_task		= yield_task_rt,
2704391e43daSPeter Zijlstra 
2705391e43daSPeter Zijlstra 	.check_preempt_curr	= check_preempt_curr_rt,
2706391e43daSPeter Zijlstra 
2707391e43daSPeter Zijlstra 	.pick_next_task		= pick_next_task_rt,
2708391e43daSPeter Zijlstra 	.put_prev_task		= put_prev_task_rt,
270903b7fad1SPeter Zijlstra 	.set_next_task          = set_next_task_rt,
2710391e43daSPeter Zijlstra 
2711391e43daSPeter Zijlstra #ifdef CONFIG_SMP
27126e2df058SPeter Zijlstra 	.balance		= balance_rt,
271321f56ffeSPeter Zijlstra 	.pick_task		= pick_task_rt,
2714391e43daSPeter Zijlstra 	.select_task_rq		= select_task_rq_rt,
27156c37067eSPeter Zijlstra 	.set_cpus_allowed       = set_cpus_allowed_common,
2716391e43daSPeter Zijlstra 	.rq_online              = rq_online_rt,
2717391e43daSPeter Zijlstra 	.rq_offline             = rq_offline_rt,
2718391e43daSPeter Zijlstra 	.task_woken		= task_woken_rt,
2719391e43daSPeter Zijlstra 	.switched_from		= switched_from_rt,
2720a7c81556SPeter Zijlstra 	.find_lock_rq		= find_lock_lowest_rq,
2721391e43daSPeter Zijlstra #endif
2722391e43daSPeter Zijlstra 
2723391e43daSPeter Zijlstra 	.task_tick		= task_tick_rt,
2724391e43daSPeter Zijlstra 
2725391e43daSPeter Zijlstra 	.get_rr_interval	= get_rr_interval_rt,
2726391e43daSPeter Zijlstra 
2727391e43daSPeter Zijlstra 	.prio_changed		= prio_changed_rt,
2728391e43daSPeter Zijlstra 	.switched_to		= switched_to_rt,
27296e998916SStanislaw Gruszka 
27306e998916SStanislaw Gruszka 	.update_curr		= update_curr_rt,
2731982d9cdcSPatrick Bellasi 
2732530bfad1SHao Jia #ifdef CONFIG_SCHED_CORE
2733530bfad1SHao Jia 	.task_is_throttled	= task_is_throttled_rt,
2734530bfad1SHao Jia #endif
2735530bfad1SHao Jia 
2736982d9cdcSPatrick Bellasi #ifdef CONFIG_UCLAMP_TASK
2737982d9cdcSPatrick Bellasi 	.uclamp_enabled		= 1,
2738982d9cdcSPatrick Bellasi #endif
2739391e43daSPeter Zijlstra };
2740391e43daSPeter Zijlstra 
27418887cd99SNicolas Pitre #ifdef CONFIG_RT_GROUP_SCHED
27428887cd99SNicolas Pitre /*
27438887cd99SNicolas Pitre  * Ensure that the real time constraints are schedulable.
27448887cd99SNicolas Pitre  */
27458887cd99SNicolas Pitre static DEFINE_MUTEX(rt_constraints_mutex);
27468887cd99SNicolas Pitre 
27478887cd99SNicolas Pitre static inline int tg_has_rt_tasks(struct task_group *tg)
27488887cd99SNicolas Pitre {
2749b4fb015eSKonstantin Khlebnikov 	struct task_struct *task;
2750b4fb015eSKonstantin Khlebnikov 	struct css_task_iter it;
2751b4fb015eSKonstantin Khlebnikov 	int ret = 0;
27528887cd99SNicolas Pitre 
27538887cd99SNicolas Pitre 	/*
27548887cd99SNicolas Pitre 	 * Autogroups do not have RT tasks; see autogroup_create().
27558887cd99SNicolas Pitre 	 */
27568887cd99SNicolas Pitre 	if (task_group_is_autogroup(tg))
27578887cd99SNicolas Pitre 		return 0;
27588887cd99SNicolas Pitre 
2759b4fb015eSKonstantin Khlebnikov 	css_task_iter_start(&tg->css, 0, &it);
2760b4fb015eSKonstantin Khlebnikov 	while (!ret && (task = css_task_iter_next(&it)))
2761b4fb015eSKonstantin Khlebnikov 		ret |= rt_task(task);
2762b4fb015eSKonstantin Khlebnikov 	css_task_iter_end(&it);
27638887cd99SNicolas Pitre 
2764b4fb015eSKonstantin Khlebnikov 	return ret;
27658887cd99SNicolas Pitre }
27668887cd99SNicolas Pitre 
27678887cd99SNicolas Pitre struct rt_schedulable_data {
27688887cd99SNicolas Pitre 	struct task_group *tg;
27698887cd99SNicolas Pitre 	u64 rt_period;
27708887cd99SNicolas Pitre 	u64 rt_runtime;
27718887cd99SNicolas Pitre };
27728887cd99SNicolas Pitre 
27738887cd99SNicolas Pitre static int tg_rt_schedulable(struct task_group *tg, void *data)
27748887cd99SNicolas Pitre {
27758887cd99SNicolas Pitre 	struct rt_schedulable_data *d = data;
27768887cd99SNicolas Pitre 	struct task_group *child;
27778887cd99SNicolas Pitre 	unsigned long total, sum = 0;
27788887cd99SNicolas Pitre 	u64 period, runtime;
27798887cd99SNicolas Pitre 
27808887cd99SNicolas Pitre 	period = ktime_to_ns(tg->rt_bandwidth.rt_period);
27818887cd99SNicolas Pitre 	runtime = tg->rt_bandwidth.rt_runtime;
27828887cd99SNicolas Pitre 
27838887cd99SNicolas Pitre 	if (tg == d->tg) {
27848887cd99SNicolas Pitre 		period = d->rt_period;
27858887cd99SNicolas Pitre 		runtime = d->rt_runtime;
27868887cd99SNicolas Pitre 	}
27878887cd99SNicolas Pitre 
27888887cd99SNicolas Pitre 	/*
27898887cd99SNicolas Pitre 	 * Cannot have more runtime than the period.
27908887cd99SNicolas Pitre 	 */
27918887cd99SNicolas Pitre 	if (runtime > period && runtime != RUNTIME_INF)
27928887cd99SNicolas Pitre 		return -EINVAL;
27938887cd99SNicolas Pitre 
27948887cd99SNicolas Pitre 	/*
2795b4fb015eSKonstantin Khlebnikov 	 * Ensure we don't starve existing RT tasks if runtime turns zero.
27968887cd99SNicolas Pitre 	 */
2797b4fb015eSKonstantin Khlebnikov 	if (rt_bandwidth_enabled() && !runtime &&
2798b4fb015eSKonstantin Khlebnikov 	    tg->rt_bandwidth.rt_runtime && tg_has_rt_tasks(tg))
27998887cd99SNicolas Pitre 		return -EBUSY;
28008887cd99SNicolas Pitre 
28018887cd99SNicolas Pitre 	total = to_ratio(period, runtime);
28028887cd99SNicolas Pitre 
28038887cd99SNicolas Pitre 	/*
28048887cd99SNicolas Pitre 	 * Nobody can have more than the global setting allows.
28058887cd99SNicolas Pitre 	 */
28068887cd99SNicolas Pitre 	if (total > to_ratio(global_rt_period(), global_rt_runtime()))
28078887cd99SNicolas Pitre 		return -EINVAL;
28088887cd99SNicolas Pitre 
28098887cd99SNicolas Pitre 	/*
28108887cd99SNicolas Pitre 	 * The sum of our children's runtime should not exceed our own.
28118887cd99SNicolas Pitre 	 */
28128887cd99SNicolas Pitre 	list_for_each_entry_rcu(child, &tg->children, siblings) {
28138887cd99SNicolas Pitre 		period = ktime_to_ns(child->rt_bandwidth.rt_period);
28148887cd99SNicolas Pitre 		runtime = child->rt_bandwidth.rt_runtime;
28158887cd99SNicolas Pitre 
28168887cd99SNicolas Pitre 		if (child == d->tg) {
28178887cd99SNicolas Pitre 			period = d->rt_period;
28188887cd99SNicolas Pitre 			runtime = d->rt_runtime;
28198887cd99SNicolas Pitre 		}
28208887cd99SNicolas Pitre 
28218887cd99SNicolas Pitre 		sum += to_ratio(period, runtime);
28228887cd99SNicolas Pitre 	}
28238887cd99SNicolas Pitre 
28248887cd99SNicolas Pitre 	if (sum > total)
28258887cd99SNicolas Pitre 		return -EINVAL;
28268887cd99SNicolas Pitre 
28278887cd99SNicolas Pitre 	return 0;
28288887cd99SNicolas Pitre }
28298887cd99SNicolas Pitre 
28308887cd99SNicolas Pitre static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
28318887cd99SNicolas Pitre {
28328887cd99SNicolas Pitre 	int ret;
28338887cd99SNicolas Pitre 
28348887cd99SNicolas Pitre 	struct rt_schedulable_data data = {
28358887cd99SNicolas Pitre 		.tg = tg,
28368887cd99SNicolas Pitre 		.rt_period = period,
28378887cd99SNicolas Pitre 		.rt_runtime = runtime,
28388887cd99SNicolas Pitre 	};
28398887cd99SNicolas Pitre 
28408887cd99SNicolas Pitre 	rcu_read_lock();
28418887cd99SNicolas Pitre 	ret = walk_tg_tree(tg_rt_schedulable, tg_nop, &data);
28428887cd99SNicolas Pitre 	rcu_read_unlock();
28438887cd99SNicolas Pitre 
28448887cd99SNicolas Pitre 	return ret;
28458887cd99SNicolas Pitre }
28468887cd99SNicolas Pitre 
28478887cd99SNicolas Pitre static int tg_set_rt_bandwidth(struct task_group *tg,
28488887cd99SNicolas Pitre 		u64 rt_period, u64 rt_runtime)
28498887cd99SNicolas Pitre {
28508887cd99SNicolas Pitre 	int i, err = 0;
28518887cd99SNicolas Pitre 
28528887cd99SNicolas Pitre 	/*
28538887cd99SNicolas Pitre 	 * Disallowing the root group RT runtime is BAD, it would disallow the
28548887cd99SNicolas Pitre 	 * kernel creating (and or operating) RT threads.
28558887cd99SNicolas Pitre 	 */
28568887cd99SNicolas Pitre 	if (tg == &root_task_group && rt_runtime == 0)
28578887cd99SNicolas Pitre 		return -EINVAL;
28588887cd99SNicolas Pitre 
28598887cd99SNicolas Pitre 	/* No period doesn't make any sense. */
28608887cd99SNicolas Pitre 	if (rt_period == 0)
28618887cd99SNicolas Pitre 		return -EINVAL;
28628887cd99SNicolas Pitre 
2863d505b8afSHuaixin Chang 	/*
2864d505b8afSHuaixin Chang 	 * Bound quota to defend quota against overflow during bandwidth shift.
2865d505b8afSHuaixin Chang 	 */
2866d505b8afSHuaixin Chang 	if (rt_runtime != RUNTIME_INF && rt_runtime > max_rt_runtime)
2867d505b8afSHuaixin Chang 		return -EINVAL;
2868d505b8afSHuaixin Chang 
28698887cd99SNicolas Pitre 	mutex_lock(&rt_constraints_mutex);
28708887cd99SNicolas Pitre 	err = __rt_schedulable(tg, rt_period, rt_runtime);
28718887cd99SNicolas Pitre 	if (err)
28728887cd99SNicolas Pitre 		goto unlock;
28738887cd99SNicolas Pitre 
28748887cd99SNicolas Pitre 	raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
28758887cd99SNicolas Pitre 	tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
28768887cd99SNicolas Pitre 	tg->rt_bandwidth.rt_runtime = rt_runtime;
28778887cd99SNicolas Pitre 
28788887cd99SNicolas Pitre 	for_each_possible_cpu(i) {
28798887cd99SNicolas Pitre 		struct rt_rq *rt_rq = tg->rt_rq[i];
28808887cd99SNicolas Pitre 
28818887cd99SNicolas Pitre 		raw_spin_lock(&rt_rq->rt_runtime_lock);
28828887cd99SNicolas Pitre 		rt_rq->rt_runtime = rt_runtime;
28838887cd99SNicolas Pitre 		raw_spin_unlock(&rt_rq->rt_runtime_lock);
28848887cd99SNicolas Pitre 	}
28858887cd99SNicolas Pitre 	raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
28868887cd99SNicolas Pitre unlock:
28878887cd99SNicolas Pitre 	mutex_unlock(&rt_constraints_mutex);
28888887cd99SNicolas Pitre 
28898887cd99SNicolas Pitre 	return err;
28908887cd99SNicolas Pitre }
28918887cd99SNicolas Pitre 
28928887cd99SNicolas Pitre int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
28938887cd99SNicolas Pitre {
28948887cd99SNicolas Pitre 	u64 rt_runtime, rt_period;
28958887cd99SNicolas Pitre 
28968887cd99SNicolas Pitre 	rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
28978887cd99SNicolas Pitre 	rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC;
28988887cd99SNicolas Pitre 	if (rt_runtime_us < 0)
28998887cd99SNicolas Pitre 		rt_runtime = RUNTIME_INF;
29001a010e29SKonstantin Khlebnikov 	else if ((u64)rt_runtime_us > U64_MAX / NSEC_PER_USEC)
29011a010e29SKonstantin Khlebnikov 		return -EINVAL;
29028887cd99SNicolas Pitre 
29038887cd99SNicolas Pitre 	return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
29048887cd99SNicolas Pitre }
29058887cd99SNicolas Pitre 
29068887cd99SNicolas Pitre long sched_group_rt_runtime(struct task_group *tg)
29078887cd99SNicolas Pitre {
29088887cd99SNicolas Pitre 	u64 rt_runtime_us;
29098887cd99SNicolas Pitre 
29108887cd99SNicolas Pitre 	if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF)
29118887cd99SNicolas Pitre 		return -1;
29128887cd99SNicolas Pitre 
29138887cd99SNicolas Pitre 	rt_runtime_us = tg->rt_bandwidth.rt_runtime;
29148887cd99SNicolas Pitre 	do_div(rt_runtime_us, NSEC_PER_USEC);
29158887cd99SNicolas Pitre 	return rt_runtime_us;
29168887cd99SNicolas Pitre }
29178887cd99SNicolas Pitre 
29188887cd99SNicolas Pitre int sched_group_set_rt_period(struct task_group *tg, u64 rt_period_us)
29198887cd99SNicolas Pitre {
29208887cd99SNicolas Pitre 	u64 rt_runtime, rt_period;
29218887cd99SNicolas Pitre 
29221a010e29SKonstantin Khlebnikov 	if (rt_period_us > U64_MAX / NSEC_PER_USEC)
29231a010e29SKonstantin Khlebnikov 		return -EINVAL;
29241a010e29SKonstantin Khlebnikov 
29258887cd99SNicolas Pitre 	rt_period = rt_period_us * NSEC_PER_USEC;
29268887cd99SNicolas Pitre 	rt_runtime = tg->rt_bandwidth.rt_runtime;
29278887cd99SNicolas Pitre 
29288887cd99SNicolas Pitre 	return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
29298887cd99SNicolas Pitre }
29308887cd99SNicolas Pitre 
29318887cd99SNicolas Pitre long sched_group_rt_period(struct task_group *tg)
29328887cd99SNicolas Pitre {
29338887cd99SNicolas Pitre 	u64 rt_period_us;
29348887cd99SNicolas Pitre 
29358887cd99SNicolas Pitre 	rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period);
29368887cd99SNicolas Pitre 	do_div(rt_period_us, NSEC_PER_USEC);
29378887cd99SNicolas Pitre 	return rt_period_us;
29388887cd99SNicolas Pitre }
29398887cd99SNicolas Pitre 
294028f152cdSBaisong Zhong #ifdef CONFIG_SYSCTL
29418887cd99SNicolas Pitre static int sched_rt_global_constraints(void)
29428887cd99SNicolas Pitre {
29438887cd99SNicolas Pitre 	int ret = 0;
29448887cd99SNicolas Pitre 
29458887cd99SNicolas Pitre 	mutex_lock(&rt_constraints_mutex);
29468887cd99SNicolas Pitre 	ret = __rt_schedulable(NULL, 0, 0);
29478887cd99SNicolas Pitre 	mutex_unlock(&rt_constraints_mutex);
29488887cd99SNicolas Pitre 
29498887cd99SNicolas Pitre 	return ret;
29508887cd99SNicolas Pitre }
295128f152cdSBaisong Zhong #endif /* CONFIG_SYSCTL */
29528887cd99SNicolas Pitre 
29538887cd99SNicolas Pitre int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk)
29548887cd99SNicolas Pitre {
29558887cd99SNicolas Pitre 	/* Don't accept realtime tasks when there is no way for them to run */
29568887cd99SNicolas Pitre 	if (rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0)
29578887cd99SNicolas Pitre 		return 0;
29588887cd99SNicolas Pitre 
29598887cd99SNicolas Pitre 	return 1;
29608887cd99SNicolas Pitre }
29618887cd99SNicolas Pitre 
29628887cd99SNicolas Pitre #else /* !CONFIG_RT_GROUP_SCHED */
296328f152cdSBaisong Zhong 
296428f152cdSBaisong Zhong #ifdef CONFIG_SYSCTL
29658887cd99SNicolas Pitre static int sched_rt_global_constraints(void)
29668887cd99SNicolas Pitre {
29678887cd99SNicolas Pitre 	unsigned long flags;
29688887cd99SNicolas Pitre 	int i;
29698887cd99SNicolas Pitre 
29708887cd99SNicolas Pitre 	raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
29718887cd99SNicolas Pitre 	for_each_possible_cpu(i) {
29728887cd99SNicolas Pitre 		struct rt_rq *rt_rq = &cpu_rq(i)->rt;
29738887cd99SNicolas Pitre 
29748887cd99SNicolas Pitre 		raw_spin_lock(&rt_rq->rt_runtime_lock);
29758887cd99SNicolas Pitre 		rt_rq->rt_runtime = global_rt_runtime();
29768887cd99SNicolas Pitre 		raw_spin_unlock(&rt_rq->rt_runtime_lock);
29778887cd99SNicolas Pitre 	}
29788887cd99SNicolas Pitre 	raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
29798887cd99SNicolas Pitre 
29808887cd99SNicolas Pitre 	return 0;
29818887cd99SNicolas Pitre }
298228f152cdSBaisong Zhong #endif /* CONFIG_SYSCTL */
29838887cd99SNicolas Pitre #endif /* CONFIG_RT_GROUP_SCHED */
29848887cd99SNicolas Pitre 
298528f152cdSBaisong Zhong #ifdef CONFIG_SYSCTL
29868887cd99SNicolas Pitre static int sched_rt_global_validate(void)
29878887cd99SNicolas Pitre {
29888887cd99SNicolas Pitre 	if (sysctl_sched_rt_period <= 0)
29898887cd99SNicolas Pitre 		return -EINVAL;
29908887cd99SNicolas Pitre 
29918887cd99SNicolas Pitre 	if ((sysctl_sched_rt_runtime != RUNTIME_INF) &&
2992d505b8afSHuaixin Chang 		((sysctl_sched_rt_runtime > sysctl_sched_rt_period) ||
2993d505b8afSHuaixin Chang 		 ((u64)sysctl_sched_rt_runtime *
2994d505b8afSHuaixin Chang 			NSEC_PER_USEC > max_rt_runtime)))
29958887cd99SNicolas Pitre 		return -EINVAL;
29968887cd99SNicolas Pitre 
29978887cd99SNicolas Pitre 	return 0;
29988887cd99SNicolas Pitre }
29998887cd99SNicolas Pitre 
30008887cd99SNicolas Pitre static void sched_rt_do_global(void)
30018887cd99SNicolas Pitre {
30029b58e976SLi Hua 	unsigned long flags;
30039b58e976SLi Hua 
30049b58e976SLi Hua 	raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
30058887cd99SNicolas Pitre 	def_rt_bandwidth.rt_runtime = global_rt_runtime();
30068887cd99SNicolas Pitre 	def_rt_bandwidth.rt_period = ns_to_ktime(global_rt_period());
30079b58e976SLi Hua 	raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
30088887cd99SNicolas Pitre }
30098887cd99SNicolas Pitre 
3010d9ab0e63SZhen Ni static int sched_rt_handler(struct ctl_table *table, int write, void *buffer,
301132927393SChristoph Hellwig 		size_t *lenp, loff_t *ppos)
30128887cd99SNicolas Pitre {
30138887cd99SNicolas Pitre 	int old_period, old_runtime;
30148887cd99SNicolas Pitre 	static DEFINE_MUTEX(mutex);
30158887cd99SNicolas Pitre 	int ret;
30168887cd99SNicolas Pitre 
30178887cd99SNicolas Pitre 	mutex_lock(&mutex);
30188887cd99SNicolas Pitre 	old_period = sysctl_sched_rt_period;
30198887cd99SNicolas Pitre 	old_runtime = sysctl_sched_rt_runtime;
30208887cd99SNicolas Pitre 
30218887cd99SNicolas Pitre 	ret = proc_dointvec(table, write, buffer, lenp, ppos);
30228887cd99SNicolas Pitre 
30238887cd99SNicolas Pitre 	if (!ret && write) {
30248887cd99SNicolas Pitre 		ret = sched_rt_global_validate();
30258887cd99SNicolas Pitre 		if (ret)
30268887cd99SNicolas Pitre 			goto undo;
30278887cd99SNicolas Pitre 
30288887cd99SNicolas Pitre 		ret = sched_dl_global_validate();
30298887cd99SNicolas Pitre 		if (ret)
30308887cd99SNicolas Pitre 			goto undo;
30318887cd99SNicolas Pitre 
30328887cd99SNicolas Pitre 		ret = sched_rt_global_constraints();
30338887cd99SNicolas Pitre 		if (ret)
30348887cd99SNicolas Pitre 			goto undo;
30358887cd99SNicolas Pitre 
30368887cd99SNicolas Pitre 		sched_rt_do_global();
30378887cd99SNicolas Pitre 		sched_dl_do_global();
30388887cd99SNicolas Pitre 	}
30398887cd99SNicolas Pitre 	if (0) {
30408887cd99SNicolas Pitre undo:
30418887cd99SNicolas Pitre 		sysctl_sched_rt_period = old_period;
30428887cd99SNicolas Pitre 		sysctl_sched_rt_runtime = old_runtime;
30438887cd99SNicolas Pitre 	}
30448887cd99SNicolas Pitre 	mutex_unlock(&mutex);
30458887cd99SNicolas Pitre 
30468887cd99SNicolas Pitre 	return ret;
30478887cd99SNicolas Pitre }
30488887cd99SNicolas Pitre 
3049dafd7a9dSZhen Ni static int sched_rr_handler(struct ctl_table *table, int write, void *buffer,
305032927393SChristoph Hellwig 		size_t *lenp, loff_t *ppos)
30518887cd99SNicolas Pitre {
30528887cd99SNicolas Pitre 	int ret;
30538887cd99SNicolas Pitre 	static DEFINE_MUTEX(mutex);
30548887cd99SNicolas Pitre 
30558887cd99SNicolas Pitre 	mutex_lock(&mutex);
30568887cd99SNicolas Pitre 	ret = proc_dointvec(table, write, buffer, lenp, ppos);
30578887cd99SNicolas Pitre 	/*
30588887cd99SNicolas Pitre 	 * Make sure that internally we keep jiffies.
30598887cd99SNicolas Pitre 	 * Also, writing zero resets the timeslice to default:
30608887cd99SNicolas Pitre 	 */
30618887cd99SNicolas Pitre 	if (!ret && write) {
30628887cd99SNicolas Pitre 		sched_rr_timeslice =
30638887cd99SNicolas Pitre 			sysctl_sched_rr_timeslice <= 0 ? RR_TIMESLICE :
30648887cd99SNicolas Pitre 			msecs_to_jiffies(sysctl_sched_rr_timeslice);
30658887cd99SNicolas Pitre 	}
30668887cd99SNicolas Pitre 	mutex_unlock(&mutex);
306797fb7a0aSIngo Molnar 
30688887cd99SNicolas Pitre 	return ret;
30698887cd99SNicolas Pitre }
307028f152cdSBaisong Zhong #endif /* CONFIG_SYSCTL */
30718887cd99SNicolas Pitre 
3072391e43daSPeter Zijlstra #ifdef CONFIG_SCHED_DEBUG
3073391e43daSPeter Zijlstra void print_rt_stats(struct seq_file *m, int cpu)
3074391e43daSPeter Zijlstra {
3075391e43daSPeter Zijlstra 	rt_rq_iter_t iter;
3076391e43daSPeter Zijlstra 	struct rt_rq *rt_rq;
3077391e43daSPeter Zijlstra 
3078391e43daSPeter Zijlstra 	rcu_read_lock();
3079391e43daSPeter Zijlstra 	for_each_rt_rq(rt_rq, iter, cpu_rq(cpu))
3080391e43daSPeter Zijlstra 		print_rt_rq(m, cpu, rt_rq);
3081391e43daSPeter Zijlstra 	rcu_read_unlock();
3082391e43daSPeter Zijlstra }
3083391e43daSPeter Zijlstra #endif /* CONFIG_SCHED_DEBUG */
3084