134 void init_rt_rq(struct rt_rq *rt_rq)  in init_rt_rq()  argument
139 	array = &rt_rq->active;  in init_rt_rq()
148 	rt_rq->highest_prio.curr = MAX_RT_PRIO-1;  in init_rt_rq()
149 	rt_rq->highest_prio.next = MAX_RT_PRIO-1;  in init_rt_rq()
150 	rt_rq->rt_nr_migratory = 0;  in init_rt_rq()
151 	rt_rq->overloaded = 0;  in init_rt_rq()
152 	plist_head_init(&rt_rq->pushable_tasks);  in init_rt_rq()
155 	rt_rq->rt_queued = 0;  in init_rt_rq()
157 	rt_rq->rt_time = 0;  in init_rt_rq()
158 	rt_rq->rt_throttled = 0;  in init_rt_rq()
159 	rt_rq->rt_runtime = 0;  in init_rt_rq()
160 	raw_spin_lock_init(&rt_rq->rt_runtime_lock);  in init_rt_rq()
179 static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)  in rq_of_rt_rq()  argument
181 	return rt_rq->rq;  in rq_of_rt_rq()
184 static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)  in rt_rq_of_se()
186 	return rt_se->rt_rq;  in rt_rq_of_se()
191 	struct rt_rq *rt_rq = rt_se->rt_rq;  in rq_of_rt_se()  local
193 	return rt_rq->rq;  in rq_of_rt_se()
208 		if (tg->rt_rq)  in free_rt_sched_group()
209 			kfree(tg->rt_rq[i]);  in free_rt_sched_group()
214 	kfree(tg->rt_rq);  in free_rt_sched_group()
218 void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,  in init_tg_rt_entry()  argument
224 	rt_rq->highest_prio.curr = MAX_RT_PRIO-1;  in init_tg_rt_entry()
225 	rt_rq->rt_nr_boosted = 0;  in init_tg_rt_entry()
226 	rt_rq->rq = rq;  in init_tg_rt_entry()
227 	rt_rq->tg = tg;  in init_tg_rt_entry()
229 	tg->rt_rq[cpu] = rt_rq;  in init_tg_rt_entry()
236 		rt_se->rt_rq = &rq->rt;  in init_tg_rt_entry()
238 		rt_se->rt_rq = parent->my_q;  in init_tg_rt_entry()
240 	rt_se->my_q = rt_rq;  in init_tg_rt_entry()
247 	struct rt_rq *rt_rq;  in alloc_rt_sched_group()  local
251 	tg->rt_rq = kcalloc(nr_cpu_ids, sizeof(rt_rq), GFP_KERNEL);  in alloc_rt_sched_group()
252 	if (!tg->rt_rq)  in alloc_rt_sched_group()
262 		rt_rq = kzalloc_node(sizeof(struct rt_rq),  in alloc_rt_sched_group()
264 		if (!rt_rq)  in alloc_rt_sched_group()
272 		init_rt_rq(rt_rq);  in alloc_rt_sched_group()
273 		rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime;  in alloc_rt_sched_group()
274 		init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]);  in alloc_rt_sched_group()
280 	kfree(rt_rq);  in alloc_rt_sched_group()
294 static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)  in rq_of_rt_rq()  argument
296 	return container_of(rt_rq, struct rq, rt);  in rq_of_rt_rq()
306 static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)  in rt_rq_of_se()
365 static void update_rt_migration(struct rt_rq *rt_rq)  in update_rt_migration()  argument
367 	if (rt_rq->rt_nr_migratory && rt_rq->rt_nr_total > 1) {  in update_rt_migration()
368 		if (!rt_rq->overloaded) {  in update_rt_migration()
369 			rt_set_overload(rq_of_rt_rq(rt_rq));  in update_rt_migration()
370 			rt_rq->overloaded = 1;  in update_rt_migration()
372 	} else if (rt_rq->overloaded) {  in update_rt_migration()
373 		rt_clear_overload(rq_of_rt_rq(rt_rq));  in update_rt_migration()
374 		rt_rq->overloaded = 0;  in update_rt_migration()
378 static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in inc_rt_migration()  argument
386 	rt_rq = &rq_of_rt_rq(rt_rq)->rt;  in inc_rt_migration()
388 	rt_rq->rt_nr_total++;  in inc_rt_migration()
390 		rt_rq->rt_nr_migratory++;  in inc_rt_migration()
392 	update_rt_migration(rt_rq);  in inc_rt_migration()
395 static void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in dec_rt_migration()  argument
403 	rt_rq = &rq_of_rt_rq(rt_rq)->rt;  in dec_rt_migration()
405 	rt_rq->rt_nr_total--;  in dec_rt_migration()
407 		rt_rq->rt_nr_migratory--;  in dec_rt_migration()
409 	update_rt_migration(rt_rq);  in dec_rt_migration()
472 void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in inc_rt_migration()  argument
477 void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in dec_rt_migration()  argument
486 static void enqueue_top_rt_rq(struct rt_rq *rt_rq);
487 static void dequeue_top_rt_rq(struct rt_rq *rt_rq, unsigned int count);
535 static inline u64 sched_rt_runtime(struct rt_rq *rt_rq)  in sched_rt_runtime()  argument
537 	if (!rt_rq->tg)  in sched_rt_runtime()
540 	return rt_rq->rt_runtime;  in sched_rt_runtime()
543 static inline u64 sched_rt_period(struct rt_rq *rt_rq)  in sched_rt_period()  argument
545 	return ktime_to_ns(rt_rq->tg->rt_bandwidth.rt_period);  in sched_rt_period()
563 #define for_each_rt_rq(rt_rq, iter, rq)					\  argument
566 		(rt_rq = iter->rt_rq[cpu_of(rq)]);)
571 static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se)  in group_rt_rq()
579 static void sched_rt_rq_enqueue(struct rt_rq *rt_rq)  in sched_rt_rq_enqueue()  argument
581 	struct task_struct *curr = rq_of_rt_rq(rt_rq)->curr;  in sched_rt_rq_enqueue()
582 	struct rq *rq = rq_of_rt_rq(rt_rq);  in sched_rt_rq_enqueue()
587 	rt_se = rt_rq->tg->rt_se[cpu];  in sched_rt_rq_enqueue()
589 	if (rt_rq->rt_nr_running) {  in sched_rt_rq_enqueue()
591 			enqueue_top_rt_rq(rt_rq);  in sched_rt_rq_enqueue()
595 		if (rt_rq->highest_prio.curr < curr->prio)  in sched_rt_rq_enqueue()
600 static void sched_rt_rq_dequeue(struct rt_rq *rt_rq)  in sched_rt_rq_dequeue()  argument
603 	int cpu = cpu_of(rq_of_rt_rq(rt_rq));  in sched_rt_rq_dequeue()
605 	rt_se = rt_rq->tg->rt_se[cpu];  in sched_rt_rq_dequeue()
608 		dequeue_top_rt_rq(rt_rq, rt_rq->rt_nr_running);  in sched_rt_rq_dequeue()
610 		cpufreq_update_util(rq_of_rt_rq(rt_rq), 0);  in sched_rt_rq_dequeue()
616 static inline int rt_rq_throttled(struct rt_rq *rt_rq)  in rt_rq_throttled()  argument
618 	return rt_rq->rt_throttled && !rt_rq->rt_nr_boosted;  in rt_rq_throttled()
623 	struct rt_rq *rt_rq = group_rt_rq(rt_se);  in rt_se_boosted()  local
626 	if (rt_rq)  in rt_se_boosted()
627 		return !!rt_rq->rt_nr_boosted;  in rt_se_boosted()
646 struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu)  in sched_rt_period_rt_rq()
648 	return container_of(rt_b, struct task_group, rt_bandwidth)->rt_rq[cpu];  in sched_rt_period_rt_rq()
651 static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq)  in sched_rt_bandwidth()  argument
653 	return &rt_rq->tg->rt_bandwidth;  in sched_rt_bandwidth()
658 static inline u64 sched_rt_runtime(struct rt_rq *rt_rq)  in sched_rt_runtime()  argument
660 	return rt_rq->rt_runtime;  in sched_rt_runtime()
663 static inline u64 sched_rt_period(struct rt_rq *rt_rq)  in sched_rt_period()  argument
668 typedef struct rt_rq *rt_rq_iter_t;
670 #define for_each_rt_rq(rt_rq, iter, rq) \  argument
671 	for ((void) iter, rt_rq = &rq->rt; rt_rq; rt_rq = NULL)
676 static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se)  in group_rt_rq()
681 static inline void sched_rt_rq_enqueue(struct rt_rq *rt_rq)  in sched_rt_rq_enqueue()  argument
683 	struct rq *rq = rq_of_rt_rq(rt_rq);  in sched_rt_rq_enqueue()
685 	if (!rt_rq->rt_nr_running)  in sched_rt_rq_enqueue()
688 	enqueue_top_rt_rq(rt_rq);  in sched_rt_rq_enqueue()
692 static inline void sched_rt_rq_dequeue(struct rt_rq *rt_rq)  in sched_rt_rq_dequeue()  argument
694 	dequeue_top_rt_rq(rt_rq, rt_rq->rt_nr_running);  in sched_rt_rq_dequeue()
697 static inline int rt_rq_throttled(struct rt_rq *rt_rq)  in rt_rq_throttled()  argument
699 	return rt_rq->rt_throttled;  in rt_rq_throttled()
708 struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu)  in sched_rt_period_rt_rq()
713 static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq)  in sched_rt_bandwidth()  argument
720 bool sched_rt_bandwidth_account(struct rt_rq *rt_rq)  in sched_rt_bandwidth_account()  argument
722 	struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);  in sched_rt_bandwidth_account()
725 		rt_rq->rt_time < rt_b->rt_runtime);  in sched_rt_bandwidth_account()
732 static void do_balance_runtime(struct rt_rq *rt_rq)  in do_balance_runtime()  argument
734 	struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);  in do_balance_runtime()
735 	struct root_domain *rd = rq_of_rt_rq(rt_rq)->rd;  in do_balance_runtime()
744 		struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);  in do_balance_runtime()
747 		if (iter == rt_rq)  in do_balance_runtime()
766 			if (rt_rq->rt_runtime + diff > rt_period)  in do_balance_runtime()
767 				diff = rt_period - rt_rq->rt_runtime;  in do_balance_runtime()
769 			rt_rq->rt_runtime += diff;  in do_balance_runtime()
770 			if (rt_rq->rt_runtime == rt_period) {  in do_balance_runtime()
788 	struct rt_rq *rt_rq;  in __disable_runtime()  local
793 	for_each_rt_rq(rt_rq, iter, rq) {  in __disable_runtime()
794 		struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);  in __disable_runtime()
799 		raw_spin_lock(&rt_rq->rt_runtime_lock);  in __disable_runtime()
805 		if (rt_rq->rt_runtime == RUNTIME_INF ||  in __disable_runtime()
806 				rt_rq->rt_runtime == rt_b->rt_runtime)  in __disable_runtime()
808 		raw_spin_unlock(&rt_rq->rt_runtime_lock);  in __disable_runtime()
815 		want = rt_b->rt_runtime - rt_rq->rt_runtime;  in __disable_runtime()
821 			struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);  in __disable_runtime()
827 			if (iter == rt_rq || iter->rt_runtime == RUNTIME_INF)  in __disable_runtime()
845 		raw_spin_lock(&rt_rq->rt_runtime_lock);  in __disable_runtime()
856 		rt_rq->rt_runtime = RUNTIME_INF;  in __disable_runtime()
857 		rt_rq->rt_throttled = 0;  in __disable_runtime()
858 		raw_spin_unlock(&rt_rq->rt_runtime_lock);  in __disable_runtime()
862 		sched_rt_rq_enqueue(rt_rq);  in __disable_runtime()
869 	struct rt_rq *rt_rq;  in __enable_runtime()  local
877 	for_each_rt_rq(rt_rq, iter, rq) {  in __enable_runtime()
878 		struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);  in __enable_runtime()
881 		raw_spin_lock(&rt_rq->rt_runtime_lock);  in __enable_runtime()
882 		rt_rq->rt_runtime = rt_b->rt_runtime;  in __enable_runtime()
883 		rt_rq->rt_time = 0;  in __enable_runtime()
884 		rt_rq->rt_throttled = 0;  in __enable_runtime()
885 		raw_spin_unlock(&rt_rq->rt_runtime_lock);  in __enable_runtime()
890 static void balance_runtime(struct rt_rq *rt_rq)  in balance_runtime()  argument
895 	if (rt_rq->rt_time > rt_rq->rt_runtime) {  in balance_runtime()
896 		raw_spin_unlock(&rt_rq->rt_runtime_lock);  in balance_runtime()
897 		do_balance_runtime(rt_rq);  in balance_runtime()
898 		raw_spin_lock(&rt_rq->rt_runtime_lock);  in balance_runtime()
902 static inline void balance_runtime(struct rt_rq *rt_rq) {}  in balance_runtime()  argument
926 		struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i);  in do_sched_rt_period_timer()  local
927 		struct rq *rq = rq_of_rt_rq(rt_rq);  in do_sched_rt_period_timer()
935 		raw_spin_lock(&rt_rq->rt_runtime_lock);  in do_sched_rt_period_timer()
936 		if (!sched_feat(RT_RUNTIME_SHARE) && rt_rq->rt_runtime != RUNTIME_INF)  in do_sched_rt_period_timer()
937 			rt_rq->rt_runtime = rt_b->rt_runtime;  in do_sched_rt_period_timer()
938 		skip = !rt_rq->rt_time && !rt_rq->rt_nr_running;  in do_sched_rt_period_timer()
939 		raw_spin_unlock(&rt_rq->rt_runtime_lock);  in do_sched_rt_period_timer()
946 		if (rt_rq->rt_time) {  in do_sched_rt_period_timer()
949 			raw_spin_lock(&rt_rq->rt_runtime_lock);  in do_sched_rt_period_timer()
950 			if (rt_rq->rt_throttled)  in do_sched_rt_period_timer()
951 				balance_runtime(rt_rq);  in do_sched_rt_period_timer()
952 			runtime = rt_rq->rt_runtime;  in do_sched_rt_period_timer()
953 			rt_rq->rt_time -= min(rt_rq->rt_time, overrun*runtime);  in do_sched_rt_period_timer()
954 			if (rt_rq->rt_throttled && rt_rq->rt_time < runtime) {  in do_sched_rt_period_timer()
955 				rt_rq->rt_throttled = 0;  in do_sched_rt_period_timer()
965 				if (rt_rq->rt_nr_running && rq->curr == rq->idle)  in do_sched_rt_period_timer()
968 			if (rt_rq->rt_time || rt_rq->rt_nr_running)  in do_sched_rt_period_timer()
970 			raw_spin_unlock(&rt_rq->rt_runtime_lock);  in do_sched_rt_period_timer()
971 		} else if (rt_rq->rt_nr_running) {  in do_sched_rt_period_timer()
973 			if (!rt_rq_throttled(rt_rq))  in do_sched_rt_period_timer()
976 		if (rt_rq->rt_throttled)  in do_sched_rt_period_timer()
980 			sched_rt_rq_enqueue(rt_rq);  in do_sched_rt_period_timer()
993 	struct rt_rq *rt_rq = group_rt_rq(rt_se);  in rt_se_prio()  local
995 	if (rt_rq)  in rt_se_prio()
996 		return rt_rq->highest_prio.curr;  in rt_se_prio()
1002 static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq)  in sched_rt_runtime_exceeded()  argument
1004 	u64 runtime = sched_rt_runtime(rt_rq);  in sched_rt_runtime_exceeded()
1006 	if (rt_rq->rt_throttled)  in sched_rt_runtime_exceeded()
1007 		return rt_rq_throttled(rt_rq);  in sched_rt_runtime_exceeded()
1009 	if (runtime >= sched_rt_period(rt_rq))  in sched_rt_runtime_exceeded()
1012 	balance_runtime(rt_rq);  in sched_rt_runtime_exceeded()
1013 	runtime = sched_rt_runtime(rt_rq);  in sched_rt_runtime_exceeded()
1017 	if (rt_rq->rt_time > runtime) {  in sched_rt_runtime_exceeded()
1018 		struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);  in sched_rt_runtime_exceeded()
1025 			rt_rq->rt_throttled = 1;  in sched_rt_runtime_exceeded()
1033 			rt_rq->rt_time = 0;  in sched_rt_runtime_exceeded()
1036 		if (rt_rq_throttled(rt_rq)) {  in sched_rt_runtime_exceeded()
1037 			sched_rt_rq_dequeue(rt_rq);  in sched_rt_runtime_exceeded()
1075 		struct rt_rq *rt_rq = rt_rq_of_se(rt_se);  in update_curr_rt()  local
1078 		if (sched_rt_runtime(rt_rq) != RUNTIME_INF) {  in update_curr_rt()
1079 			raw_spin_lock(&rt_rq->rt_runtime_lock);  in update_curr_rt()
1080 			rt_rq->rt_time += delta_exec;  in update_curr_rt()
1081 			exceeded = sched_rt_runtime_exceeded(rt_rq);  in update_curr_rt()
1084 			raw_spin_unlock(&rt_rq->rt_runtime_lock);  in update_curr_rt()
1086 				do_start_rt_bandwidth(sched_rt_bandwidth(rt_rq));  in update_curr_rt()
1092 dequeue_top_rt_rq(struct rt_rq *rt_rq, unsigned int count)  in dequeue_top_rt_rq()  argument
1094 	struct rq *rq = rq_of_rt_rq(rt_rq);  in dequeue_top_rt_rq()
1096 	BUG_ON(&rq->rt != rt_rq);  in dequeue_top_rt_rq()
1098 	if (!rt_rq->rt_queued)  in dequeue_top_rt_rq()
1104 	rt_rq->rt_queued = 0;  in dequeue_top_rt_rq()
1109 enqueue_top_rt_rq(struct rt_rq *rt_rq)  in enqueue_top_rt_rq()  argument
1111 	struct rq *rq = rq_of_rt_rq(rt_rq);  in enqueue_top_rt_rq()
1113 	BUG_ON(&rq->rt != rt_rq);  in enqueue_top_rt_rq()
1115 	if (rt_rq->rt_queued)  in enqueue_top_rt_rq()
1118 	if (rt_rq_throttled(rt_rq))  in enqueue_top_rt_rq()
1121 	if (rt_rq->rt_nr_running) {  in enqueue_top_rt_rq()
1122 		add_nr_running(rq, rt_rq->rt_nr_running);  in enqueue_top_rt_rq()
1123 		rt_rq->rt_queued = 1;  in enqueue_top_rt_rq()
1133 inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio)  in inc_rt_prio_smp()  argument
1135 	struct rq *rq = rq_of_rt_rq(rt_rq);  in inc_rt_prio_smp()
1141 	if (&rq->rt != rt_rq)  in inc_rt_prio_smp()
1149 dec_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio)  in dec_rt_prio_smp()  argument
1151 	struct rq *rq = rq_of_rt_rq(rt_rq);  in dec_rt_prio_smp()
1157 	if (&rq->rt != rt_rq)  in dec_rt_prio_smp()
1160 	if (rq->online && rt_rq->highest_prio.curr != prev_prio)  in dec_rt_prio_smp()
1161 		cpupri_set(&rq->rd->cpupri, rq->cpu, rt_rq->highest_prio.curr);  in dec_rt_prio_smp()
1167 void inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) {}  in inc_rt_prio_smp()  argument
1169 void dec_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) {}  in dec_rt_prio_smp()  argument
1175 inc_rt_prio(struct rt_rq *rt_rq, int prio)  in inc_rt_prio()  argument
1177 	int prev_prio = rt_rq->highest_prio.curr;  in inc_rt_prio()
1180 		rt_rq->highest_prio.curr = prio;  in inc_rt_prio()
1182 	inc_rt_prio_smp(rt_rq, prio, prev_prio);  in inc_rt_prio()
1186 dec_rt_prio(struct rt_rq *rt_rq, int prio)  in dec_rt_prio()  argument
1188 	int prev_prio = rt_rq->highest_prio.curr;  in dec_rt_prio()
1190 	if (rt_rq->rt_nr_running) {  in dec_rt_prio()
1199 			struct rt_prio_array *array = &rt_rq->active;  in dec_rt_prio()
1201 			rt_rq->highest_prio.curr =  in dec_rt_prio()
1206 		rt_rq->highest_prio.curr = MAX_RT_PRIO-1;  in dec_rt_prio()
1209 	dec_rt_prio_smp(rt_rq, prio, prev_prio);  in dec_rt_prio()
1214 static inline void inc_rt_prio(struct rt_rq *rt_rq, int prio) {}  in inc_rt_prio()  argument
1215 static inline void dec_rt_prio(struct rt_rq *rt_rq, int prio) {}  in dec_rt_prio()  argument
1222 inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in inc_rt_group()  argument
1225 		rt_rq->rt_nr_boosted++;  in inc_rt_group()
1227 	if (rt_rq->tg)  in inc_rt_group()
1228 		start_rt_bandwidth(&rt_rq->tg->rt_bandwidth);  in inc_rt_group()
1232 dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in dec_rt_group()  argument
1235 		rt_rq->rt_nr_boosted--;  in dec_rt_group()
1237 	WARN_ON(!rt_rq->rt_nr_running && rt_rq->rt_nr_boosted);  in dec_rt_group()
1243 inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in inc_rt_group()  argument
1249 void dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) {}  in dec_rt_group()  argument
1256 	struct rt_rq *group_rq = group_rt_rq(rt_se);  in rt_se_nr_running()
1267 	struct rt_rq *group_rq = group_rt_rq(rt_se);  in rt_se_rr_nr_running()
1279 void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in inc_rt_tasks()  argument
1284 	rt_rq->rt_nr_running += rt_se_nr_running(rt_se);  in inc_rt_tasks()
1285 	rt_rq->rr_nr_running += rt_se_rr_nr_running(rt_se);  in inc_rt_tasks()
1287 	inc_rt_prio(rt_rq, prio);  in inc_rt_tasks()
1288 	inc_rt_migration(rt_se, rt_rq);  in inc_rt_tasks()
1289 	inc_rt_group(rt_se, rt_rq);  in inc_rt_tasks()
1293 void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)  in dec_rt_tasks()  argument
1296 	WARN_ON(!rt_rq->rt_nr_running);  in dec_rt_tasks()
1297 	rt_rq->rt_nr_running -= rt_se_nr_running(rt_se);  in dec_rt_tasks()
1298 	rt_rq->rr_nr_running -= rt_se_rr_nr_running(rt_se);  in dec_rt_tasks()
1300 	dec_rt_prio(rt_rq, rt_se_prio(rt_se));  in dec_rt_tasks()
1301 	dec_rt_migration(rt_se, rt_rq);  in dec_rt_tasks()
1302 	dec_rt_group(rt_se, rt_rq);  in dec_rt_tasks()
1341 update_stats_wait_start_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se)  in update_stats_wait_start_rt()  argument
1356 	__update_stats_wait_start(rq_of_rt_rq(rt_rq), p, stats);  in update_stats_wait_start_rt()
1360 update_stats_enqueue_sleeper_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se)  in update_stats_enqueue_sleeper_rt()  argument
1375 	__update_stats_enqueue_sleeper(rq_of_rt_rq(rt_rq), p, stats);  in update_stats_enqueue_sleeper_rt()
1379 update_stats_enqueue_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se,  in update_stats_enqueue_rt()  argument
1386 		update_stats_enqueue_sleeper_rt(rt_rq, rt_se);  in update_stats_enqueue_rt()
1390 update_stats_wait_end_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se)  in update_stats_wait_end_rt()  argument
1405 	__update_stats_wait_end(rq_of_rt_rq(rt_rq), p, stats);  in update_stats_wait_end_rt()
1409 update_stats_dequeue_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se,  in update_stats_dequeue_rt()  argument
1426 					rq_clock(rq_of_rt_rq(rt_rq)));  in update_stats_dequeue_rt()
1430 					rq_clock(rq_of_rt_rq(rt_rq)));  in update_stats_dequeue_rt()
1436 	struct rt_rq *rt_rq = rt_rq_of_se(rt_se);  in __enqueue_rt_entity()  local
1437 	struct rt_prio_array *array = &rt_rq->active;  in __enqueue_rt_entity()
1438 	struct rt_rq *group_rq = group_rt_rq(rt_se);  in __enqueue_rt_entity()
1465 	inc_rt_tasks(rt_se, rt_rq);  in __enqueue_rt_entity()
1470 	struct rt_rq *rt_rq = rt_rq_of_se(rt_se);  in __dequeue_rt_entity()  local
1471 	struct rt_prio_array *array = &rt_rq->active;  in __dequeue_rt_entity()
1479 	dec_rt_tasks(rt_se, rt_rq);  in __dequeue_rt_entity()
1527 		struct rt_rq *rt_rq = group_rt_rq(rt_se);  in dequeue_rt_entity()  local
1529 		if (rt_rq && rt_rq->rt_nr_running)  in dequeue_rt_entity()
1570 requeue_rt_entity(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, int head)  in requeue_rt_entity()  argument
1573 		struct rt_prio_array *array = &rt_rq->active;  in requeue_rt_entity()
1586 	struct rt_rq *rt_rq;  in requeue_task_rt()  local
1589 		rt_rq = rt_rq_of_se(rt_se);  in requeue_task_rt()
1590 		requeue_rt_entity(rt_rq, rt_se, head);  in requeue_task_rt()
1750 	struct rt_rq *rt_rq = &rq->rt;  in set_next_task_rt()  local
1754 		update_stats_wait_end_rt(rt_rq, rt_se);  in set_next_task_rt()
1773 static struct sched_rt_entity *pick_next_rt_entity(struct rt_rq *rt_rq)  in pick_next_rt_entity()  argument
1775 	struct rt_prio_array *array = &rt_rq->active;  in pick_next_rt_entity()
1794 	struct rt_rq *rt_rq  = &rq->rt;  in _pick_next_task_rt()  local
1797 		rt_se = pick_next_rt_entity(rt_rq);  in _pick_next_task_rt()
1800 		rt_rq = group_rt_rq(rt_se);  in _pick_next_task_rt()
1801 	} while (rt_rq);  in _pick_next_task_rt()
1831 	struct rt_rq *rt_rq = &rq->rt;  in put_prev_task_rt()  local
1834 		update_stats_wait_start_rt(rt_rq, rt_se);  in put_prev_task_rt()
2695 	struct rt_rq *rt_rq;  in task_is_throttled_rt()  local
2698 	rt_rq = task_group(p)->rt_rq[cpu];  in task_is_throttled_rt()
2700 	rt_rq = &cpu_rq(cpu)->rt;  in task_is_throttled_rt()
2703 	return rt_rq_throttled(rt_rq);  in task_is_throttled_rt()
2887 		struct rt_rq *rt_rq = tg->rt_rq[i];  in tg_set_rt_bandwidth()  local
2889 		raw_spin_lock(&rt_rq->rt_runtime_lock);  in tg_set_rt_bandwidth()
2890 		rt_rq->rt_runtime = rt_runtime;  in tg_set_rt_bandwidth()
2891 		raw_spin_unlock(&rt_rq->rt_runtime_lock);  in tg_set_rt_bandwidth()
2980 		struct rt_rq *rt_rq = &cpu_rq(i)->rt;  in sched_rt_global_constraints()  local
2982 		raw_spin_lock(&rt_rq->rt_runtime_lock);  in sched_rt_global_constraints()
2983 		rt_rq->rt_runtime = global_rt_runtime();  in sched_rt_global_constraints()
2984 		raw_spin_unlock(&rt_rq->rt_runtime_lock);  in sched_rt_global_constraints()
3084 	struct rt_rq *rt_rq;  in print_rt_stats()  local
3087 	for_each_rt_rq(rt_rq, iter, cpu_rq(cpu))  in print_rt_stats()
3088 		print_rt_rq(m, cpu, rt_rq);  in print_rt_stats()