1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 2391e43daSPeter Zijlstra /* 3391e43daSPeter Zijlstra * Real-Time Scheduling Class (mapped to the SCHED_FIFO and SCHED_RR 4391e43daSPeter Zijlstra * policies) 5391e43daSPeter Zijlstra */ 6371bf427SVincent Guittot 7ce0dbbbbSClark Williams int sched_rr_timeslice = RR_TIMESLICE; 8d505b8afSHuaixin Chang /* More than 4 hours if BW_SHIFT equals 20. */ 9d505b8afSHuaixin Chang static const u64 max_rt_runtime = MAX_BW; 10ce0dbbbbSClark Williams 11391e43daSPeter Zijlstra static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun); 12391e43daSPeter Zijlstra 13391e43daSPeter Zijlstra struct rt_bandwidth def_rt_bandwidth; 14391e43daSPeter Zijlstra 15d9ab0e63SZhen Ni /* 16d9ab0e63SZhen Ni * period over which we measure -rt task CPU usage in us. 17d9ab0e63SZhen Ni * default: 1s 18d9ab0e63SZhen Ni */ 19d9ab0e63SZhen Ni unsigned int sysctl_sched_rt_period = 1000000; 20d9ab0e63SZhen Ni 21d9ab0e63SZhen Ni /* 22d9ab0e63SZhen Ni * part of the period that we allow rt tasks to run in us. 23d9ab0e63SZhen Ni * default: 0.95s 24d9ab0e63SZhen Ni */ 25d9ab0e63SZhen Ni int sysctl_sched_rt_runtime = 950000; 26d9ab0e63SZhen Ni 2728f152cdSBaisong Zhong #ifdef CONFIG_SYSCTL 28*c7fcb998SCyril Hrubis static int sysctl_sched_rr_timeslice = (MSEC_PER_SEC * RR_TIMESLICE) / HZ; 29d9ab0e63SZhen Ni static int sched_rt_handler(struct ctl_table *table, int write, void *buffer, 30d9ab0e63SZhen Ni size_t *lenp, loff_t *ppos); 31dafd7a9dSZhen Ni static int sched_rr_handler(struct ctl_table *table, int write, void *buffer, 32dafd7a9dSZhen Ni size_t *lenp, loff_t *ppos); 33d9ab0e63SZhen Ni static struct ctl_table sched_rt_sysctls[] = { 34d9ab0e63SZhen Ni { 35d9ab0e63SZhen Ni .procname = "sched_rt_period_us", 36d9ab0e63SZhen Ni .data = &sysctl_sched_rt_period, 37d9ab0e63SZhen Ni .maxlen = sizeof(unsigned int), 38d9ab0e63SZhen Ni .mode = 0644, 39d9ab0e63SZhen Ni .proc_handler = sched_rt_handler, 40d9ab0e63SZhen Ni }, 41d9ab0e63SZhen Ni { 42d9ab0e63SZhen Ni .procname = "sched_rt_runtime_us", 43d9ab0e63SZhen Ni .data = &sysctl_sched_rt_runtime, 44d9ab0e63SZhen Ni .maxlen = sizeof(int), 45d9ab0e63SZhen Ni .mode = 0644, 46d9ab0e63SZhen Ni .proc_handler = sched_rt_handler, 47d9ab0e63SZhen Ni }, 48dafd7a9dSZhen Ni { 49dafd7a9dSZhen Ni .procname = "sched_rr_timeslice_ms", 50dafd7a9dSZhen Ni .data = &sysctl_sched_rr_timeslice, 51dafd7a9dSZhen Ni .maxlen = sizeof(int), 52dafd7a9dSZhen Ni .mode = 0644, 53dafd7a9dSZhen Ni .proc_handler = sched_rr_handler, 54dafd7a9dSZhen Ni }, 55d9ab0e63SZhen Ni {} 56d9ab0e63SZhen Ni }; 57d9ab0e63SZhen Ni 58d9ab0e63SZhen Ni static int __init sched_rt_sysctl_init(void) 59d9ab0e63SZhen Ni { 60d9ab0e63SZhen Ni register_sysctl_init("kernel", sched_rt_sysctls); 61d9ab0e63SZhen Ni return 0; 62d9ab0e63SZhen Ni } 63d9ab0e63SZhen Ni late_initcall(sched_rt_sysctl_init); 64d9ab0e63SZhen Ni #endif 65d9ab0e63SZhen Ni 66391e43daSPeter Zijlstra static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer) 67391e43daSPeter Zijlstra { 68391e43daSPeter Zijlstra struct rt_bandwidth *rt_b = 69391e43daSPeter Zijlstra container_of(timer, struct rt_bandwidth, rt_period_timer); 70391e43daSPeter Zijlstra int idle = 0; 7177a4d1a1SPeter Zijlstra int overrun; 72391e43daSPeter Zijlstra 7377a4d1a1SPeter Zijlstra raw_spin_lock(&rt_b->rt_runtime_lock); 74391e43daSPeter Zijlstra for (;;) { 7577a4d1a1SPeter Zijlstra overrun = hrtimer_forward_now(timer, rt_b->rt_period); 76391e43daSPeter Zijlstra if (!overrun) 77391e43daSPeter Zijlstra break; 78391e43daSPeter Zijlstra 7977a4d1a1SPeter Zijlstra raw_spin_unlock(&rt_b->rt_runtime_lock); 80391e43daSPeter Zijlstra idle = do_sched_rt_period_timer(rt_b, overrun); 8177a4d1a1SPeter Zijlstra raw_spin_lock(&rt_b->rt_runtime_lock); 82391e43daSPeter Zijlstra } 834cfafd30SPeter Zijlstra if (idle) 844cfafd30SPeter Zijlstra rt_b->rt_period_active = 0; 8577a4d1a1SPeter Zijlstra raw_spin_unlock(&rt_b->rt_runtime_lock); 86391e43daSPeter Zijlstra 87391e43daSPeter Zijlstra return idle ? HRTIMER_NORESTART : HRTIMER_RESTART; 88391e43daSPeter Zijlstra } 89391e43daSPeter Zijlstra 90391e43daSPeter Zijlstra void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime) 91391e43daSPeter Zijlstra { 92391e43daSPeter Zijlstra rt_b->rt_period = ns_to_ktime(period); 93391e43daSPeter Zijlstra rt_b->rt_runtime = runtime; 94391e43daSPeter Zijlstra 95391e43daSPeter Zijlstra raw_spin_lock_init(&rt_b->rt_runtime_lock); 96391e43daSPeter Zijlstra 97d5096aa6SSebastian Andrzej Siewior hrtimer_init(&rt_b->rt_period_timer, CLOCK_MONOTONIC, 98d5096aa6SSebastian Andrzej Siewior HRTIMER_MODE_REL_HARD); 99391e43daSPeter Zijlstra rt_b->rt_period_timer.function = sched_rt_period_timer; 100391e43daSPeter Zijlstra } 101391e43daSPeter Zijlstra 1029b58e976SLi Hua static inline void do_start_rt_bandwidth(struct rt_bandwidth *rt_b) 103391e43daSPeter Zijlstra { 104391e43daSPeter Zijlstra raw_spin_lock(&rt_b->rt_runtime_lock); 1054cfafd30SPeter Zijlstra if (!rt_b->rt_period_active) { 1064cfafd30SPeter Zijlstra rt_b->rt_period_active = 1; 107c3a990dcSSteven Rostedt /* 108c3a990dcSSteven Rostedt * SCHED_DEADLINE updates the bandwidth, as a run away 109c3a990dcSSteven Rostedt * RT task with a DL task could hog a CPU. But DL does 110c3a990dcSSteven Rostedt * not reset the period. If a deadline task was running 111c3a990dcSSteven Rostedt * without an RT task running, it can cause RT tasks to 112c3a990dcSSteven Rostedt * throttle when they start up. Kick the timer right away 113c3a990dcSSteven Rostedt * to update the period. 114c3a990dcSSteven Rostedt */ 115c3a990dcSSteven Rostedt hrtimer_forward_now(&rt_b->rt_period_timer, ns_to_ktime(0)); 116d5096aa6SSebastian Andrzej Siewior hrtimer_start_expires(&rt_b->rt_period_timer, 117d5096aa6SSebastian Andrzej Siewior HRTIMER_MODE_ABS_PINNED_HARD); 1184cfafd30SPeter Zijlstra } 119391e43daSPeter Zijlstra raw_spin_unlock(&rt_b->rt_runtime_lock); 120391e43daSPeter Zijlstra } 121391e43daSPeter Zijlstra 1229b58e976SLi Hua static void start_rt_bandwidth(struct rt_bandwidth *rt_b) 1239b58e976SLi Hua { 1249b58e976SLi Hua if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF) 1259b58e976SLi Hua return; 1269b58e976SLi Hua 1279b58e976SLi Hua do_start_rt_bandwidth(rt_b); 1289b58e976SLi Hua } 1299b58e976SLi Hua 13007c54f7aSAbel Vesa void init_rt_rq(struct rt_rq *rt_rq) 131391e43daSPeter Zijlstra { 132391e43daSPeter Zijlstra struct rt_prio_array *array; 133391e43daSPeter Zijlstra int i; 134391e43daSPeter Zijlstra 135391e43daSPeter Zijlstra array = &rt_rq->active; 136391e43daSPeter Zijlstra for (i = 0; i < MAX_RT_PRIO; i++) { 137391e43daSPeter Zijlstra INIT_LIST_HEAD(array->queue + i); 138391e43daSPeter Zijlstra __clear_bit(i, array->bitmap); 139391e43daSPeter Zijlstra } 140391e43daSPeter Zijlstra /* delimiter for bitsearch: */ 141391e43daSPeter Zijlstra __set_bit(MAX_RT_PRIO, array->bitmap); 142391e43daSPeter Zijlstra 143391e43daSPeter Zijlstra #if defined CONFIG_SMP 144934fc331SPeter Zijlstra rt_rq->highest_prio.curr = MAX_RT_PRIO-1; 145934fc331SPeter Zijlstra rt_rq->highest_prio.next = MAX_RT_PRIO-1; 146391e43daSPeter Zijlstra rt_rq->rt_nr_migratory = 0; 147391e43daSPeter Zijlstra rt_rq->overloaded = 0; 148391e43daSPeter Zijlstra plist_head_init(&rt_rq->pushable_tasks); 149b6366f04SSteven Rostedt #endif /* CONFIG_SMP */ 150f4ebcbc0SKirill Tkhai /* We start is dequeued state, because no RT tasks are queued */ 151f4ebcbc0SKirill Tkhai rt_rq->rt_queued = 0; 152391e43daSPeter Zijlstra 153391e43daSPeter Zijlstra rt_rq->rt_time = 0; 154391e43daSPeter Zijlstra rt_rq->rt_throttled = 0; 155391e43daSPeter Zijlstra rt_rq->rt_runtime = 0; 156391e43daSPeter Zijlstra raw_spin_lock_init(&rt_rq->rt_runtime_lock); 157391e43daSPeter Zijlstra } 158391e43daSPeter Zijlstra 159391e43daSPeter Zijlstra #ifdef CONFIG_RT_GROUP_SCHED 160391e43daSPeter Zijlstra static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b) 161391e43daSPeter Zijlstra { 162391e43daSPeter Zijlstra hrtimer_cancel(&rt_b->rt_period_timer); 163391e43daSPeter Zijlstra } 164391e43daSPeter Zijlstra 165391e43daSPeter Zijlstra #define rt_entity_is_task(rt_se) (!(rt_se)->my_q) 166391e43daSPeter Zijlstra 167391e43daSPeter Zijlstra static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se) 168391e43daSPeter Zijlstra { 169391e43daSPeter Zijlstra #ifdef CONFIG_SCHED_DEBUG 170391e43daSPeter Zijlstra WARN_ON_ONCE(!rt_entity_is_task(rt_se)); 171391e43daSPeter Zijlstra #endif 172391e43daSPeter Zijlstra return container_of(rt_se, struct task_struct, rt); 173391e43daSPeter Zijlstra } 174391e43daSPeter Zijlstra 175391e43daSPeter Zijlstra static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq) 176391e43daSPeter Zijlstra { 177391e43daSPeter Zijlstra return rt_rq->rq; 178391e43daSPeter Zijlstra } 179391e43daSPeter Zijlstra 180391e43daSPeter Zijlstra static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) 181391e43daSPeter Zijlstra { 182391e43daSPeter Zijlstra return rt_se->rt_rq; 183391e43daSPeter Zijlstra } 184391e43daSPeter Zijlstra 185653d07a6SKirill Tkhai static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se) 186653d07a6SKirill Tkhai { 187653d07a6SKirill Tkhai struct rt_rq *rt_rq = rt_se->rt_rq; 188653d07a6SKirill Tkhai 189653d07a6SKirill Tkhai return rt_rq->rq; 190653d07a6SKirill Tkhai } 191653d07a6SKirill Tkhai 192b027789eSMathias Krause void unregister_rt_sched_group(struct task_group *tg) 193b027789eSMathias Krause { 194b027789eSMathias Krause if (tg->rt_se) 195b027789eSMathias Krause destroy_rt_bandwidth(&tg->rt_bandwidth); 196b027789eSMathias Krause 197b027789eSMathias Krause } 198b027789eSMathias Krause 199391e43daSPeter Zijlstra void free_rt_sched_group(struct task_group *tg) 200391e43daSPeter Zijlstra { 201391e43daSPeter Zijlstra int i; 202391e43daSPeter Zijlstra 203391e43daSPeter Zijlstra for_each_possible_cpu(i) { 204391e43daSPeter Zijlstra if (tg->rt_rq) 205391e43daSPeter Zijlstra kfree(tg->rt_rq[i]); 206391e43daSPeter Zijlstra if (tg->rt_se) 207391e43daSPeter Zijlstra kfree(tg->rt_se[i]); 208391e43daSPeter Zijlstra } 209391e43daSPeter Zijlstra 210391e43daSPeter Zijlstra kfree(tg->rt_rq); 211391e43daSPeter Zijlstra kfree(tg->rt_se); 212391e43daSPeter Zijlstra } 213391e43daSPeter Zijlstra 214391e43daSPeter Zijlstra void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq, 215391e43daSPeter Zijlstra struct sched_rt_entity *rt_se, int cpu, 216391e43daSPeter Zijlstra struct sched_rt_entity *parent) 217391e43daSPeter Zijlstra { 218391e43daSPeter Zijlstra struct rq *rq = cpu_rq(cpu); 219391e43daSPeter Zijlstra 220934fc331SPeter Zijlstra rt_rq->highest_prio.curr = MAX_RT_PRIO-1; 221391e43daSPeter Zijlstra rt_rq->rt_nr_boosted = 0; 222391e43daSPeter Zijlstra rt_rq->rq = rq; 223391e43daSPeter Zijlstra rt_rq->tg = tg; 224391e43daSPeter Zijlstra 225391e43daSPeter Zijlstra tg->rt_rq[cpu] = rt_rq; 226391e43daSPeter Zijlstra tg->rt_se[cpu] = rt_se; 227391e43daSPeter Zijlstra 228391e43daSPeter Zijlstra if (!rt_se) 229391e43daSPeter Zijlstra return; 230391e43daSPeter Zijlstra 231391e43daSPeter Zijlstra if (!parent) 232391e43daSPeter Zijlstra rt_se->rt_rq = &rq->rt; 233391e43daSPeter Zijlstra else 234391e43daSPeter Zijlstra rt_se->rt_rq = parent->my_q; 235391e43daSPeter Zijlstra 236391e43daSPeter Zijlstra rt_se->my_q = rt_rq; 237391e43daSPeter Zijlstra rt_se->parent = parent; 238391e43daSPeter Zijlstra INIT_LIST_HEAD(&rt_se->run_list); 239391e43daSPeter Zijlstra } 240391e43daSPeter Zijlstra 241391e43daSPeter Zijlstra int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent) 242391e43daSPeter Zijlstra { 243391e43daSPeter Zijlstra struct rt_rq *rt_rq; 244391e43daSPeter Zijlstra struct sched_rt_entity *rt_se; 245391e43daSPeter Zijlstra int i; 246391e43daSPeter Zijlstra 2476396bb22SKees Cook tg->rt_rq = kcalloc(nr_cpu_ids, sizeof(rt_rq), GFP_KERNEL); 248391e43daSPeter Zijlstra if (!tg->rt_rq) 249391e43daSPeter Zijlstra goto err; 2506396bb22SKees Cook tg->rt_se = kcalloc(nr_cpu_ids, sizeof(rt_se), GFP_KERNEL); 251391e43daSPeter Zijlstra if (!tg->rt_se) 252391e43daSPeter Zijlstra goto err; 253391e43daSPeter Zijlstra 254391e43daSPeter Zijlstra init_rt_bandwidth(&tg->rt_bandwidth, 255391e43daSPeter Zijlstra ktime_to_ns(def_rt_bandwidth.rt_period), 0); 256391e43daSPeter Zijlstra 257391e43daSPeter Zijlstra for_each_possible_cpu(i) { 258391e43daSPeter Zijlstra rt_rq = kzalloc_node(sizeof(struct rt_rq), 259391e43daSPeter Zijlstra GFP_KERNEL, cpu_to_node(i)); 260391e43daSPeter Zijlstra if (!rt_rq) 261391e43daSPeter Zijlstra goto err; 262391e43daSPeter Zijlstra 263391e43daSPeter Zijlstra rt_se = kzalloc_node(sizeof(struct sched_rt_entity), 264391e43daSPeter Zijlstra GFP_KERNEL, cpu_to_node(i)); 265391e43daSPeter Zijlstra if (!rt_se) 266391e43daSPeter Zijlstra goto err_free_rq; 267391e43daSPeter Zijlstra 26807c54f7aSAbel Vesa init_rt_rq(rt_rq); 269391e43daSPeter Zijlstra rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime; 270391e43daSPeter Zijlstra init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]); 271391e43daSPeter Zijlstra } 272391e43daSPeter Zijlstra 273391e43daSPeter Zijlstra return 1; 274391e43daSPeter Zijlstra 275391e43daSPeter Zijlstra err_free_rq: 276391e43daSPeter Zijlstra kfree(rt_rq); 277391e43daSPeter Zijlstra err: 278391e43daSPeter Zijlstra return 0; 279391e43daSPeter Zijlstra } 280391e43daSPeter Zijlstra 281391e43daSPeter Zijlstra #else /* CONFIG_RT_GROUP_SCHED */ 282391e43daSPeter Zijlstra 283391e43daSPeter Zijlstra #define rt_entity_is_task(rt_se) (1) 284391e43daSPeter Zijlstra 285391e43daSPeter Zijlstra static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se) 286391e43daSPeter Zijlstra { 287391e43daSPeter Zijlstra return container_of(rt_se, struct task_struct, rt); 288391e43daSPeter Zijlstra } 289391e43daSPeter Zijlstra 290391e43daSPeter Zijlstra static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq) 291391e43daSPeter Zijlstra { 292391e43daSPeter Zijlstra return container_of(rt_rq, struct rq, rt); 293391e43daSPeter Zijlstra } 294391e43daSPeter Zijlstra 295653d07a6SKirill Tkhai static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se) 296391e43daSPeter Zijlstra { 297391e43daSPeter Zijlstra struct task_struct *p = rt_task_of(rt_se); 298653d07a6SKirill Tkhai 299653d07a6SKirill Tkhai return task_rq(p); 300653d07a6SKirill Tkhai } 301653d07a6SKirill Tkhai 302653d07a6SKirill Tkhai static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) 303653d07a6SKirill Tkhai { 304653d07a6SKirill Tkhai struct rq *rq = rq_of_rt_se(rt_se); 305391e43daSPeter Zijlstra 306391e43daSPeter Zijlstra return &rq->rt; 307391e43daSPeter Zijlstra } 308391e43daSPeter Zijlstra 309b027789eSMathias Krause void unregister_rt_sched_group(struct task_group *tg) { } 310b027789eSMathias Krause 311391e43daSPeter Zijlstra void free_rt_sched_group(struct task_group *tg) { } 312391e43daSPeter Zijlstra 313391e43daSPeter Zijlstra int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent) 314391e43daSPeter Zijlstra { 315391e43daSPeter Zijlstra return 1; 316391e43daSPeter Zijlstra } 317391e43daSPeter Zijlstra #endif /* CONFIG_RT_GROUP_SCHED */ 318391e43daSPeter Zijlstra 319391e43daSPeter Zijlstra #ifdef CONFIG_SMP 320391e43daSPeter Zijlstra 321dc877341SPeter Zijlstra static inline bool need_pull_rt_task(struct rq *rq, struct task_struct *prev) 322dc877341SPeter Zijlstra { 323dc877341SPeter Zijlstra /* Try to pull RT tasks here if we lower this rq's prio */ 324120455c5SPeter Zijlstra return rq->online && rq->rt.highest_prio.curr > prev->prio; 325dc877341SPeter Zijlstra } 326dc877341SPeter Zijlstra 327391e43daSPeter Zijlstra static inline int rt_overloaded(struct rq *rq) 328391e43daSPeter Zijlstra { 329391e43daSPeter Zijlstra return atomic_read(&rq->rd->rto_count); 330391e43daSPeter Zijlstra } 331391e43daSPeter Zijlstra 332391e43daSPeter Zijlstra static inline void rt_set_overload(struct rq *rq) 333391e43daSPeter Zijlstra { 334391e43daSPeter Zijlstra if (!rq->online) 335391e43daSPeter Zijlstra return; 336391e43daSPeter Zijlstra 337391e43daSPeter Zijlstra cpumask_set_cpu(rq->cpu, rq->rd->rto_mask); 338391e43daSPeter Zijlstra /* 339391e43daSPeter Zijlstra * Make sure the mask is visible before we set 340391e43daSPeter Zijlstra * the overload count. That is checked to determine 341391e43daSPeter Zijlstra * if we should look at the mask. It would be a shame 342391e43daSPeter Zijlstra * if we looked at the mask, but the mask was not 343391e43daSPeter Zijlstra * updated yet. 3447c3f2ab7SPeter Zijlstra * 3457c3f2ab7SPeter Zijlstra * Matched by the barrier in pull_rt_task(). 346391e43daSPeter Zijlstra */ 3477c3f2ab7SPeter Zijlstra smp_wmb(); 348391e43daSPeter Zijlstra atomic_inc(&rq->rd->rto_count); 349391e43daSPeter Zijlstra } 350391e43daSPeter Zijlstra 351391e43daSPeter Zijlstra static inline void rt_clear_overload(struct rq *rq) 352391e43daSPeter Zijlstra { 353391e43daSPeter Zijlstra if (!rq->online) 354391e43daSPeter Zijlstra return; 355391e43daSPeter Zijlstra 356391e43daSPeter Zijlstra /* the order here really doesn't matter */ 357391e43daSPeter Zijlstra atomic_dec(&rq->rd->rto_count); 358391e43daSPeter Zijlstra cpumask_clear_cpu(rq->cpu, rq->rd->rto_mask); 359391e43daSPeter Zijlstra } 360391e43daSPeter Zijlstra 361391e43daSPeter Zijlstra static void update_rt_migration(struct rt_rq *rt_rq) 362391e43daSPeter Zijlstra { 363391e43daSPeter Zijlstra if (rt_rq->rt_nr_migratory && rt_rq->rt_nr_total > 1) { 364391e43daSPeter Zijlstra if (!rt_rq->overloaded) { 365391e43daSPeter Zijlstra rt_set_overload(rq_of_rt_rq(rt_rq)); 366391e43daSPeter Zijlstra rt_rq->overloaded = 1; 367391e43daSPeter Zijlstra } 368391e43daSPeter Zijlstra } else if (rt_rq->overloaded) { 369391e43daSPeter Zijlstra rt_clear_overload(rq_of_rt_rq(rt_rq)); 370391e43daSPeter Zijlstra rt_rq->overloaded = 0; 371391e43daSPeter Zijlstra } 372391e43daSPeter Zijlstra } 373391e43daSPeter Zijlstra 374391e43daSPeter Zijlstra static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) 375391e43daSPeter Zijlstra { 37629baa747SPeter Zijlstra struct task_struct *p; 37729baa747SPeter Zijlstra 378391e43daSPeter Zijlstra if (!rt_entity_is_task(rt_se)) 379391e43daSPeter Zijlstra return; 380391e43daSPeter Zijlstra 38129baa747SPeter Zijlstra p = rt_task_of(rt_se); 382391e43daSPeter Zijlstra rt_rq = &rq_of_rt_rq(rt_rq)->rt; 383391e43daSPeter Zijlstra 384391e43daSPeter Zijlstra rt_rq->rt_nr_total++; 3854b53a341SIngo Molnar if (p->nr_cpus_allowed > 1) 386391e43daSPeter Zijlstra rt_rq->rt_nr_migratory++; 387391e43daSPeter Zijlstra 388391e43daSPeter Zijlstra update_rt_migration(rt_rq); 389391e43daSPeter Zijlstra } 390391e43daSPeter Zijlstra 391391e43daSPeter Zijlstra static void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) 392391e43daSPeter Zijlstra { 39329baa747SPeter Zijlstra struct task_struct *p; 39429baa747SPeter Zijlstra 395391e43daSPeter Zijlstra if (!rt_entity_is_task(rt_se)) 396391e43daSPeter Zijlstra return; 397391e43daSPeter Zijlstra 39829baa747SPeter Zijlstra p = rt_task_of(rt_se); 399391e43daSPeter Zijlstra rt_rq = &rq_of_rt_rq(rt_rq)->rt; 400391e43daSPeter Zijlstra 401391e43daSPeter Zijlstra rt_rq->rt_nr_total--; 4024b53a341SIngo Molnar if (p->nr_cpus_allowed > 1) 403391e43daSPeter Zijlstra rt_rq->rt_nr_migratory--; 404391e43daSPeter Zijlstra 405391e43daSPeter Zijlstra update_rt_migration(rt_rq); 406391e43daSPeter Zijlstra } 407391e43daSPeter Zijlstra 408391e43daSPeter Zijlstra static inline int has_pushable_tasks(struct rq *rq) 409391e43daSPeter Zijlstra { 410391e43daSPeter Zijlstra return !plist_head_empty(&rq->rt.pushable_tasks); 411391e43daSPeter Zijlstra } 412391e43daSPeter Zijlstra 4138e5bad7dSKees Cook static DEFINE_PER_CPU(struct balance_callback, rt_push_head); 4148e5bad7dSKees Cook static DEFINE_PER_CPU(struct balance_callback, rt_pull_head); 415e3fca9e7SPeter Zijlstra 416e3fca9e7SPeter Zijlstra static void push_rt_tasks(struct rq *); 417fd7a4bedSPeter Zijlstra static void pull_rt_task(struct rq *); 418e3fca9e7SPeter Zijlstra 41902d8ec94SIngo Molnar static inline void rt_queue_push_tasks(struct rq *rq) 420dc877341SPeter Zijlstra { 421e3fca9e7SPeter Zijlstra if (!has_pushable_tasks(rq)) 422e3fca9e7SPeter Zijlstra return; 423e3fca9e7SPeter Zijlstra 424fd7a4bedSPeter Zijlstra queue_balance_callback(rq, &per_cpu(rt_push_head, rq->cpu), push_rt_tasks); 425fd7a4bedSPeter Zijlstra } 426fd7a4bedSPeter Zijlstra 42702d8ec94SIngo Molnar static inline void rt_queue_pull_task(struct rq *rq) 428fd7a4bedSPeter Zijlstra { 429fd7a4bedSPeter Zijlstra queue_balance_callback(rq, &per_cpu(rt_pull_head, rq->cpu), pull_rt_task); 430dc877341SPeter Zijlstra } 431dc877341SPeter Zijlstra 432391e43daSPeter Zijlstra static void enqueue_pushable_task(struct rq *rq, struct task_struct *p) 433391e43daSPeter Zijlstra { 434391e43daSPeter Zijlstra plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks); 435391e43daSPeter Zijlstra plist_node_init(&p->pushable_tasks, p->prio); 436391e43daSPeter Zijlstra plist_add(&p->pushable_tasks, &rq->rt.pushable_tasks); 437391e43daSPeter Zijlstra 438391e43daSPeter Zijlstra /* Update the highest prio pushable task */ 439391e43daSPeter Zijlstra if (p->prio < rq->rt.highest_prio.next) 440391e43daSPeter Zijlstra rq->rt.highest_prio.next = p->prio; 441391e43daSPeter Zijlstra } 442391e43daSPeter Zijlstra 443391e43daSPeter Zijlstra static void dequeue_pushable_task(struct rq *rq, struct task_struct *p) 444391e43daSPeter Zijlstra { 445391e43daSPeter Zijlstra plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks); 446391e43daSPeter Zijlstra 447391e43daSPeter Zijlstra /* Update the new highest prio pushable task */ 448391e43daSPeter Zijlstra if (has_pushable_tasks(rq)) { 449391e43daSPeter Zijlstra p = plist_first_entry(&rq->rt.pushable_tasks, 450391e43daSPeter Zijlstra struct task_struct, pushable_tasks); 451391e43daSPeter Zijlstra rq->rt.highest_prio.next = p->prio; 452934fc331SPeter Zijlstra } else { 453934fc331SPeter Zijlstra rq->rt.highest_prio.next = MAX_RT_PRIO-1; 454934fc331SPeter Zijlstra } 455391e43daSPeter Zijlstra } 456391e43daSPeter Zijlstra 457391e43daSPeter Zijlstra #else 458391e43daSPeter Zijlstra 459391e43daSPeter Zijlstra static inline void enqueue_pushable_task(struct rq *rq, struct task_struct *p) 460391e43daSPeter Zijlstra { 461391e43daSPeter Zijlstra } 462391e43daSPeter Zijlstra 463391e43daSPeter Zijlstra static inline void dequeue_pushable_task(struct rq *rq, struct task_struct *p) 464391e43daSPeter Zijlstra { 465391e43daSPeter Zijlstra } 466391e43daSPeter Zijlstra 467391e43daSPeter Zijlstra static inline 468391e43daSPeter Zijlstra void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) 469391e43daSPeter Zijlstra { 470391e43daSPeter Zijlstra } 471391e43daSPeter Zijlstra 472391e43daSPeter Zijlstra static inline 473391e43daSPeter Zijlstra void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) 474391e43daSPeter Zijlstra { 475391e43daSPeter Zijlstra } 476391e43daSPeter Zijlstra 47702d8ec94SIngo Molnar static inline void rt_queue_push_tasks(struct rq *rq) 478dc877341SPeter Zijlstra { 479dc877341SPeter Zijlstra } 480391e43daSPeter Zijlstra #endif /* CONFIG_SMP */ 481391e43daSPeter Zijlstra 482f4ebcbc0SKirill Tkhai static void enqueue_top_rt_rq(struct rt_rq *rt_rq); 4835c66d1b9SNicolas Saenz Julienne static void dequeue_top_rt_rq(struct rt_rq *rt_rq, unsigned int count); 484f4ebcbc0SKirill Tkhai 485391e43daSPeter Zijlstra static inline int on_rt_rq(struct sched_rt_entity *rt_se) 486391e43daSPeter Zijlstra { 487ff77e468SPeter Zijlstra return rt_se->on_rq; 488391e43daSPeter Zijlstra } 489391e43daSPeter Zijlstra 490804d402fSQais Yousef #ifdef CONFIG_UCLAMP_TASK 491804d402fSQais Yousef /* 492804d402fSQais Yousef * Verify the fitness of task @p to run on @cpu taking into account the uclamp 493804d402fSQais Yousef * settings. 494804d402fSQais Yousef * 495804d402fSQais Yousef * This check is only important for heterogeneous systems where uclamp_min value 496804d402fSQais Yousef * is higher than the capacity of a @cpu. For non-heterogeneous system this 497804d402fSQais Yousef * function will always return true. 498804d402fSQais Yousef * 499804d402fSQais Yousef * The function will return true if the capacity of the @cpu is >= the 500804d402fSQais Yousef * uclamp_min and false otherwise. 501804d402fSQais Yousef * 502804d402fSQais Yousef * Note that uclamp_min will be clamped to uclamp_max if uclamp_min 503804d402fSQais Yousef * > uclamp_max. 504804d402fSQais Yousef */ 505804d402fSQais Yousef static inline bool rt_task_fits_capacity(struct task_struct *p, int cpu) 506804d402fSQais Yousef { 507804d402fSQais Yousef unsigned int min_cap; 508804d402fSQais Yousef unsigned int max_cap; 509804d402fSQais Yousef unsigned int cpu_cap; 510804d402fSQais Yousef 511804d402fSQais Yousef /* Only heterogeneous systems can benefit from this check */ 512740cf8a7SDietmar Eggemann if (!sched_asym_cpucap_active()) 513804d402fSQais Yousef return true; 514804d402fSQais Yousef 515804d402fSQais Yousef min_cap = uclamp_eff_value(p, UCLAMP_MIN); 516804d402fSQais Yousef max_cap = uclamp_eff_value(p, UCLAMP_MAX); 517804d402fSQais Yousef 518804d402fSQais Yousef cpu_cap = capacity_orig_of(cpu); 519804d402fSQais Yousef 520804d402fSQais Yousef return cpu_cap >= min(min_cap, max_cap); 521804d402fSQais Yousef } 522804d402fSQais Yousef #else 523804d402fSQais Yousef static inline bool rt_task_fits_capacity(struct task_struct *p, int cpu) 524804d402fSQais Yousef { 525804d402fSQais Yousef return true; 526804d402fSQais Yousef } 527804d402fSQais Yousef #endif 528804d402fSQais Yousef 529391e43daSPeter Zijlstra #ifdef CONFIG_RT_GROUP_SCHED 530391e43daSPeter Zijlstra 531391e43daSPeter Zijlstra static inline u64 sched_rt_runtime(struct rt_rq *rt_rq) 532391e43daSPeter Zijlstra { 533391e43daSPeter Zijlstra if (!rt_rq->tg) 534391e43daSPeter Zijlstra return RUNTIME_INF; 535391e43daSPeter Zijlstra 536391e43daSPeter Zijlstra return rt_rq->rt_runtime; 537391e43daSPeter Zijlstra } 538391e43daSPeter Zijlstra 539391e43daSPeter Zijlstra static inline u64 sched_rt_period(struct rt_rq *rt_rq) 540391e43daSPeter Zijlstra { 541391e43daSPeter Zijlstra return ktime_to_ns(rt_rq->tg->rt_bandwidth.rt_period); 542391e43daSPeter Zijlstra } 543391e43daSPeter Zijlstra 544391e43daSPeter Zijlstra typedef struct task_group *rt_rq_iter_t; 545391e43daSPeter Zijlstra 546391e43daSPeter Zijlstra static inline struct task_group *next_task_group(struct task_group *tg) 547391e43daSPeter Zijlstra { 548391e43daSPeter Zijlstra do { 549391e43daSPeter Zijlstra tg = list_entry_rcu(tg->list.next, 550391e43daSPeter Zijlstra typeof(struct task_group), list); 551391e43daSPeter Zijlstra } while (&tg->list != &task_groups && task_group_is_autogroup(tg)); 552391e43daSPeter Zijlstra 553391e43daSPeter Zijlstra if (&tg->list == &task_groups) 554391e43daSPeter Zijlstra tg = NULL; 555391e43daSPeter Zijlstra 556391e43daSPeter Zijlstra return tg; 557391e43daSPeter Zijlstra } 558391e43daSPeter Zijlstra 559391e43daSPeter Zijlstra #define for_each_rt_rq(rt_rq, iter, rq) \ 560391e43daSPeter Zijlstra for (iter = container_of(&task_groups, typeof(*iter), list); \ 561391e43daSPeter Zijlstra (iter = next_task_group(iter)) && \ 562391e43daSPeter Zijlstra (rt_rq = iter->rt_rq[cpu_of(rq)]);) 563391e43daSPeter Zijlstra 564391e43daSPeter Zijlstra #define for_each_sched_rt_entity(rt_se) \ 565391e43daSPeter Zijlstra for (; rt_se; rt_se = rt_se->parent) 566391e43daSPeter Zijlstra 567391e43daSPeter Zijlstra static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se) 568391e43daSPeter Zijlstra { 569391e43daSPeter Zijlstra return rt_se->my_q; 570391e43daSPeter Zijlstra } 571391e43daSPeter Zijlstra 572ff77e468SPeter Zijlstra static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags); 573ff77e468SPeter Zijlstra static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags); 574391e43daSPeter Zijlstra 575391e43daSPeter Zijlstra static void sched_rt_rq_enqueue(struct rt_rq *rt_rq) 576391e43daSPeter Zijlstra { 577391e43daSPeter Zijlstra struct task_struct *curr = rq_of_rt_rq(rt_rq)->curr; 5788875125eSKirill Tkhai struct rq *rq = rq_of_rt_rq(rt_rq); 579391e43daSPeter Zijlstra struct sched_rt_entity *rt_se; 580391e43daSPeter Zijlstra 5818875125eSKirill Tkhai int cpu = cpu_of(rq); 582391e43daSPeter Zijlstra 583391e43daSPeter Zijlstra rt_se = rt_rq->tg->rt_se[cpu]; 584391e43daSPeter Zijlstra 585391e43daSPeter Zijlstra if (rt_rq->rt_nr_running) { 586f4ebcbc0SKirill Tkhai if (!rt_se) 587f4ebcbc0SKirill Tkhai enqueue_top_rt_rq(rt_rq); 588f4ebcbc0SKirill Tkhai else if (!on_rt_rq(rt_se)) 589ff77e468SPeter Zijlstra enqueue_rt_entity(rt_se, 0); 590f4ebcbc0SKirill Tkhai 591391e43daSPeter Zijlstra if (rt_rq->highest_prio.curr < curr->prio) 5928875125eSKirill Tkhai resched_curr(rq); 593391e43daSPeter Zijlstra } 594391e43daSPeter Zijlstra } 595391e43daSPeter Zijlstra 596391e43daSPeter Zijlstra static void sched_rt_rq_dequeue(struct rt_rq *rt_rq) 597391e43daSPeter Zijlstra { 598391e43daSPeter Zijlstra struct sched_rt_entity *rt_se; 599391e43daSPeter Zijlstra int cpu = cpu_of(rq_of_rt_rq(rt_rq)); 600391e43daSPeter Zijlstra 601391e43daSPeter Zijlstra rt_se = rt_rq->tg->rt_se[cpu]; 602391e43daSPeter Zijlstra 603296b2ffeSVincent Guittot if (!rt_se) { 6045c66d1b9SNicolas Saenz Julienne dequeue_top_rt_rq(rt_rq, rt_rq->rt_nr_running); 605296b2ffeSVincent Guittot /* Kick cpufreq (see the comment in kernel/sched/sched.h). */ 606296b2ffeSVincent Guittot cpufreq_update_util(rq_of_rt_rq(rt_rq), 0); 607296b2ffeSVincent Guittot } 608f4ebcbc0SKirill Tkhai else if (on_rt_rq(rt_se)) 609ff77e468SPeter Zijlstra dequeue_rt_entity(rt_se, 0); 610391e43daSPeter Zijlstra } 611391e43daSPeter Zijlstra 61246383648SKirill Tkhai static inline int rt_rq_throttled(struct rt_rq *rt_rq) 61346383648SKirill Tkhai { 61446383648SKirill Tkhai return rt_rq->rt_throttled && !rt_rq->rt_nr_boosted; 61546383648SKirill Tkhai } 61646383648SKirill Tkhai 617391e43daSPeter Zijlstra static int rt_se_boosted(struct sched_rt_entity *rt_se) 618391e43daSPeter Zijlstra { 619391e43daSPeter Zijlstra struct rt_rq *rt_rq = group_rt_rq(rt_se); 620391e43daSPeter Zijlstra struct task_struct *p; 621391e43daSPeter Zijlstra 622391e43daSPeter Zijlstra if (rt_rq) 623391e43daSPeter Zijlstra return !!rt_rq->rt_nr_boosted; 624391e43daSPeter Zijlstra 625391e43daSPeter Zijlstra p = rt_task_of(rt_se); 626391e43daSPeter Zijlstra return p->prio != p->normal_prio; 627391e43daSPeter Zijlstra } 628391e43daSPeter Zijlstra 629391e43daSPeter Zijlstra #ifdef CONFIG_SMP 630391e43daSPeter Zijlstra static inline const struct cpumask *sched_rt_period_mask(void) 631391e43daSPeter Zijlstra { 632424c93feSNathan Zimmer return this_rq()->rd->span; 633391e43daSPeter Zijlstra } 634391e43daSPeter Zijlstra #else 635391e43daSPeter Zijlstra static inline const struct cpumask *sched_rt_period_mask(void) 636391e43daSPeter Zijlstra { 637391e43daSPeter Zijlstra return cpu_online_mask; 638391e43daSPeter Zijlstra } 639391e43daSPeter Zijlstra #endif 640391e43daSPeter Zijlstra 641391e43daSPeter Zijlstra static inline 642391e43daSPeter Zijlstra struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu) 643391e43daSPeter Zijlstra { 644391e43daSPeter Zijlstra return container_of(rt_b, struct task_group, rt_bandwidth)->rt_rq[cpu]; 645391e43daSPeter Zijlstra } 646391e43daSPeter Zijlstra 647391e43daSPeter Zijlstra static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq) 648391e43daSPeter Zijlstra { 649391e43daSPeter Zijlstra return &rt_rq->tg->rt_bandwidth; 650391e43daSPeter Zijlstra } 651391e43daSPeter Zijlstra 652391e43daSPeter Zijlstra #else /* !CONFIG_RT_GROUP_SCHED */ 653391e43daSPeter Zijlstra 654391e43daSPeter Zijlstra static inline u64 sched_rt_runtime(struct rt_rq *rt_rq) 655391e43daSPeter Zijlstra { 656391e43daSPeter Zijlstra return rt_rq->rt_runtime; 657391e43daSPeter Zijlstra } 658391e43daSPeter Zijlstra 659391e43daSPeter Zijlstra static inline u64 sched_rt_period(struct rt_rq *rt_rq) 660391e43daSPeter Zijlstra { 661391e43daSPeter Zijlstra return ktime_to_ns(def_rt_bandwidth.rt_period); 662391e43daSPeter Zijlstra } 663391e43daSPeter Zijlstra 664391e43daSPeter Zijlstra typedef struct rt_rq *rt_rq_iter_t; 665391e43daSPeter Zijlstra 666391e43daSPeter Zijlstra #define for_each_rt_rq(rt_rq, iter, rq) \ 667391e43daSPeter Zijlstra for ((void) iter, rt_rq = &rq->rt; rt_rq; rt_rq = NULL) 668391e43daSPeter Zijlstra 669391e43daSPeter Zijlstra #define for_each_sched_rt_entity(rt_se) \ 670391e43daSPeter Zijlstra for (; rt_se; rt_se = NULL) 671391e43daSPeter Zijlstra 672391e43daSPeter Zijlstra static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se) 673391e43daSPeter Zijlstra { 674391e43daSPeter Zijlstra return NULL; 675391e43daSPeter Zijlstra } 676391e43daSPeter Zijlstra 677391e43daSPeter Zijlstra static inline void sched_rt_rq_enqueue(struct rt_rq *rt_rq) 678391e43daSPeter Zijlstra { 679f4ebcbc0SKirill Tkhai struct rq *rq = rq_of_rt_rq(rt_rq); 680f4ebcbc0SKirill Tkhai 681f4ebcbc0SKirill Tkhai if (!rt_rq->rt_nr_running) 682f4ebcbc0SKirill Tkhai return; 683f4ebcbc0SKirill Tkhai 684f4ebcbc0SKirill Tkhai enqueue_top_rt_rq(rt_rq); 6858875125eSKirill Tkhai resched_curr(rq); 686391e43daSPeter Zijlstra } 687391e43daSPeter Zijlstra 688391e43daSPeter Zijlstra static inline void sched_rt_rq_dequeue(struct rt_rq *rt_rq) 689391e43daSPeter Zijlstra { 6905c66d1b9SNicolas Saenz Julienne dequeue_top_rt_rq(rt_rq, rt_rq->rt_nr_running); 691391e43daSPeter Zijlstra } 692391e43daSPeter Zijlstra 69346383648SKirill Tkhai static inline int rt_rq_throttled(struct rt_rq *rt_rq) 69446383648SKirill Tkhai { 69546383648SKirill Tkhai return rt_rq->rt_throttled; 69646383648SKirill Tkhai } 69746383648SKirill Tkhai 698391e43daSPeter Zijlstra static inline const struct cpumask *sched_rt_period_mask(void) 699391e43daSPeter Zijlstra { 700391e43daSPeter Zijlstra return cpu_online_mask; 701391e43daSPeter Zijlstra } 702391e43daSPeter Zijlstra 703391e43daSPeter Zijlstra static inline 704391e43daSPeter Zijlstra struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu) 705391e43daSPeter Zijlstra { 706391e43daSPeter Zijlstra return &cpu_rq(cpu)->rt; 707391e43daSPeter Zijlstra } 708391e43daSPeter Zijlstra 709391e43daSPeter Zijlstra static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq) 710391e43daSPeter Zijlstra { 711391e43daSPeter Zijlstra return &def_rt_bandwidth; 712391e43daSPeter Zijlstra } 713391e43daSPeter Zijlstra 714391e43daSPeter Zijlstra #endif /* CONFIG_RT_GROUP_SCHED */ 715391e43daSPeter Zijlstra 716faa59937SJuri Lelli bool sched_rt_bandwidth_account(struct rt_rq *rt_rq) 717faa59937SJuri Lelli { 718faa59937SJuri Lelli struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq); 719faa59937SJuri Lelli 720faa59937SJuri Lelli return (hrtimer_active(&rt_b->rt_period_timer) || 721faa59937SJuri Lelli rt_rq->rt_time < rt_b->rt_runtime); 722faa59937SJuri Lelli } 723faa59937SJuri Lelli 724391e43daSPeter Zijlstra #ifdef CONFIG_SMP 725391e43daSPeter Zijlstra /* 726391e43daSPeter Zijlstra * We ran out of runtime, see if we can borrow some from our neighbours. 727391e43daSPeter Zijlstra */ 728269b26a5SJuri Lelli static void do_balance_runtime(struct rt_rq *rt_rq) 729391e43daSPeter Zijlstra { 730391e43daSPeter Zijlstra struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq); 731aa7f6730SShawn Bohrer struct root_domain *rd = rq_of_rt_rq(rt_rq)->rd; 732269b26a5SJuri Lelli int i, weight; 733391e43daSPeter Zijlstra u64 rt_period; 734391e43daSPeter Zijlstra 735391e43daSPeter Zijlstra weight = cpumask_weight(rd->span); 736391e43daSPeter Zijlstra 737391e43daSPeter Zijlstra raw_spin_lock(&rt_b->rt_runtime_lock); 738391e43daSPeter Zijlstra rt_period = ktime_to_ns(rt_b->rt_period); 739391e43daSPeter Zijlstra for_each_cpu(i, rd->span) { 740391e43daSPeter Zijlstra struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i); 741391e43daSPeter Zijlstra s64 diff; 742391e43daSPeter Zijlstra 743391e43daSPeter Zijlstra if (iter == rt_rq) 744391e43daSPeter Zijlstra continue; 745391e43daSPeter Zijlstra 746391e43daSPeter Zijlstra raw_spin_lock(&iter->rt_runtime_lock); 747391e43daSPeter Zijlstra /* 748391e43daSPeter Zijlstra * Either all rqs have inf runtime and there's nothing to steal 749391e43daSPeter Zijlstra * or __disable_runtime() below sets a specific rq to inf to 7503b03706fSIngo Molnar * indicate its been disabled and disallow stealing. 751391e43daSPeter Zijlstra */ 752391e43daSPeter Zijlstra if (iter->rt_runtime == RUNTIME_INF) 753391e43daSPeter Zijlstra goto next; 754391e43daSPeter Zijlstra 755391e43daSPeter Zijlstra /* 756391e43daSPeter Zijlstra * From runqueues with spare time, take 1/n part of their 757391e43daSPeter Zijlstra * spare time, but no more than our period. 758391e43daSPeter Zijlstra */ 759391e43daSPeter Zijlstra diff = iter->rt_runtime - iter->rt_time; 760391e43daSPeter Zijlstra if (diff > 0) { 761391e43daSPeter Zijlstra diff = div_u64((u64)diff, weight); 762391e43daSPeter Zijlstra if (rt_rq->rt_runtime + diff > rt_period) 763391e43daSPeter Zijlstra diff = rt_period - rt_rq->rt_runtime; 764391e43daSPeter Zijlstra iter->rt_runtime -= diff; 765391e43daSPeter Zijlstra rt_rq->rt_runtime += diff; 766391e43daSPeter Zijlstra if (rt_rq->rt_runtime == rt_period) { 767391e43daSPeter Zijlstra raw_spin_unlock(&iter->rt_runtime_lock); 768391e43daSPeter Zijlstra break; 769391e43daSPeter Zijlstra } 770391e43daSPeter Zijlstra } 771391e43daSPeter Zijlstra next: 772391e43daSPeter Zijlstra raw_spin_unlock(&iter->rt_runtime_lock); 773391e43daSPeter Zijlstra } 774391e43daSPeter Zijlstra raw_spin_unlock(&rt_b->rt_runtime_lock); 775391e43daSPeter Zijlstra } 776391e43daSPeter Zijlstra 777391e43daSPeter Zijlstra /* 778391e43daSPeter Zijlstra * Ensure this RQ takes back all the runtime it lend to its neighbours. 779391e43daSPeter Zijlstra */ 780391e43daSPeter Zijlstra static void __disable_runtime(struct rq *rq) 781391e43daSPeter Zijlstra { 782391e43daSPeter Zijlstra struct root_domain *rd = rq->rd; 783391e43daSPeter Zijlstra rt_rq_iter_t iter; 784391e43daSPeter Zijlstra struct rt_rq *rt_rq; 785391e43daSPeter Zijlstra 786391e43daSPeter Zijlstra if (unlikely(!scheduler_running)) 787391e43daSPeter Zijlstra return; 788391e43daSPeter Zijlstra 789391e43daSPeter Zijlstra for_each_rt_rq(rt_rq, iter, rq) { 790391e43daSPeter Zijlstra struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq); 791391e43daSPeter Zijlstra s64 want; 792391e43daSPeter Zijlstra int i; 793391e43daSPeter Zijlstra 794391e43daSPeter Zijlstra raw_spin_lock(&rt_b->rt_runtime_lock); 795391e43daSPeter Zijlstra raw_spin_lock(&rt_rq->rt_runtime_lock); 796391e43daSPeter Zijlstra /* 797391e43daSPeter Zijlstra * Either we're all inf and nobody needs to borrow, or we're 798391e43daSPeter Zijlstra * already disabled and thus have nothing to do, or we have 799391e43daSPeter Zijlstra * exactly the right amount of runtime to take out. 800391e43daSPeter Zijlstra */ 801391e43daSPeter Zijlstra if (rt_rq->rt_runtime == RUNTIME_INF || 802391e43daSPeter Zijlstra rt_rq->rt_runtime == rt_b->rt_runtime) 803391e43daSPeter Zijlstra goto balanced; 804391e43daSPeter Zijlstra raw_spin_unlock(&rt_rq->rt_runtime_lock); 805391e43daSPeter Zijlstra 806391e43daSPeter Zijlstra /* 807391e43daSPeter Zijlstra * Calculate the difference between what we started out with 808391e43daSPeter Zijlstra * and what we current have, that's the amount of runtime 809391e43daSPeter Zijlstra * we lend and now have to reclaim. 810391e43daSPeter Zijlstra */ 811391e43daSPeter Zijlstra want = rt_b->rt_runtime - rt_rq->rt_runtime; 812391e43daSPeter Zijlstra 813391e43daSPeter Zijlstra /* 814391e43daSPeter Zijlstra * Greedy reclaim, take back as much as we can. 815391e43daSPeter Zijlstra */ 816391e43daSPeter Zijlstra for_each_cpu(i, rd->span) { 817391e43daSPeter Zijlstra struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i); 818391e43daSPeter Zijlstra s64 diff; 819391e43daSPeter Zijlstra 820391e43daSPeter Zijlstra /* 821391e43daSPeter Zijlstra * Can't reclaim from ourselves or disabled runqueues. 822391e43daSPeter Zijlstra */ 823391e43daSPeter Zijlstra if (iter == rt_rq || iter->rt_runtime == RUNTIME_INF) 824391e43daSPeter Zijlstra continue; 825391e43daSPeter Zijlstra 826391e43daSPeter Zijlstra raw_spin_lock(&iter->rt_runtime_lock); 827391e43daSPeter Zijlstra if (want > 0) { 828391e43daSPeter Zijlstra diff = min_t(s64, iter->rt_runtime, want); 829391e43daSPeter Zijlstra iter->rt_runtime -= diff; 830391e43daSPeter Zijlstra want -= diff; 831391e43daSPeter Zijlstra } else { 832391e43daSPeter Zijlstra iter->rt_runtime -= want; 833391e43daSPeter Zijlstra want -= want; 834391e43daSPeter Zijlstra } 835391e43daSPeter Zijlstra raw_spin_unlock(&iter->rt_runtime_lock); 836391e43daSPeter Zijlstra 837391e43daSPeter Zijlstra if (!want) 838391e43daSPeter Zijlstra break; 839391e43daSPeter Zijlstra } 840391e43daSPeter Zijlstra 841391e43daSPeter Zijlstra raw_spin_lock(&rt_rq->rt_runtime_lock); 842391e43daSPeter Zijlstra /* 843391e43daSPeter Zijlstra * We cannot be left wanting - that would mean some runtime 844391e43daSPeter Zijlstra * leaked out of the system. 845391e43daSPeter Zijlstra */ 84609348d75SIngo Molnar WARN_ON_ONCE(want); 847391e43daSPeter Zijlstra balanced: 848391e43daSPeter Zijlstra /* 849391e43daSPeter Zijlstra * Disable all the borrow logic by pretending we have inf 850391e43daSPeter Zijlstra * runtime - in which case borrowing doesn't make sense. 851391e43daSPeter Zijlstra */ 852391e43daSPeter Zijlstra rt_rq->rt_runtime = RUNTIME_INF; 853a4c96ae3SPeter Boonstoppel rt_rq->rt_throttled = 0; 854391e43daSPeter Zijlstra raw_spin_unlock(&rt_rq->rt_runtime_lock); 855391e43daSPeter Zijlstra raw_spin_unlock(&rt_b->rt_runtime_lock); 85699b62567SKirill Tkhai 85799b62567SKirill Tkhai /* Make rt_rq available for pick_next_task() */ 85899b62567SKirill Tkhai sched_rt_rq_enqueue(rt_rq); 859391e43daSPeter Zijlstra } 860391e43daSPeter Zijlstra } 861391e43daSPeter Zijlstra 862391e43daSPeter Zijlstra static void __enable_runtime(struct rq *rq) 863391e43daSPeter Zijlstra { 864391e43daSPeter Zijlstra rt_rq_iter_t iter; 865391e43daSPeter Zijlstra struct rt_rq *rt_rq; 866391e43daSPeter Zijlstra 867391e43daSPeter Zijlstra if (unlikely(!scheduler_running)) 868391e43daSPeter Zijlstra return; 869391e43daSPeter Zijlstra 870391e43daSPeter Zijlstra /* 871391e43daSPeter Zijlstra * Reset each runqueue's bandwidth settings 872391e43daSPeter Zijlstra */ 873391e43daSPeter Zijlstra for_each_rt_rq(rt_rq, iter, rq) { 874391e43daSPeter Zijlstra struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq); 875391e43daSPeter Zijlstra 876391e43daSPeter Zijlstra raw_spin_lock(&rt_b->rt_runtime_lock); 877391e43daSPeter Zijlstra raw_spin_lock(&rt_rq->rt_runtime_lock); 878391e43daSPeter Zijlstra rt_rq->rt_runtime = rt_b->rt_runtime; 879391e43daSPeter Zijlstra rt_rq->rt_time = 0; 880391e43daSPeter Zijlstra rt_rq->rt_throttled = 0; 881391e43daSPeter Zijlstra raw_spin_unlock(&rt_rq->rt_runtime_lock); 882391e43daSPeter Zijlstra raw_spin_unlock(&rt_b->rt_runtime_lock); 883391e43daSPeter Zijlstra } 884391e43daSPeter Zijlstra } 885391e43daSPeter Zijlstra 886269b26a5SJuri Lelli static void balance_runtime(struct rt_rq *rt_rq) 887391e43daSPeter Zijlstra { 888391e43daSPeter Zijlstra if (!sched_feat(RT_RUNTIME_SHARE)) 889269b26a5SJuri Lelli return; 890391e43daSPeter Zijlstra 891391e43daSPeter Zijlstra if (rt_rq->rt_time > rt_rq->rt_runtime) { 892391e43daSPeter Zijlstra raw_spin_unlock(&rt_rq->rt_runtime_lock); 893269b26a5SJuri Lelli do_balance_runtime(rt_rq); 894391e43daSPeter Zijlstra raw_spin_lock(&rt_rq->rt_runtime_lock); 895391e43daSPeter Zijlstra } 896391e43daSPeter Zijlstra } 897391e43daSPeter Zijlstra #else /* !CONFIG_SMP */ 898269b26a5SJuri Lelli static inline void balance_runtime(struct rt_rq *rt_rq) {} 899391e43daSPeter Zijlstra #endif /* CONFIG_SMP */ 900391e43daSPeter Zijlstra 901391e43daSPeter Zijlstra static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) 902391e43daSPeter Zijlstra { 90342c62a58SPeter Zijlstra int i, idle = 1, throttled = 0; 904391e43daSPeter Zijlstra const struct cpumask *span; 905391e43daSPeter Zijlstra 906391e43daSPeter Zijlstra span = sched_rt_period_mask(); 907e221d028SMike Galbraith #ifdef CONFIG_RT_GROUP_SCHED 908e221d028SMike Galbraith /* 909e221d028SMike Galbraith * FIXME: isolated CPUs should really leave the root task group, 910e221d028SMike Galbraith * whether they are isolcpus or were isolated via cpusets, lest 911e221d028SMike Galbraith * the timer run on a CPU which does not service all runqueues, 912e221d028SMike Galbraith * potentially leaving other CPUs indefinitely throttled. If 913e221d028SMike Galbraith * isolation is really required, the user will turn the throttle 914e221d028SMike Galbraith * off to kill the perturbations it causes anyway. Meanwhile, 915e221d028SMike Galbraith * this maintains functionality for boot and/or troubleshooting. 916e221d028SMike Galbraith */ 917e221d028SMike Galbraith if (rt_b == &root_task_group.rt_bandwidth) 918e221d028SMike Galbraith span = cpu_online_mask; 919e221d028SMike Galbraith #endif 920391e43daSPeter Zijlstra for_each_cpu(i, span) { 921391e43daSPeter Zijlstra int enqueue = 0; 922391e43daSPeter Zijlstra struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i); 923391e43daSPeter Zijlstra struct rq *rq = rq_of_rt_rq(rt_rq); 9242679a837SHao Jia struct rq_flags rf; 925c249f255SDave Kleikamp int skip; 926c249f255SDave Kleikamp 927c249f255SDave Kleikamp /* 928c249f255SDave Kleikamp * When span == cpu_online_mask, taking each rq->lock 929c249f255SDave Kleikamp * can be time-consuming. Try to avoid it when possible. 930c249f255SDave Kleikamp */ 931c249f255SDave Kleikamp raw_spin_lock(&rt_rq->rt_runtime_lock); 932f3d133eeSHailong Liu if (!sched_feat(RT_RUNTIME_SHARE) && rt_rq->rt_runtime != RUNTIME_INF) 933f3d133eeSHailong Liu rt_rq->rt_runtime = rt_b->rt_runtime; 934c249f255SDave Kleikamp skip = !rt_rq->rt_time && !rt_rq->rt_nr_running; 935c249f255SDave Kleikamp raw_spin_unlock(&rt_rq->rt_runtime_lock); 936c249f255SDave Kleikamp if (skip) 937c249f255SDave Kleikamp continue; 938391e43daSPeter Zijlstra 9392679a837SHao Jia rq_lock(rq, &rf); 940d29a2064SDavidlohr Bueso update_rq_clock(rq); 941d29a2064SDavidlohr Bueso 942391e43daSPeter Zijlstra if (rt_rq->rt_time) { 943391e43daSPeter Zijlstra u64 runtime; 944391e43daSPeter Zijlstra 945391e43daSPeter Zijlstra raw_spin_lock(&rt_rq->rt_runtime_lock); 946391e43daSPeter Zijlstra if (rt_rq->rt_throttled) 947391e43daSPeter Zijlstra balance_runtime(rt_rq); 948391e43daSPeter Zijlstra runtime = rt_rq->rt_runtime; 949391e43daSPeter Zijlstra rt_rq->rt_time -= min(rt_rq->rt_time, overrun*runtime); 950391e43daSPeter Zijlstra if (rt_rq->rt_throttled && rt_rq->rt_time < runtime) { 951391e43daSPeter Zijlstra rt_rq->rt_throttled = 0; 952391e43daSPeter Zijlstra enqueue = 1; 953391e43daSPeter Zijlstra 954391e43daSPeter Zijlstra /* 9559edfbfedSPeter Zijlstra * When we're idle and a woken (rt) task is 9569edfbfedSPeter Zijlstra * throttled check_preempt_curr() will set 9579edfbfedSPeter Zijlstra * skip_update and the time between the wakeup 9589edfbfedSPeter Zijlstra * and this unthrottle will get accounted as 9599edfbfedSPeter Zijlstra * 'runtime'. 960391e43daSPeter Zijlstra */ 961391e43daSPeter Zijlstra if (rt_rq->rt_nr_running && rq->curr == rq->idle) 962adcc8da8SDavidlohr Bueso rq_clock_cancel_skipupdate(rq); 963391e43daSPeter Zijlstra } 964391e43daSPeter Zijlstra if (rt_rq->rt_time || rt_rq->rt_nr_running) 965391e43daSPeter Zijlstra idle = 0; 966391e43daSPeter Zijlstra raw_spin_unlock(&rt_rq->rt_runtime_lock); 967391e43daSPeter Zijlstra } else if (rt_rq->rt_nr_running) { 968391e43daSPeter Zijlstra idle = 0; 969391e43daSPeter Zijlstra if (!rt_rq_throttled(rt_rq)) 970391e43daSPeter Zijlstra enqueue = 1; 971391e43daSPeter Zijlstra } 97242c62a58SPeter Zijlstra if (rt_rq->rt_throttled) 97342c62a58SPeter Zijlstra throttled = 1; 974391e43daSPeter Zijlstra 975391e43daSPeter Zijlstra if (enqueue) 976391e43daSPeter Zijlstra sched_rt_rq_enqueue(rt_rq); 9772679a837SHao Jia rq_unlock(rq, &rf); 978391e43daSPeter Zijlstra } 979391e43daSPeter Zijlstra 98042c62a58SPeter Zijlstra if (!throttled && (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)) 98142c62a58SPeter Zijlstra return 1; 98242c62a58SPeter Zijlstra 983391e43daSPeter Zijlstra return idle; 984391e43daSPeter Zijlstra } 985391e43daSPeter Zijlstra 986391e43daSPeter Zijlstra static inline int rt_se_prio(struct sched_rt_entity *rt_se) 987391e43daSPeter Zijlstra { 988391e43daSPeter Zijlstra #ifdef CONFIG_RT_GROUP_SCHED 989391e43daSPeter Zijlstra struct rt_rq *rt_rq = group_rt_rq(rt_se); 990391e43daSPeter Zijlstra 991391e43daSPeter Zijlstra if (rt_rq) 992391e43daSPeter Zijlstra return rt_rq->highest_prio.curr; 993391e43daSPeter Zijlstra #endif 994391e43daSPeter Zijlstra 995391e43daSPeter Zijlstra return rt_task_of(rt_se)->prio; 996391e43daSPeter Zijlstra } 997391e43daSPeter Zijlstra 998391e43daSPeter Zijlstra static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq) 999391e43daSPeter Zijlstra { 1000391e43daSPeter Zijlstra u64 runtime = sched_rt_runtime(rt_rq); 1001391e43daSPeter Zijlstra 1002391e43daSPeter Zijlstra if (rt_rq->rt_throttled) 1003391e43daSPeter Zijlstra return rt_rq_throttled(rt_rq); 1004391e43daSPeter Zijlstra 10055b680fd6SShan Hai if (runtime >= sched_rt_period(rt_rq)) 1006391e43daSPeter Zijlstra return 0; 1007391e43daSPeter Zijlstra 1008391e43daSPeter Zijlstra balance_runtime(rt_rq); 1009391e43daSPeter Zijlstra runtime = sched_rt_runtime(rt_rq); 1010391e43daSPeter Zijlstra if (runtime == RUNTIME_INF) 1011391e43daSPeter Zijlstra return 0; 1012391e43daSPeter Zijlstra 1013391e43daSPeter Zijlstra if (rt_rq->rt_time > runtime) { 10147abc63b1SPeter Zijlstra struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq); 10157abc63b1SPeter Zijlstra 10167abc63b1SPeter Zijlstra /* 10177abc63b1SPeter Zijlstra * Don't actually throttle groups that have no runtime assigned 10187abc63b1SPeter Zijlstra * but accrue some time due to boosting. 10197abc63b1SPeter Zijlstra */ 10207abc63b1SPeter Zijlstra if (likely(rt_b->rt_runtime)) { 1021391e43daSPeter Zijlstra rt_rq->rt_throttled = 1; 1022c224815dSJohn Stultz printk_deferred_once("sched: RT throttling activated\n"); 10237abc63b1SPeter Zijlstra } else { 10247abc63b1SPeter Zijlstra /* 10257abc63b1SPeter Zijlstra * In case we did anyway, make it go away, 10267abc63b1SPeter Zijlstra * replenishment is a joke, since it will replenish us 10277abc63b1SPeter Zijlstra * with exactly 0 ns. 10287abc63b1SPeter Zijlstra */ 10297abc63b1SPeter Zijlstra rt_rq->rt_time = 0; 10307abc63b1SPeter Zijlstra } 10317abc63b1SPeter Zijlstra 1032391e43daSPeter Zijlstra if (rt_rq_throttled(rt_rq)) { 1033391e43daSPeter Zijlstra sched_rt_rq_dequeue(rt_rq); 1034391e43daSPeter Zijlstra return 1; 1035391e43daSPeter Zijlstra } 1036391e43daSPeter Zijlstra } 1037391e43daSPeter Zijlstra 1038391e43daSPeter Zijlstra return 0; 1039391e43daSPeter Zijlstra } 1040391e43daSPeter Zijlstra 1041391e43daSPeter Zijlstra /* 1042391e43daSPeter Zijlstra * Update the current task's runtime statistics. Skip current tasks that 1043391e43daSPeter Zijlstra * are not in our scheduling class. 1044391e43daSPeter Zijlstra */ 1045391e43daSPeter Zijlstra static void update_curr_rt(struct rq *rq) 1046391e43daSPeter Zijlstra { 1047391e43daSPeter Zijlstra struct task_struct *curr = rq->curr; 1048391e43daSPeter Zijlstra struct sched_rt_entity *rt_se = &curr->rt; 1049391e43daSPeter Zijlstra u64 delta_exec; 1050a7711602SWen Yang u64 now; 1051391e43daSPeter Zijlstra 1052391e43daSPeter Zijlstra if (curr->sched_class != &rt_sched_class) 1053391e43daSPeter Zijlstra return; 1054391e43daSPeter Zijlstra 1055a7711602SWen Yang now = rq_clock_task(rq); 1056e7ad2031SWen Yang delta_exec = now - curr->se.exec_start; 1057fc79e240SKirill Tkhai if (unlikely((s64)delta_exec <= 0)) 1058fc79e240SKirill Tkhai return; 1059391e43daSPeter Zijlstra 1060ceeadb83SYafang Shao schedstat_set(curr->stats.exec_max, 1061ceeadb83SYafang Shao max(curr->stats.exec_max, delta_exec)); 1062391e43daSPeter Zijlstra 1063ed7b564cSYafang Shao trace_sched_stat_runtime(curr, delta_exec, 0); 1064ed7b564cSYafang Shao 10655531ecffSShang XiaoJing update_current_exec_runtime(curr, now, delta_exec); 1066391e43daSPeter Zijlstra 1067391e43daSPeter Zijlstra if (!rt_bandwidth_enabled()) 1068391e43daSPeter Zijlstra return; 1069391e43daSPeter Zijlstra 1070391e43daSPeter Zijlstra for_each_sched_rt_entity(rt_se) { 10710b07939cSGiedrius Rekasius struct rt_rq *rt_rq = rt_rq_of_se(rt_se); 10729b58e976SLi Hua int exceeded; 1073391e43daSPeter Zijlstra 1074391e43daSPeter Zijlstra if (sched_rt_runtime(rt_rq) != RUNTIME_INF) { 1075391e43daSPeter Zijlstra raw_spin_lock(&rt_rq->rt_runtime_lock); 1076391e43daSPeter Zijlstra rt_rq->rt_time += delta_exec; 10779b58e976SLi Hua exceeded = sched_rt_runtime_exceeded(rt_rq); 10789b58e976SLi Hua if (exceeded) 10798875125eSKirill Tkhai resched_curr(rq); 1080391e43daSPeter Zijlstra raw_spin_unlock(&rt_rq->rt_runtime_lock); 10819b58e976SLi Hua if (exceeded) 10829b58e976SLi Hua do_start_rt_bandwidth(sched_rt_bandwidth(rt_rq)); 1083391e43daSPeter Zijlstra } 1084391e43daSPeter Zijlstra } 1085391e43daSPeter Zijlstra } 1086391e43daSPeter Zijlstra 1087f4ebcbc0SKirill Tkhai static void 10885c66d1b9SNicolas Saenz Julienne dequeue_top_rt_rq(struct rt_rq *rt_rq, unsigned int count) 1089f4ebcbc0SKirill Tkhai { 1090f4ebcbc0SKirill Tkhai struct rq *rq = rq_of_rt_rq(rt_rq); 1091f4ebcbc0SKirill Tkhai 1092f4ebcbc0SKirill Tkhai BUG_ON(&rq->rt != rt_rq); 1093f4ebcbc0SKirill Tkhai 1094f4ebcbc0SKirill Tkhai if (!rt_rq->rt_queued) 1095f4ebcbc0SKirill Tkhai return; 1096f4ebcbc0SKirill Tkhai 1097f4ebcbc0SKirill Tkhai BUG_ON(!rq->nr_running); 1098f4ebcbc0SKirill Tkhai 10995c66d1b9SNicolas Saenz Julienne sub_nr_running(rq, count); 1100f4ebcbc0SKirill Tkhai rt_rq->rt_queued = 0; 11018f111bc3SPeter Zijlstra 1102f4ebcbc0SKirill Tkhai } 1103f4ebcbc0SKirill Tkhai 1104f4ebcbc0SKirill Tkhai static void 1105f4ebcbc0SKirill Tkhai enqueue_top_rt_rq(struct rt_rq *rt_rq) 1106f4ebcbc0SKirill Tkhai { 1107f4ebcbc0SKirill Tkhai struct rq *rq = rq_of_rt_rq(rt_rq); 1108f4ebcbc0SKirill Tkhai 1109f4ebcbc0SKirill Tkhai BUG_ON(&rq->rt != rt_rq); 1110f4ebcbc0SKirill Tkhai 1111f4ebcbc0SKirill Tkhai if (rt_rq->rt_queued) 1112f4ebcbc0SKirill Tkhai return; 1113296b2ffeSVincent Guittot 1114296b2ffeSVincent Guittot if (rt_rq_throttled(rt_rq)) 1115f4ebcbc0SKirill Tkhai return; 1116f4ebcbc0SKirill Tkhai 1117296b2ffeSVincent Guittot if (rt_rq->rt_nr_running) { 111872465447SKirill Tkhai add_nr_running(rq, rt_rq->rt_nr_running); 1119f4ebcbc0SKirill Tkhai rt_rq->rt_queued = 1; 1120296b2ffeSVincent Guittot } 11218f111bc3SPeter Zijlstra 11228f111bc3SPeter Zijlstra /* Kick cpufreq (see the comment in kernel/sched/sched.h). */ 11238f111bc3SPeter Zijlstra cpufreq_update_util(rq, 0); 1124f4ebcbc0SKirill Tkhai } 1125f4ebcbc0SKirill Tkhai 1126391e43daSPeter Zijlstra #if defined CONFIG_SMP 1127391e43daSPeter Zijlstra 1128391e43daSPeter Zijlstra static void 1129391e43daSPeter Zijlstra inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) 1130391e43daSPeter Zijlstra { 1131391e43daSPeter Zijlstra struct rq *rq = rq_of_rt_rq(rt_rq); 1132391e43daSPeter Zijlstra 1133757dfcaaSKirill Tkhai #ifdef CONFIG_RT_GROUP_SCHED 1134757dfcaaSKirill Tkhai /* 1135757dfcaaSKirill Tkhai * Change rq's cpupri only if rt_rq is the top queue. 1136757dfcaaSKirill Tkhai */ 1137757dfcaaSKirill Tkhai if (&rq->rt != rt_rq) 1138757dfcaaSKirill Tkhai return; 1139757dfcaaSKirill Tkhai #endif 1140391e43daSPeter Zijlstra if (rq->online && prio < prev_prio) 1141391e43daSPeter Zijlstra cpupri_set(&rq->rd->cpupri, rq->cpu, prio); 1142391e43daSPeter Zijlstra } 1143391e43daSPeter Zijlstra 1144391e43daSPeter Zijlstra static void 1145391e43daSPeter Zijlstra dec_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) 1146391e43daSPeter Zijlstra { 1147391e43daSPeter Zijlstra struct rq *rq = rq_of_rt_rq(rt_rq); 1148391e43daSPeter Zijlstra 1149757dfcaaSKirill Tkhai #ifdef CONFIG_RT_GROUP_SCHED 1150757dfcaaSKirill Tkhai /* 1151757dfcaaSKirill Tkhai * Change rq's cpupri only if rt_rq is the top queue. 1152757dfcaaSKirill Tkhai */ 1153757dfcaaSKirill Tkhai if (&rq->rt != rt_rq) 1154757dfcaaSKirill Tkhai return; 1155757dfcaaSKirill Tkhai #endif 1156391e43daSPeter Zijlstra if (rq->online && rt_rq->highest_prio.curr != prev_prio) 1157391e43daSPeter Zijlstra cpupri_set(&rq->rd->cpupri, rq->cpu, rt_rq->highest_prio.curr); 1158391e43daSPeter Zijlstra } 1159391e43daSPeter Zijlstra 1160391e43daSPeter Zijlstra #else /* CONFIG_SMP */ 1161391e43daSPeter Zijlstra 1162391e43daSPeter Zijlstra static inline 1163391e43daSPeter Zijlstra void inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) {} 1164391e43daSPeter Zijlstra static inline 1165391e43daSPeter Zijlstra void dec_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) {} 1166391e43daSPeter Zijlstra 1167391e43daSPeter Zijlstra #endif /* CONFIG_SMP */ 1168391e43daSPeter Zijlstra 1169391e43daSPeter Zijlstra #if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED 1170391e43daSPeter Zijlstra static void 1171391e43daSPeter Zijlstra inc_rt_prio(struct rt_rq *rt_rq, int prio) 1172391e43daSPeter Zijlstra { 1173391e43daSPeter Zijlstra int prev_prio = rt_rq->highest_prio.curr; 1174391e43daSPeter Zijlstra 1175391e43daSPeter Zijlstra if (prio < prev_prio) 1176391e43daSPeter Zijlstra rt_rq->highest_prio.curr = prio; 1177391e43daSPeter Zijlstra 1178391e43daSPeter Zijlstra inc_rt_prio_smp(rt_rq, prio, prev_prio); 1179391e43daSPeter Zijlstra } 1180391e43daSPeter Zijlstra 1181391e43daSPeter Zijlstra static void 1182391e43daSPeter Zijlstra dec_rt_prio(struct rt_rq *rt_rq, int prio) 1183391e43daSPeter Zijlstra { 1184391e43daSPeter Zijlstra int prev_prio = rt_rq->highest_prio.curr; 1185391e43daSPeter Zijlstra 1186391e43daSPeter Zijlstra if (rt_rq->rt_nr_running) { 1187391e43daSPeter Zijlstra 1188391e43daSPeter Zijlstra WARN_ON(prio < prev_prio); 1189391e43daSPeter Zijlstra 1190391e43daSPeter Zijlstra /* 1191391e43daSPeter Zijlstra * This may have been our highest task, and therefore 1192391e43daSPeter Zijlstra * we may have some recomputation to do 1193391e43daSPeter Zijlstra */ 1194391e43daSPeter Zijlstra if (prio == prev_prio) { 1195391e43daSPeter Zijlstra struct rt_prio_array *array = &rt_rq->active; 1196391e43daSPeter Zijlstra 1197391e43daSPeter Zijlstra rt_rq->highest_prio.curr = 1198391e43daSPeter Zijlstra sched_find_first_bit(array->bitmap); 1199391e43daSPeter Zijlstra } 1200391e43daSPeter Zijlstra 1201934fc331SPeter Zijlstra } else { 1202934fc331SPeter Zijlstra rt_rq->highest_prio.curr = MAX_RT_PRIO-1; 1203934fc331SPeter Zijlstra } 1204391e43daSPeter Zijlstra 1205391e43daSPeter Zijlstra dec_rt_prio_smp(rt_rq, prio, prev_prio); 1206391e43daSPeter Zijlstra } 1207391e43daSPeter Zijlstra 1208391e43daSPeter Zijlstra #else 1209391e43daSPeter Zijlstra 1210391e43daSPeter Zijlstra static inline void inc_rt_prio(struct rt_rq *rt_rq, int prio) {} 1211391e43daSPeter Zijlstra static inline void dec_rt_prio(struct rt_rq *rt_rq, int prio) {} 1212391e43daSPeter Zijlstra 1213391e43daSPeter Zijlstra #endif /* CONFIG_SMP || CONFIG_RT_GROUP_SCHED */ 1214391e43daSPeter Zijlstra 1215391e43daSPeter Zijlstra #ifdef CONFIG_RT_GROUP_SCHED 1216391e43daSPeter Zijlstra 1217391e43daSPeter Zijlstra static void 1218391e43daSPeter Zijlstra inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) 1219391e43daSPeter Zijlstra { 1220391e43daSPeter Zijlstra if (rt_se_boosted(rt_se)) 1221391e43daSPeter Zijlstra rt_rq->rt_nr_boosted++; 1222391e43daSPeter Zijlstra 1223391e43daSPeter Zijlstra if (rt_rq->tg) 1224391e43daSPeter Zijlstra start_rt_bandwidth(&rt_rq->tg->rt_bandwidth); 1225391e43daSPeter Zijlstra } 1226391e43daSPeter Zijlstra 1227391e43daSPeter Zijlstra static void 1228391e43daSPeter Zijlstra dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) 1229391e43daSPeter Zijlstra { 1230391e43daSPeter Zijlstra if (rt_se_boosted(rt_se)) 1231391e43daSPeter Zijlstra rt_rq->rt_nr_boosted--; 1232391e43daSPeter Zijlstra 1233391e43daSPeter Zijlstra WARN_ON(!rt_rq->rt_nr_running && rt_rq->rt_nr_boosted); 1234391e43daSPeter Zijlstra } 1235391e43daSPeter Zijlstra 1236391e43daSPeter Zijlstra #else /* CONFIG_RT_GROUP_SCHED */ 1237391e43daSPeter Zijlstra 1238391e43daSPeter Zijlstra static void 1239391e43daSPeter Zijlstra inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) 1240391e43daSPeter Zijlstra { 1241391e43daSPeter Zijlstra start_rt_bandwidth(&def_rt_bandwidth); 1242391e43daSPeter Zijlstra } 1243391e43daSPeter Zijlstra 1244391e43daSPeter Zijlstra static inline 1245391e43daSPeter Zijlstra void dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) {} 1246391e43daSPeter Zijlstra 1247391e43daSPeter Zijlstra #endif /* CONFIG_RT_GROUP_SCHED */ 1248391e43daSPeter Zijlstra 1249391e43daSPeter Zijlstra static inline 125022abdef3SKirill Tkhai unsigned int rt_se_nr_running(struct sched_rt_entity *rt_se) 125122abdef3SKirill Tkhai { 125222abdef3SKirill Tkhai struct rt_rq *group_rq = group_rt_rq(rt_se); 125322abdef3SKirill Tkhai 125422abdef3SKirill Tkhai if (group_rq) 125522abdef3SKirill Tkhai return group_rq->rt_nr_running; 125622abdef3SKirill Tkhai else 125722abdef3SKirill Tkhai return 1; 125822abdef3SKirill Tkhai } 125922abdef3SKirill Tkhai 126022abdef3SKirill Tkhai static inline 126101d36d0aSFrederic Weisbecker unsigned int rt_se_rr_nr_running(struct sched_rt_entity *rt_se) 126201d36d0aSFrederic Weisbecker { 126301d36d0aSFrederic Weisbecker struct rt_rq *group_rq = group_rt_rq(rt_se); 126401d36d0aSFrederic Weisbecker struct task_struct *tsk; 126501d36d0aSFrederic Weisbecker 126601d36d0aSFrederic Weisbecker if (group_rq) 126701d36d0aSFrederic Weisbecker return group_rq->rr_nr_running; 126801d36d0aSFrederic Weisbecker 126901d36d0aSFrederic Weisbecker tsk = rt_task_of(rt_se); 127001d36d0aSFrederic Weisbecker 127101d36d0aSFrederic Weisbecker return (tsk->policy == SCHED_RR) ? 1 : 0; 127201d36d0aSFrederic Weisbecker } 127301d36d0aSFrederic Weisbecker 127401d36d0aSFrederic Weisbecker static inline 1275391e43daSPeter Zijlstra void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) 1276391e43daSPeter Zijlstra { 1277391e43daSPeter Zijlstra int prio = rt_se_prio(rt_se); 1278391e43daSPeter Zijlstra 1279391e43daSPeter Zijlstra WARN_ON(!rt_prio(prio)); 128022abdef3SKirill Tkhai rt_rq->rt_nr_running += rt_se_nr_running(rt_se); 128101d36d0aSFrederic Weisbecker rt_rq->rr_nr_running += rt_se_rr_nr_running(rt_se); 1282391e43daSPeter Zijlstra 1283391e43daSPeter Zijlstra inc_rt_prio(rt_rq, prio); 1284391e43daSPeter Zijlstra inc_rt_migration(rt_se, rt_rq); 1285391e43daSPeter Zijlstra inc_rt_group(rt_se, rt_rq); 1286391e43daSPeter Zijlstra } 1287391e43daSPeter Zijlstra 1288391e43daSPeter Zijlstra static inline 1289391e43daSPeter Zijlstra void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) 1290391e43daSPeter Zijlstra { 1291391e43daSPeter Zijlstra WARN_ON(!rt_prio(rt_se_prio(rt_se))); 1292391e43daSPeter Zijlstra WARN_ON(!rt_rq->rt_nr_running); 129322abdef3SKirill Tkhai rt_rq->rt_nr_running -= rt_se_nr_running(rt_se); 129401d36d0aSFrederic Weisbecker rt_rq->rr_nr_running -= rt_se_rr_nr_running(rt_se); 1295391e43daSPeter Zijlstra 1296391e43daSPeter Zijlstra dec_rt_prio(rt_rq, rt_se_prio(rt_se)); 1297391e43daSPeter Zijlstra dec_rt_migration(rt_se, rt_rq); 1298391e43daSPeter Zijlstra dec_rt_group(rt_se, rt_rq); 1299391e43daSPeter Zijlstra } 1300391e43daSPeter Zijlstra 1301ff77e468SPeter Zijlstra /* 1302ff77e468SPeter Zijlstra * Change rt_se->run_list location unless SAVE && !MOVE 1303ff77e468SPeter Zijlstra * 1304ff77e468SPeter Zijlstra * assumes ENQUEUE/DEQUEUE flags match 1305ff77e468SPeter Zijlstra */ 1306ff77e468SPeter Zijlstra static inline bool move_entity(unsigned int flags) 1307ff77e468SPeter Zijlstra { 1308ff77e468SPeter Zijlstra if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) == DEQUEUE_SAVE) 1309ff77e468SPeter Zijlstra return false; 1310ff77e468SPeter Zijlstra 1311ff77e468SPeter Zijlstra return true; 1312ff77e468SPeter Zijlstra } 1313ff77e468SPeter Zijlstra 1314ff77e468SPeter Zijlstra static void __delist_rt_entity(struct sched_rt_entity *rt_se, struct rt_prio_array *array) 1315ff77e468SPeter Zijlstra { 1316ff77e468SPeter Zijlstra list_del_init(&rt_se->run_list); 1317ff77e468SPeter Zijlstra 1318ff77e468SPeter Zijlstra if (list_empty(array->queue + rt_se_prio(rt_se))) 1319ff77e468SPeter Zijlstra __clear_bit(rt_se_prio(rt_se), array->bitmap); 1320ff77e468SPeter Zijlstra 1321ff77e468SPeter Zijlstra rt_se->on_list = 0; 1322ff77e468SPeter Zijlstra } 1323ff77e468SPeter Zijlstra 132457a5c2daSYafang Shao static inline struct sched_statistics * 132557a5c2daSYafang Shao __schedstats_from_rt_se(struct sched_rt_entity *rt_se) 132657a5c2daSYafang Shao { 132757a5c2daSYafang Shao #ifdef CONFIG_RT_GROUP_SCHED 132857a5c2daSYafang Shao /* schedstats is not supported for rt group. */ 132957a5c2daSYafang Shao if (!rt_entity_is_task(rt_se)) 133057a5c2daSYafang Shao return NULL; 133157a5c2daSYafang Shao #endif 133257a5c2daSYafang Shao 133357a5c2daSYafang Shao return &rt_task_of(rt_se)->stats; 133457a5c2daSYafang Shao } 133557a5c2daSYafang Shao 133657a5c2daSYafang Shao static inline void 133757a5c2daSYafang Shao update_stats_wait_start_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se) 133857a5c2daSYafang Shao { 133957a5c2daSYafang Shao struct sched_statistics *stats; 134057a5c2daSYafang Shao struct task_struct *p = NULL; 134157a5c2daSYafang Shao 134257a5c2daSYafang Shao if (!schedstat_enabled()) 134357a5c2daSYafang Shao return; 134457a5c2daSYafang Shao 134557a5c2daSYafang Shao if (rt_entity_is_task(rt_se)) 134657a5c2daSYafang Shao p = rt_task_of(rt_se); 134757a5c2daSYafang Shao 134857a5c2daSYafang Shao stats = __schedstats_from_rt_se(rt_se); 134957a5c2daSYafang Shao if (!stats) 135057a5c2daSYafang Shao return; 135157a5c2daSYafang Shao 135257a5c2daSYafang Shao __update_stats_wait_start(rq_of_rt_rq(rt_rq), p, stats); 135357a5c2daSYafang Shao } 135457a5c2daSYafang Shao 135557a5c2daSYafang Shao static inline void 135657a5c2daSYafang Shao update_stats_enqueue_sleeper_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se) 135757a5c2daSYafang Shao { 135857a5c2daSYafang Shao struct sched_statistics *stats; 135957a5c2daSYafang Shao struct task_struct *p = NULL; 136057a5c2daSYafang Shao 136157a5c2daSYafang Shao if (!schedstat_enabled()) 136257a5c2daSYafang Shao return; 136357a5c2daSYafang Shao 136457a5c2daSYafang Shao if (rt_entity_is_task(rt_se)) 136557a5c2daSYafang Shao p = rt_task_of(rt_se); 136657a5c2daSYafang Shao 136757a5c2daSYafang Shao stats = __schedstats_from_rt_se(rt_se); 136857a5c2daSYafang Shao if (!stats) 136957a5c2daSYafang Shao return; 137057a5c2daSYafang Shao 137157a5c2daSYafang Shao __update_stats_enqueue_sleeper(rq_of_rt_rq(rt_rq), p, stats); 137257a5c2daSYafang Shao } 137357a5c2daSYafang Shao 137457a5c2daSYafang Shao static inline void 137557a5c2daSYafang Shao update_stats_enqueue_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, 137657a5c2daSYafang Shao int flags) 137757a5c2daSYafang Shao { 137857a5c2daSYafang Shao if (!schedstat_enabled()) 137957a5c2daSYafang Shao return; 138057a5c2daSYafang Shao 138157a5c2daSYafang Shao if (flags & ENQUEUE_WAKEUP) 138257a5c2daSYafang Shao update_stats_enqueue_sleeper_rt(rt_rq, rt_se); 138357a5c2daSYafang Shao } 138457a5c2daSYafang Shao 138557a5c2daSYafang Shao static inline void 138657a5c2daSYafang Shao update_stats_wait_end_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se) 138757a5c2daSYafang Shao { 138857a5c2daSYafang Shao struct sched_statistics *stats; 138957a5c2daSYafang Shao struct task_struct *p = NULL; 139057a5c2daSYafang Shao 139157a5c2daSYafang Shao if (!schedstat_enabled()) 139257a5c2daSYafang Shao return; 139357a5c2daSYafang Shao 139457a5c2daSYafang Shao if (rt_entity_is_task(rt_se)) 139557a5c2daSYafang Shao p = rt_task_of(rt_se); 139657a5c2daSYafang Shao 139757a5c2daSYafang Shao stats = __schedstats_from_rt_se(rt_se); 139857a5c2daSYafang Shao if (!stats) 139957a5c2daSYafang Shao return; 140057a5c2daSYafang Shao 140157a5c2daSYafang Shao __update_stats_wait_end(rq_of_rt_rq(rt_rq), p, stats); 140257a5c2daSYafang Shao } 140357a5c2daSYafang Shao 140457a5c2daSYafang Shao static inline void 140557a5c2daSYafang Shao update_stats_dequeue_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, 140657a5c2daSYafang Shao int flags) 140757a5c2daSYafang Shao { 140857a5c2daSYafang Shao struct task_struct *p = NULL; 140957a5c2daSYafang Shao 141057a5c2daSYafang Shao if (!schedstat_enabled()) 141157a5c2daSYafang Shao return; 141257a5c2daSYafang Shao 141357a5c2daSYafang Shao if (rt_entity_is_task(rt_se)) 141457a5c2daSYafang Shao p = rt_task_of(rt_se); 141557a5c2daSYafang Shao 141657a5c2daSYafang Shao if ((flags & DEQUEUE_SLEEP) && p) { 141757a5c2daSYafang Shao unsigned int state; 141857a5c2daSYafang Shao 141957a5c2daSYafang Shao state = READ_ONCE(p->__state); 142057a5c2daSYafang Shao if (state & TASK_INTERRUPTIBLE) 142157a5c2daSYafang Shao __schedstat_set(p->stats.sleep_start, 142257a5c2daSYafang Shao rq_clock(rq_of_rt_rq(rt_rq))); 142357a5c2daSYafang Shao 142457a5c2daSYafang Shao if (state & TASK_UNINTERRUPTIBLE) 142557a5c2daSYafang Shao __schedstat_set(p->stats.block_start, 142657a5c2daSYafang Shao rq_clock(rq_of_rt_rq(rt_rq))); 142757a5c2daSYafang Shao } 142857a5c2daSYafang Shao } 142957a5c2daSYafang Shao 1430ff77e468SPeter Zijlstra static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags) 1431391e43daSPeter Zijlstra { 1432391e43daSPeter Zijlstra struct rt_rq *rt_rq = rt_rq_of_se(rt_se); 1433391e43daSPeter Zijlstra struct rt_prio_array *array = &rt_rq->active; 1434391e43daSPeter Zijlstra struct rt_rq *group_rq = group_rt_rq(rt_se); 1435391e43daSPeter Zijlstra struct list_head *queue = array->queue + rt_se_prio(rt_se); 1436391e43daSPeter Zijlstra 1437391e43daSPeter Zijlstra /* 1438391e43daSPeter Zijlstra * Don't enqueue the group if its throttled, or when empty. 1439391e43daSPeter Zijlstra * The latter is a consequence of the former when a child group 1440391e43daSPeter Zijlstra * get throttled and the current group doesn't have any other 1441391e43daSPeter Zijlstra * active members. 1442391e43daSPeter Zijlstra */ 1443ff77e468SPeter Zijlstra if (group_rq && (rt_rq_throttled(group_rq) || !group_rq->rt_nr_running)) { 1444ff77e468SPeter Zijlstra if (rt_se->on_list) 1445ff77e468SPeter Zijlstra __delist_rt_entity(rt_se, array); 1446391e43daSPeter Zijlstra return; 1447ff77e468SPeter Zijlstra } 1448391e43daSPeter Zijlstra 1449ff77e468SPeter Zijlstra if (move_entity(flags)) { 1450ff77e468SPeter Zijlstra WARN_ON_ONCE(rt_se->on_list); 1451ff77e468SPeter Zijlstra if (flags & ENQUEUE_HEAD) 1452391e43daSPeter Zijlstra list_add(&rt_se->run_list, queue); 1453391e43daSPeter Zijlstra else 1454391e43daSPeter Zijlstra list_add_tail(&rt_se->run_list, queue); 1455ff77e468SPeter Zijlstra 1456391e43daSPeter Zijlstra __set_bit(rt_se_prio(rt_se), array->bitmap); 1457ff77e468SPeter Zijlstra rt_se->on_list = 1; 1458ff77e468SPeter Zijlstra } 1459ff77e468SPeter Zijlstra rt_se->on_rq = 1; 1460391e43daSPeter Zijlstra 1461391e43daSPeter Zijlstra inc_rt_tasks(rt_se, rt_rq); 1462391e43daSPeter Zijlstra } 1463391e43daSPeter Zijlstra 1464ff77e468SPeter Zijlstra static void __dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags) 1465391e43daSPeter Zijlstra { 1466391e43daSPeter Zijlstra struct rt_rq *rt_rq = rt_rq_of_se(rt_se); 1467391e43daSPeter Zijlstra struct rt_prio_array *array = &rt_rq->active; 1468391e43daSPeter Zijlstra 1469ff77e468SPeter Zijlstra if (move_entity(flags)) { 1470ff77e468SPeter Zijlstra WARN_ON_ONCE(!rt_se->on_list); 1471ff77e468SPeter Zijlstra __delist_rt_entity(rt_se, array); 1472ff77e468SPeter Zijlstra } 1473ff77e468SPeter Zijlstra rt_se->on_rq = 0; 1474391e43daSPeter Zijlstra 1475391e43daSPeter Zijlstra dec_rt_tasks(rt_se, rt_rq); 1476391e43daSPeter Zijlstra } 1477391e43daSPeter Zijlstra 1478391e43daSPeter Zijlstra /* 1479391e43daSPeter Zijlstra * Because the prio of an upper entry depends on the lower 1480391e43daSPeter Zijlstra * entries, we must remove entries top - down. 1481391e43daSPeter Zijlstra */ 1482ff77e468SPeter Zijlstra static void dequeue_rt_stack(struct sched_rt_entity *rt_se, unsigned int flags) 1483391e43daSPeter Zijlstra { 1484391e43daSPeter Zijlstra struct sched_rt_entity *back = NULL; 14855c66d1b9SNicolas Saenz Julienne unsigned int rt_nr_running; 1486391e43daSPeter Zijlstra 1487391e43daSPeter Zijlstra for_each_sched_rt_entity(rt_se) { 1488391e43daSPeter Zijlstra rt_se->back = back; 1489391e43daSPeter Zijlstra back = rt_se; 1490391e43daSPeter Zijlstra } 1491391e43daSPeter Zijlstra 14925c66d1b9SNicolas Saenz Julienne rt_nr_running = rt_rq_of_se(back)->rt_nr_running; 1493f4ebcbc0SKirill Tkhai 1494391e43daSPeter Zijlstra for (rt_se = back; rt_se; rt_se = rt_se->back) { 1495391e43daSPeter Zijlstra if (on_rt_rq(rt_se)) 1496ff77e468SPeter Zijlstra __dequeue_rt_entity(rt_se, flags); 1497391e43daSPeter Zijlstra } 14985c66d1b9SNicolas Saenz Julienne 14995c66d1b9SNicolas Saenz Julienne dequeue_top_rt_rq(rt_rq_of_se(back), rt_nr_running); 1500391e43daSPeter Zijlstra } 1501391e43daSPeter Zijlstra 1502ff77e468SPeter Zijlstra static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags) 1503391e43daSPeter Zijlstra { 1504f4ebcbc0SKirill Tkhai struct rq *rq = rq_of_rt_se(rt_se); 1505f4ebcbc0SKirill Tkhai 150657a5c2daSYafang Shao update_stats_enqueue_rt(rt_rq_of_se(rt_se), rt_se, flags); 150757a5c2daSYafang Shao 1508ff77e468SPeter Zijlstra dequeue_rt_stack(rt_se, flags); 1509391e43daSPeter Zijlstra for_each_sched_rt_entity(rt_se) 1510ff77e468SPeter Zijlstra __enqueue_rt_entity(rt_se, flags); 1511f4ebcbc0SKirill Tkhai enqueue_top_rt_rq(&rq->rt); 1512391e43daSPeter Zijlstra } 1513391e43daSPeter Zijlstra 1514ff77e468SPeter Zijlstra static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags) 1515391e43daSPeter Zijlstra { 1516f4ebcbc0SKirill Tkhai struct rq *rq = rq_of_rt_se(rt_se); 1517f4ebcbc0SKirill Tkhai 151857a5c2daSYafang Shao update_stats_dequeue_rt(rt_rq_of_se(rt_se), rt_se, flags); 151957a5c2daSYafang Shao 1520ff77e468SPeter Zijlstra dequeue_rt_stack(rt_se, flags); 1521391e43daSPeter Zijlstra 1522391e43daSPeter Zijlstra for_each_sched_rt_entity(rt_se) { 1523391e43daSPeter Zijlstra struct rt_rq *rt_rq = group_rt_rq(rt_se); 1524391e43daSPeter Zijlstra 1525391e43daSPeter Zijlstra if (rt_rq && rt_rq->rt_nr_running) 1526ff77e468SPeter Zijlstra __enqueue_rt_entity(rt_se, flags); 1527391e43daSPeter Zijlstra } 1528f4ebcbc0SKirill Tkhai enqueue_top_rt_rq(&rq->rt); 1529391e43daSPeter Zijlstra } 1530391e43daSPeter Zijlstra 1531391e43daSPeter Zijlstra /* 1532391e43daSPeter Zijlstra * Adding/removing a task to/from a priority array: 1533391e43daSPeter Zijlstra */ 1534391e43daSPeter Zijlstra static void 1535391e43daSPeter Zijlstra enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags) 1536391e43daSPeter Zijlstra { 1537391e43daSPeter Zijlstra struct sched_rt_entity *rt_se = &p->rt; 1538391e43daSPeter Zijlstra 1539391e43daSPeter Zijlstra if (flags & ENQUEUE_WAKEUP) 1540391e43daSPeter Zijlstra rt_se->timeout = 0; 1541391e43daSPeter Zijlstra 154257a5c2daSYafang Shao check_schedstat_required(); 154357a5c2daSYafang Shao update_stats_wait_start_rt(rt_rq_of_se(rt_se), rt_se); 154457a5c2daSYafang Shao 1545ff77e468SPeter Zijlstra enqueue_rt_entity(rt_se, flags); 1546391e43daSPeter Zijlstra 15474b53a341SIngo Molnar if (!task_current(rq, p) && p->nr_cpus_allowed > 1) 1548391e43daSPeter Zijlstra enqueue_pushable_task(rq, p); 1549391e43daSPeter Zijlstra } 1550391e43daSPeter Zijlstra 1551391e43daSPeter Zijlstra static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags) 1552391e43daSPeter Zijlstra { 1553391e43daSPeter Zijlstra struct sched_rt_entity *rt_se = &p->rt; 1554391e43daSPeter Zijlstra 1555391e43daSPeter Zijlstra update_curr_rt(rq); 1556ff77e468SPeter Zijlstra dequeue_rt_entity(rt_se, flags); 1557391e43daSPeter Zijlstra 1558391e43daSPeter Zijlstra dequeue_pushable_task(rq, p); 1559391e43daSPeter Zijlstra } 1560391e43daSPeter Zijlstra 1561391e43daSPeter Zijlstra /* 1562391e43daSPeter Zijlstra * Put task to the head or the end of the run list without the overhead of 1563391e43daSPeter Zijlstra * dequeue followed by enqueue. 1564391e43daSPeter Zijlstra */ 1565391e43daSPeter Zijlstra static void 1566391e43daSPeter Zijlstra requeue_rt_entity(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, int head) 1567391e43daSPeter Zijlstra { 1568391e43daSPeter Zijlstra if (on_rt_rq(rt_se)) { 1569391e43daSPeter Zijlstra struct rt_prio_array *array = &rt_rq->active; 1570391e43daSPeter Zijlstra struct list_head *queue = array->queue + rt_se_prio(rt_se); 1571391e43daSPeter Zijlstra 1572391e43daSPeter Zijlstra if (head) 1573391e43daSPeter Zijlstra list_move(&rt_se->run_list, queue); 1574391e43daSPeter Zijlstra else 1575391e43daSPeter Zijlstra list_move_tail(&rt_se->run_list, queue); 1576391e43daSPeter Zijlstra } 1577391e43daSPeter Zijlstra } 1578391e43daSPeter Zijlstra 1579391e43daSPeter Zijlstra static void requeue_task_rt(struct rq *rq, struct task_struct *p, int head) 1580391e43daSPeter Zijlstra { 1581391e43daSPeter Zijlstra struct sched_rt_entity *rt_se = &p->rt; 1582391e43daSPeter Zijlstra struct rt_rq *rt_rq; 1583391e43daSPeter Zijlstra 1584391e43daSPeter Zijlstra for_each_sched_rt_entity(rt_se) { 1585391e43daSPeter Zijlstra rt_rq = rt_rq_of_se(rt_se); 1586391e43daSPeter Zijlstra requeue_rt_entity(rt_rq, rt_se, head); 1587391e43daSPeter Zijlstra } 1588391e43daSPeter Zijlstra } 1589391e43daSPeter Zijlstra 1590391e43daSPeter Zijlstra static void yield_task_rt(struct rq *rq) 1591391e43daSPeter Zijlstra { 1592391e43daSPeter Zijlstra requeue_task_rt(rq, rq->curr, 0); 1593391e43daSPeter Zijlstra } 1594391e43daSPeter Zijlstra 1595391e43daSPeter Zijlstra #ifdef CONFIG_SMP 1596391e43daSPeter Zijlstra static int find_lowest_rq(struct task_struct *task); 1597391e43daSPeter Zijlstra 1598391e43daSPeter Zijlstra static int 15993aef1551SValentin Schneider select_task_rq_rt(struct task_struct *p, int cpu, int flags) 1600391e43daSPeter Zijlstra { 1601391e43daSPeter Zijlstra struct task_struct *curr; 1602391e43daSPeter Zijlstra struct rq *rq; 1603804d402fSQais Yousef bool test; 1604391e43daSPeter Zijlstra 1605391e43daSPeter Zijlstra /* For anything but wake ups, just return the task_cpu */ 16063aef1551SValentin Schneider if (!(flags & (WF_TTWU | WF_FORK))) 1607391e43daSPeter Zijlstra goto out; 1608391e43daSPeter Zijlstra 1609391e43daSPeter Zijlstra rq = cpu_rq(cpu); 1610391e43daSPeter Zijlstra 1611391e43daSPeter Zijlstra rcu_read_lock(); 1612316c1608SJason Low curr = READ_ONCE(rq->curr); /* unlocked access */ 1613391e43daSPeter Zijlstra 1614391e43daSPeter Zijlstra /* 1615391e43daSPeter Zijlstra * If the current task on @p's runqueue is an RT task, then 1616391e43daSPeter Zijlstra * try to see if we can wake this RT task up on another 1617391e43daSPeter Zijlstra * runqueue. Otherwise simply start this RT task 1618391e43daSPeter Zijlstra * on its current runqueue. 1619391e43daSPeter Zijlstra * 1620391e43daSPeter Zijlstra * We want to avoid overloading runqueues. If the woken 1621391e43daSPeter Zijlstra * task is a higher priority, then it will stay on this CPU 1622391e43daSPeter Zijlstra * and the lower prio task should be moved to another CPU. 1623391e43daSPeter Zijlstra * Even though this will probably make the lower prio task 1624391e43daSPeter Zijlstra * lose its cache, we do not want to bounce a higher task 1625391e43daSPeter Zijlstra * around just because it gave up its CPU, perhaps for a 1626391e43daSPeter Zijlstra * lock? 1627391e43daSPeter Zijlstra * 1628391e43daSPeter Zijlstra * For equal prio tasks, we just let the scheduler sort it out. 1629391e43daSPeter Zijlstra * 1630391e43daSPeter Zijlstra * Otherwise, just let it ride on the affined RQ and the 1631391e43daSPeter Zijlstra * post-schedule router will push the preempted task away 1632391e43daSPeter Zijlstra * 1633391e43daSPeter Zijlstra * This test is optimistic, if we get it wrong the load-balancer 1634391e43daSPeter Zijlstra * will have to sort it out. 1635804d402fSQais Yousef * 1636804d402fSQais Yousef * We take into account the capacity of the CPU to ensure it fits the 1637804d402fSQais Yousef * requirement of the task - which is only important on heterogeneous 1638804d402fSQais Yousef * systems like big.LITTLE. 1639391e43daSPeter Zijlstra */ 1640804d402fSQais Yousef test = curr && 1641804d402fSQais Yousef unlikely(rt_task(curr)) && 1642804d402fSQais Yousef (curr->nr_cpus_allowed < 2 || curr->prio <= p->prio); 1643804d402fSQais Yousef 1644804d402fSQais Yousef if (test || !rt_task_fits_capacity(p, cpu)) { 1645391e43daSPeter Zijlstra int target = find_lowest_rq(p); 1646391e43daSPeter Zijlstra 164780e3d87bSTim Chen /* 1648b28bc1e0SQais Yousef * Bail out if we were forcing a migration to find a better 1649b28bc1e0SQais Yousef * fitting CPU but our search failed. 1650b28bc1e0SQais Yousef */ 1651b28bc1e0SQais Yousef if (!test && target != -1 && !rt_task_fits_capacity(p, target)) 1652b28bc1e0SQais Yousef goto out_unlock; 1653b28bc1e0SQais Yousef 1654b28bc1e0SQais Yousef /* 165580e3d87bSTim Chen * Don't bother moving it if the destination CPU is 165680e3d87bSTim Chen * not running a lower priority task. 165780e3d87bSTim Chen */ 165880e3d87bSTim Chen if (target != -1 && 165980e3d87bSTim Chen p->prio < cpu_rq(target)->rt.highest_prio.curr) 1660391e43daSPeter Zijlstra cpu = target; 1661391e43daSPeter Zijlstra } 1662b28bc1e0SQais Yousef 1663b28bc1e0SQais Yousef out_unlock: 1664391e43daSPeter Zijlstra rcu_read_unlock(); 1665391e43daSPeter Zijlstra 1666391e43daSPeter Zijlstra out: 1667391e43daSPeter Zijlstra return cpu; 1668391e43daSPeter Zijlstra } 1669391e43daSPeter Zijlstra 1670391e43daSPeter Zijlstra static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p) 1671391e43daSPeter Zijlstra { 1672308a623aSWanpeng Li /* 1673308a623aSWanpeng Li * Current can't be migrated, useless to reschedule, 1674308a623aSWanpeng Li * let's hope p can move out. 1675308a623aSWanpeng Li */ 16764b53a341SIngo Molnar if (rq->curr->nr_cpus_allowed == 1 || 1677a1bd02e1SQais Yousef !cpupri_find(&rq->rd->cpupri, rq->curr, NULL)) 1678391e43daSPeter Zijlstra return; 1679391e43daSPeter Zijlstra 1680308a623aSWanpeng Li /* 1681308a623aSWanpeng Li * p is migratable, so let's not schedule it and 1682308a623aSWanpeng Li * see if it is pushed or pulled somewhere else. 1683308a623aSWanpeng Li */ 1684804d402fSQais Yousef if (p->nr_cpus_allowed != 1 && 1685a1bd02e1SQais Yousef cpupri_find(&rq->rd->cpupri, p, NULL)) 1686391e43daSPeter Zijlstra return; 1687391e43daSPeter Zijlstra 1688391e43daSPeter Zijlstra /* 168997fb7a0aSIngo Molnar * There appear to be other CPUs that can accept 169097fb7a0aSIngo Molnar * the current task but none can run 'p', so lets reschedule 169197fb7a0aSIngo Molnar * to try and push the current task away: 1692391e43daSPeter Zijlstra */ 1693391e43daSPeter Zijlstra requeue_task_rt(rq, p, 1); 16948875125eSKirill Tkhai resched_curr(rq); 1695391e43daSPeter Zijlstra } 1696391e43daSPeter Zijlstra 16976e2df058SPeter Zijlstra static int balance_rt(struct rq *rq, struct task_struct *p, struct rq_flags *rf) 16986e2df058SPeter Zijlstra { 16996e2df058SPeter Zijlstra if (!on_rt_rq(&p->rt) && need_pull_rt_task(rq, p)) { 17006e2df058SPeter Zijlstra /* 17016e2df058SPeter Zijlstra * This is OK, because current is on_cpu, which avoids it being 17026e2df058SPeter Zijlstra * picked for load-balance and preemption/IRQs are still 17036e2df058SPeter Zijlstra * disabled avoiding further scheduler activity on it and we've 17046e2df058SPeter Zijlstra * not yet started the picking loop. 17056e2df058SPeter Zijlstra */ 17066e2df058SPeter Zijlstra rq_unpin_lock(rq, rf); 17076e2df058SPeter Zijlstra pull_rt_task(rq); 17086e2df058SPeter Zijlstra rq_repin_lock(rq, rf); 17096e2df058SPeter Zijlstra } 17106e2df058SPeter Zijlstra 17116e2df058SPeter Zijlstra return sched_stop_runnable(rq) || sched_dl_runnable(rq) || sched_rt_runnable(rq); 17126e2df058SPeter Zijlstra } 1713391e43daSPeter Zijlstra #endif /* CONFIG_SMP */ 1714391e43daSPeter Zijlstra 1715391e43daSPeter Zijlstra /* 1716391e43daSPeter Zijlstra * Preempt the current task with a newly woken task if needed: 1717391e43daSPeter Zijlstra */ 1718391e43daSPeter Zijlstra static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p, int flags) 1719391e43daSPeter Zijlstra { 1720391e43daSPeter Zijlstra if (p->prio < rq->curr->prio) { 17218875125eSKirill Tkhai resched_curr(rq); 1722391e43daSPeter Zijlstra return; 1723391e43daSPeter Zijlstra } 1724391e43daSPeter Zijlstra 1725391e43daSPeter Zijlstra #ifdef CONFIG_SMP 1726391e43daSPeter Zijlstra /* 1727391e43daSPeter Zijlstra * If: 1728391e43daSPeter Zijlstra * 1729391e43daSPeter Zijlstra * - the newly woken task is of equal priority to the current task 1730391e43daSPeter Zijlstra * - the newly woken task is non-migratable while current is migratable 1731391e43daSPeter Zijlstra * - current will be preempted on the next reschedule 1732391e43daSPeter Zijlstra * 1733391e43daSPeter Zijlstra * we should check to see if current can readily move to a different 1734391e43daSPeter Zijlstra * cpu. If so, we will reschedule to allow the push logic to try 1735391e43daSPeter Zijlstra * to move current somewhere else, making room for our non-migratable 1736391e43daSPeter Zijlstra * task. 1737391e43daSPeter Zijlstra */ 1738391e43daSPeter Zijlstra if (p->prio == rq->curr->prio && !test_tsk_need_resched(rq->curr)) 1739391e43daSPeter Zijlstra check_preempt_equal_prio(rq, p); 1740391e43daSPeter Zijlstra #endif 1741391e43daSPeter Zijlstra } 1742391e43daSPeter Zijlstra 1743a0e813f2SPeter Zijlstra static inline void set_next_task_rt(struct rq *rq, struct task_struct *p, bool first) 1744ff1cdc94SMuchun Song { 174557a5c2daSYafang Shao struct sched_rt_entity *rt_se = &p->rt; 174657a5c2daSYafang Shao struct rt_rq *rt_rq = &rq->rt; 174757a5c2daSYafang Shao 1748ff1cdc94SMuchun Song p->se.exec_start = rq_clock_task(rq); 174957a5c2daSYafang Shao if (on_rt_rq(&p->rt)) 175057a5c2daSYafang Shao update_stats_wait_end_rt(rt_rq, rt_se); 1751ff1cdc94SMuchun Song 1752ff1cdc94SMuchun Song /* The running task is never eligible for pushing */ 1753ff1cdc94SMuchun Song dequeue_pushable_task(rq, p); 1754f95d4eaeSPeter Zijlstra 1755a0e813f2SPeter Zijlstra if (!first) 1756a0e813f2SPeter Zijlstra return; 1757a0e813f2SPeter Zijlstra 1758f95d4eaeSPeter Zijlstra /* 1759f95d4eaeSPeter Zijlstra * If prev task was rt, put_prev_task() has already updated the 1760f95d4eaeSPeter Zijlstra * utilization. We only care of the case where we start to schedule a 1761f95d4eaeSPeter Zijlstra * rt task 1762f95d4eaeSPeter Zijlstra */ 1763f95d4eaeSPeter Zijlstra if (rq->curr->sched_class != &rt_sched_class) 1764f95d4eaeSPeter Zijlstra update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 0); 1765f95d4eaeSPeter Zijlstra 1766f95d4eaeSPeter Zijlstra rt_queue_push_tasks(rq); 1767ff1cdc94SMuchun Song } 1768ff1cdc94SMuchun Song 1769821aecd0SDietmar Eggemann static struct sched_rt_entity *pick_next_rt_entity(struct rt_rq *rt_rq) 1770391e43daSPeter Zijlstra { 1771391e43daSPeter Zijlstra struct rt_prio_array *array = &rt_rq->active; 1772391e43daSPeter Zijlstra struct sched_rt_entity *next = NULL; 1773391e43daSPeter Zijlstra struct list_head *queue; 1774391e43daSPeter Zijlstra int idx; 1775391e43daSPeter Zijlstra 1776391e43daSPeter Zijlstra idx = sched_find_first_bit(array->bitmap); 1777391e43daSPeter Zijlstra BUG_ON(idx >= MAX_RT_PRIO); 1778391e43daSPeter Zijlstra 1779391e43daSPeter Zijlstra queue = array->queue + idx; 17807c4a5b89SPietro Borrello if (SCHED_WARN_ON(list_empty(queue))) 17817c4a5b89SPietro Borrello return NULL; 1782391e43daSPeter Zijlstra next = list_entry(queue->next, struct sched_rt_entity, run_list); 1783391e43daSPeter Zijlstra 1784391e43daSPeter Zijlstra return next; 1785391e43daSPeter Zijlstra } 1786391e43daSPeter Zijlstra 1787391e43daSPeter Zijlstra static struct task_struct *_pick_next_task_rt(struct rq *rq) 1788391e43daSPeter Zijlstra { 1789391e43daSPeter Zijlstra struct sched_rt_entity *rt_se; 1790606dba2eSPeter Zijlstra struct rt_rq *rt_rq = &rq->rt; 1791391e43daSPeter Zijlstra 1792391e43daSPeter Zijlstra do { 1793821aecd0SDietmar Eggemann rt_se = pick_next_rt_entity(rt_rq); 17947c4a5b89SPietro Borrello if (unlikely(!rt_se)) 17957c4a5b89SPietro Borrello return NULL; 1796391e43daSPeter Zijlstra rt_rq = group_rt_rq(rt_se); 1797391e43daSPeter Zijlstra } while (rt_rq); 1798391e43daSPeter Zijlstra 1799ff1cdc94SMuchun Song return rt_task_of(rt_se); 1800391e43daSPeter Zijlstra } 1801391e43daSPeter Zijlstra 180221f56ffeSPeter Zijlstra static struct task_struct *pick_task_rt(struct rq *rq) 1803391e43daSPeter Zijlstra { 1804606dba2eSPeter Zijlstra struct task_struct *p; 1805606dba2eSPeter Zijlstra 18066e2df058SPeter Zijlstra if (!sched_rt_runnable(rq)) 1807606dba2eSPeter Zijlstra return NULL; 1808606dba2eSPeter Zijlstra 1809606dba2eSPeter Zijlstra p = _pick_next_task_rt(rq); 181021f56ffeSPeter Zijlstra 181121f56ffeSPeter Zijlstra return p; 181221f56ffeSPeter Zijlstra } 181321f56ffeSPeter Zijlstra 181421f56ffeSPeter Zijlstra static struct task_struct *pick_next_task_rt(struct rq *rq) 181521f56ffeSPeter Zijlstra { 181621f56ffeSPeter Zijlstra struct task_struct *p = pick_task_rt(rq); 181721f56ffeSPeter Zijlstra 181821f56ffeSPeter Zijlstra if (p) 1819a0e813f2SPeter Zijlstra set_next_task_rt(rq, p, true); 182021f56ffeSPeter Zijlstra 1821391e43daSPeter Zijlstra return p; 1822391e43daSPeter Zijlstra } 1823391e43daSPeter Zijlstra 18246e2df058SPeter Zijlstra static void put_prev_task_rt(struct rq *rq, struct task_struct *p) 1825391e43daSPeter Zijlstra { 182657a5c2daSYafang Shao struct sched_rt_entity *rt_se = &p->rt; 182757a5c2daSYafang Shao struct rt_rq *rt_rq = &rq->rt; 182857a5c2daSYafang Shao 182957a5c2daSYafang Shao if (on_rt_rq(&p->rt)) 183057a5c2daSYafang Shao update_stats_wait_start_rt(rt_rq, rt_se); 183157a5c2daSYafang Shao 1832391e43daSPeter Zijlstra update_curr_rt(rq); 1833391e43daSPeter Zijlstra 183423127296SVincent Guittot update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 1); 1835371bf427SVincent Guittot 1836391e43daSPeter Zijlstra /* 1837391e43daSPeter Zijlstra * The previous task needs to be made eligible for pushing 1838391e43daSPeter Zijlstra * if it is still active 1839391e43daSPeter Zijlstra */ 18404b53a341SIngo Molnar if (on_rt_rq(&p->rt) && p->nr_cpus_allowed > 1) 1841391e43daSPeter Zijlstra enqueue_pushable_task(rq, p); 1842391e43daSPeter Zijlstra } 1843391e43daSPeter Zijlstra 1844391e43daSPeter Zijlstra #ifdef CONFIG_SMP 1845391e43daSPeter Zijlstra 1846391e43daSPeter Zijlstra /* Only try algorithms three times */ 1847391e43daSPeter Zijlstra #define RT_MAX_TRIES 3 1848391e43daSPeter Zijlstra 1849391e43daSPeter Zijlstra static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu) 1850391e43daSPeter Zijlstra { 18510b9d46fcSPeter Zijlstra if (!task_on_cpu(rq, p) && 185295158a89SPeter Zijlstra cpumask_test_cpu(cpu, &p->cpus_mask)) 1853391e43daSPeter Zijlstra return 1; 185497fb7a0aSIngo Molnar 1855391e43daSPeter Zijlstra return 0; 1856391e43daSPeter Zijlstra } 1857391e43daSPeter Zijlstra 1858e23ee747SKirill Tkhai /* 1859e23ee747SKirill Tkhai * Return the highest pushable rq's task, which is suitable to be executed 186097fb7a0aSIngo Molnar * on the CPU, NULL otherwise 1861e23ee747SKirill Tkhai */ 1862e23ee747SKirill Tkhai static struct task_struct *pick_highest_pushable_task(struct rq *rq, int cpu) 1863391e43daSPeter Zijlstra { 1864e23ee747SKirill Tkhai struct plist_head *head = &rq->rt.pushable_tasks; 1865391e43daSPeter Zijlstra struct task_struct *p; 1866391e43daSPeter Zijlstra 1867e23ee747SKirill Tkhai if (!has_pushable_tasks(rq)) 1868e23ee747SKirill Tkhai return NULL; 1869391e43daSPeter Zijlstra 1870e23ee747SKirill Tkhai plist_for_each_entry(p, head, pushable_tasks) { 1871e23ee747SKirill Tkhai if (pick_rt_task(rq, p, cpu)) 1872e23ee747SKirill Tkhai return p; 1873391e43daSPeter Zijlstra } 1874391e43daSPeter Zijlstra 1875e23ee747SKirill Tkhai return NULL; 1876391e43daSPeter Zijlstra } 1877391e43daSPeter Zijlstra 1878391e43daSPeter Zijlstra static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask); 1879391e43daSPeter Zijlstra 1880391e43daSPeter Zijlstra static int find_lowest_rq(struct task_struct *task) 1881391e43daSPeter Zijlstra { 1882391e43daSPeter Zijlstra struct sched_domain *sd; 18834ba29684SChristoph Lameter struct cpumask *lowest_mask = this_cpu_cpumask_var_ptr(local_cpu_mask); 1884391e43daSPeter Zijlstra int this_cpu = smp_processor_id(); 1885391e43daSPeter Zijlstra int cpu = task_cpu(task); 1886a1bd02e1SQais Yousef int ret; 1887391e43daSPeter Zijlstra 1888391e43daSPeter Zijlstra /* Make sure the mask is initialized first */ 1889391e43daSPeter Zijlstra if (unlikely(!lowest_mask)) 1890391e43daSPeter Zijlstra return -1; 1891391e43daSPeter Zijlstra 18924b53a341SIngo Molnar if (task->nr_cpus_allowed == 1) 1893391e43daSPeter Zijlstra return -1; /* No other targets possible */ 1894391e43daSPeter Zijlstra 1895a1bd02e1SQais Yousef /* 1896a1bd02e1SQais Yousef * If we're on asym system ensure we consider the different capacities 1897a1bd02e1SQais Yousef * of the CPUs when searching for the lowest_mask. 1898a1bd02e1SQais Yousef */ 1899740cf8a7SDietmar Eggemann if (sched_asym_cpucap_active()) { 1900a1bd02e1SQais Yousef 1901a1bd02e1SQais Yousef ret = cpupri_find_fitness(&task_rq(task)->rd->cpupri, 1902a1bd02e1SQais Yousef task, lowest_mask, 1903a1bd02e1SQais Yousef rt_task_fits_capacity); 1904a1bd02e1SQais Yousef } else { 1905a1bd02e1SQais Yousef 1906a1bd02e1SQais Yousef ret = cpupri_find(&task_rq(task)->rd->cpupri, 1907a1bd02e1SQais Yousef task, lowest_mask); 1908a1bd02e1SQais Yousef } 1909a1bd02e1SQais Yousef 1910a1bd02e1SQais Yousef if (!ret) 1911391e43daSPeter Zijlstra return -1; /* No targets found */ 1912391e43daSPeter Zijlstra 1913391e43daSPeter Zijlstra /* 191497fb7a0aSIngo Molnar * At this point we have built a mask of CPUs representing the 1915391e43daSPeter Zijlstra * lowest priority tasks in the system. Now we want to elect 1916391e43daSPeter Zijlstra * the best one based on our affinity and topology. 1917391e43daSPeter Zijlstra * 191897fb7a0aSIngo Molnar * We prioritize the last CPU that the task executed on since 1919391e43daSPeter Zijlstra * it is most likely cache-hot in that location. 1920391e43daSPeter Zijlstra */ 1921391e43daSPeter Zijlstra if (cpumask_test_cpu(cpu, lowest_mask)) 1922391e43daSPeter Zijlstra return cpu; 1923391e43daSPeter Zijlstra 1924391e43daSPeter Zijlstra /* 1925391e43daSPeter Zijlstra * Otherwise, we consult the sched_domains span maps to figure 192697fb7a0aSIngo Molnar * out which CPU is logically closest to our hot cache data. 1927391e43daSPeter Zijlstra */ 1928391e43daSPeter Zijlstra if (!cpumask_test_cpu(this_cpu, lowest_mask)) 1929391e43daSPeter Zijlstra this_cpu = -1; /* Skip this_cpu opt if not among lowest */ 1930391e43daSPeter Zijlstra 1931391e43daSPeter Zijlstra rcu_read_lock(); 1932391e43daSPeter Zijlstra for_each_domain(cpu, sd) { 1933391e43daSPeter Zijlstra if (sd->flags & SD_WAKE_AFFINE) { 1934391e43daSPeter Zijlstra int best_cpu; 1935391e43daSPeter Zijlstra 1936391e43daSPeter Zijlstra /* 1937391e43daSPeter Zijlstra * "this_cpu" is cheaper to preempt than a 1938391e43daSPeter Zijlstra * remote processor. 1939391e43daSPeter Zijlstra */ 1940391e43daSPeter Zijlstra if (this_cpu != -1 && 1941391e43daSPeter Zijlstra cpumask_test_cpu(this_cpu, sched_domain_span(sd))) { 1942391e43daSPeter Zijlstra rcu_read_unlock(); 1943391e43daSPeter Zijlstra return this_cpu; 1944391e43daSPeter Zijlstra } 1945391e43daSPeter Zijlstra 194614e292f8SPeter Zijlstra best_cpu = cpumask_any_and_distribute(lowest_mask, 1947391e43daSPeter Zijlstra sched_domain_span(sd)); 1948391e43daSPeter Zijlstra if (best_cpu < nr_cpu_ids) { 1949391e43daSPeter Zijlstra rcu_read_unlock(); 1950391e43daSPeter Zijlstra return best_cpu; 1951391e43daSPeter Zijlstra } 1952391e43daSPeter Zijlstra } 1953391e43daSPeter Zijlstra } 1954391e43daSPeter Zijlstra rcu_read_unlock(); 1955391e43daSPeter Zijlstra 1956391e43daSPeter Zijlstra /* 1957391e43daSPeter Zijlstra * And finally, if there were no matches within the domains 1958391e43daSPeter Zijlstra * just give the caller *something* to work with from the compatible 1959391e43daSPeter Zijlstra * locations. 1960391e43daSPeter Zijlstra */ 1961391e43daSPeter Zijlstra if (this_cpu != -1) 1962391e43daSPeter Zijlstra return this_cpu; 1963391e43daSPeter Zijlstra 196414e292f8SPeter Zijlstra cpu = cpumask_any_distribute(lowest_mask); 1965391e43daSPeter Zijlstra if (cpu < nr_cpu_ids) 1966391e43daSPeter Zijlstra return cpu; 196797fb7a0aSIngo Molnar 1968391e43daSPeter Zijlstra return -1; 1969391e43daSPeter Zijlstra } 1970391e43daSPeter Zijlstra 1971391e43daSPeter Zijlstra /* Will lock the rq it finds */ 1972391e43daSPeter Zijlstra static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq) 1973391e43daSPeter Zijlstra { 1974391e43daSPeter Zijlstra struct rq *lowest_rq = NULL; 1975391e43daSPeter Zijlstra int tries; 1976391e43daSPeter Zijlstra int cpu; 1977391e43daSPeter Zijlstra 1978391e43daSPeter Zijlstra for (tries = 0; tries < RT_MAX_TRIES; tries++) { 1979391e43daSPeter Zijlstra cpu = find_lowest_rq(task); 1980391e43daSPeter Zijlstra 1981391e43daSPeter Zijlstra if ((cpu == -1) || (cpu == rq->cpu)) 1982391e43daSPeter Zijlstra break; 1983391e43daSPeter Zijlstra 1984391e43daSPeter Zijlstra lowest_rq = cpu_rq(cpu); 1985391e43daSPeter Zijlstra 198680e3d87bSTim Chen if (lowest_rq->rt.highest_prio.curr <= task->prio) { 198780e3d87bSTim Chen /* 198880e3d87bSTim Chen * Target rq has tasks of equal or higher priority, 198980e3d87bSTim Chen * retrying does not release any lock and is unlikely 199080e3d87bSTim Chen * to yield a different result. 199180e3d87bSTim Chen */ 199280e3d87bSTim Chen lowest_rq = NULL; 199380e3d87bSTim Chen break; 199480e3d87bSTim Chen } 199580e3d87bSTim Chen 1996391e43daSPeter Zijlstra /* if the prio of this runqueue changed, try again */ 1997391e43daSPeter Zijlstra if (double_lock_balance(rq, lowest_rq)) { 1998391e43daSPeter Zijlstra /* 1999391e43daSPeter Zijlstra * We had to unlock the run queue. In 2000391e43daSPeter Zijlstra * the mean time, task could have 2001391e43daSPeter Zijlstra * migrated already or had its affinity changed. 2002391e43daSPeter Zijlstra * Also make sure that it wasn't scheduled on its rq. 2003feffe5bbSSchspa Shi * It is possible the task was scheduled, set 2004feffe5bbSSchspa Shi * "migrate_disabled" and then got preempted, so we must 2005feffe5bbSSchspa Shi * check the task migration disable flag here too. 2006391e43daSPeter Zijlstra */ 2007391e43daSPeter Zijlstra if (unlikely(task_rq(task) != rq || 200895158a89SPeter Zijlstra !cpumask_test_cpu(lowest_rq->cpu, &task->cpus_mask) || 20090b9d46fcSPeter Zijlstra task_on_cpu(rq, task) || 201013b5ab02SXunlei Pang !rt_task(task) || 2011feffe5bbSSchspa Shi is_migration_disabled(task) || 2012da0c1e65SKirill Tkhai !task_on_rq_queued(task))) { 2013391e43daSPeter Zijlstra 20147f1b4393SPeter Zijlstra double_unlock_balance(rq, lowest_rq); 2015391e43daSPeter Zijlstra lowest_rq = NULL; 2016391e43daSPeter Zijlstra break; 2017391e43daSPeter Zijlstra } 2018391e43daSPeter Zijlstra } 2019391e43daSPeter Zijlstra 2020391e43daSPeter Zijlstra /* If this rq is still suitable use it. */ 2021391e43daSPeter Zijlstra if (lowest_rq->rt.highest_prio.curr > task->prio) 2022391e43daSPeter Zijlstra break; 2023391e43daSPeter Zijlstra 2024391e43daSPeter Zijlstra /* try again */ 2025391e43daSPeter Zijlstra double_unlock_balance(rq, lowest_rq); 2026391e43daSPeter Zijlstra lowest_rq = NULL; 2027391e43daSPeter Zijlstra } 2028391e43daSPeter Zijlstra 2029391e43daSPeter Zijlstra return lowest_rq; 2030391e43daSPeter Zijlstra } 2031391e43daSPeter Zijlstra 2032391e43daSPeter Zijlstra static struct task_struct *pick_next_pushable_task(struct rq *rq) 2033391e43daSPeter Zijlstra { 2034391e43daSPeter Zijlstra struct task_struct *p; 2035391e43daSPeter Zijlstra 2036391e43daSPeter Zijlstra if (!has_pushable_tasks(rq)) 2037391e43daSPeter Zijlstra return NULL; 2038391e43daSPeter Zijlstra 2039391e43daSPeter Zijlstra p = plist_first_entry(&rq->rt.pushable_tasks, 2040391e43daSPeter Zijlstra struct task_struct, pushable_tasks); 2041391e43daSPeter Zijlstra 2042391e43daSPeter Zijlstra BUG_ON(rq->cpu != task_cpu(p)); 2043391e43daSPeter Zijlstra BUG_ON(task_current(rq, p)); 20444b53a341SIngo Molnar BUG_ON(p->nr_cpus_allowed <= 1); 2045391e43daSPeter Zijlstra 2046da0c1e65SKirill Tkhai BUG_ON(!task_on_rq_queued(p)); 2047391e43daSPeter Zijlstra BUG_ON(!rt_task(p)); 2048391e43daSPeter Zijlstra 2049391e43daSPeter Zijlstra return p; 2050391e43daSPeter Zijlstra } 2051391e43daSPeter Zijlstra 2052391e43daSPeter Zijlstra /* 2053391e43daSPeter Zijlstra * If the current CPU has more than one RT task, see if the non 2054391e43daSPeter Zijlstra * running task can migrate over to a CPU that is running a task 2055391e43daSPeter Zijlstra * of lesser priority. 2056391e43daSPeter Zijlstra */ 2057a7c81556SPeter Zijlstra static int push_rt_task(struct rq *rq, bool pull) 2058391e43daSPeter Zijlstra { 2059391e43daSPeter Zijlstra struct task_struct *next_task; 2060391e43daSPeter Zijlstra struct rq *lowest_rq; 2061391e43daSPeter Zijlstra int ret = 0; 2062391e43daSPeter Zijlstra 2063391e43daSPeter Zijlstra if (!rq->rt.overloaded) 2064391e43daSPeter Zijlstra return 0; 2065391e43daSPeter Zijlstra 2066391e43daSPeter Zijlstra next_task = pick_next_pushable_task(rq); 2067391e43daSPeter Zijlstra if (!next_task) 2068391e43daSPeter Zijlstra return 0; 2069391e43daSPeter Zijlstra 2070391e43daSPeter Zijlstra retry: 207149bef33eSValentin Schneider /* 207249bef33eSValentin Schneider * It's possible that the next_task slipped in of 207349bef33eSValentin Schneider * higher priority than current. If that's the case 207449bef33eSValentin Schneider * just reschedule current. 207549bef33eSValentin Schneider */ 207649bef33eSValentin Schneider if (unlikely(next_task->prio < rq->curr->prio)) { 207749bef33eSValentin Schneider resched_curr(rq); 207849bef33eSValentin Schneider return 0; 207949bef33eSValentin Schneider } 208049bef33eSValentin Schneider 2081a7c81556SPeter Zijlstra if (is_migration_disabled(next_task)) { 2082a7c81556SPeter Zijlstra struct task_struct *push_task = NULL; 2083a7c81556SPeter Zijlstra int cpu; 2084a7c81556SPeter Zijlstra 2085a7c81556SPeter Zijlstra if (!pull || rq->push_busy) 2086a7c81556SPeter Zijlstra return 0; 2087a7c81556SPeter Zijlstra 208849bef33eSValentin Schneider /* 208949bef33eSValentin Schneider * Invoking find_lowest_rq() on anything but an RT task doesn't 209049bef33eSValentin Schneider * make sense. Per the above priority check, curr has to 209149bef33eSValentin Schneider * be of higher priority than next_task, so no need to 209249bef33eSValentin Schneider * reschedule when bailing out. 209349bef33eSValentin Schneider * 209449bef33eSValentin Schneider * Note that the stoppers are masqueraded as SCHED_FIFO 209549bef33eSValentin Schneider * (cf. sched_set_stop_task()), so we can't rely on rt_task(). 209649bef33eSValentin Schneider */ 209749bef33eSValentin Schneider if (rq->curr->sched_class != &rt_sched_class) 209849bef33eSValentin Schneider return 0; 209949bef33eSValentin Schneider 2100a7c81556SPeter Zijlstra cpu = find_lowest_rq(rq->curr); 2101a7c81556SPeter Zijlstra if (cpu == -1 || cpu == rq->cpu) 2102a7c81556SPeter Zijlstra return 0; 2103a7c81556SPeter Zijlstra 2104a7c81556SPeter Zijlstra /* 2105a7c81556SPeter Zijlstra * Given we found a CPU with lower priority than @next_task, 2106a7c81556SPeter Zijlstra * therefore it should be running. However we cannot migrate it 2107a7c81556SPeter Zijlstra * to this other CPU, instead attempt to push the current 2108a7c81556SPeter Zijlstra * running task on this CPU away. 2109a7c81556SPeter Zijlstra */ 2110a7c81556SPeter Zijlstra push_task = get_push_task(rq); 2111a7c81556SPeter Zijlstra if (push_task) { 21125cb9eaa3SPeter Zijlstra raw_spin_rq_unlock(rq); 2113a7c81556SPeter Zijlstra stop_one_cpu_nowait(rq->cpu, push_cpu_stop, 2114a7c81556SPeter Zijlstra push_task, &rq->push_work); 21155cb9eaa3SPeter Zijlstra raw_spin_rq_lock(rq); 2116a7c81556SPeter Zijlstra } 2117a7c81556SPeter Zijlstra 2118a7c81556SPeter Zijlstra return 0; 2119a7c81556SPeter Zijlstra } 2120a7c81556SPeter Zijlstra 21219ebc6053SYangtao Li if (WARN_ON(next_task == rq->curr)) 2122391e43daSPeter Zijlstra return 0; 2123391e43daSPeter Zijlstra 2124391e43daSPeter Zijlstra /* We might release rq lock */ 2125391e43daSPeter Zijlstra get_task_struct(next_task); 2126391e43daSPeter Zijlstra 2127391e43daSPeter Zijlstra /* find_lock_lowest_rq locks the rq if found */ 2128391e43daSPeter Zijlstra lowest_rq = find_lock_lowest_rq(next_task, rq); 2129391e43daSPeter Zijlstra if (!lowest_rq) { 2130391e43daSPeter Zijlstra struct task_struct *task; 2131391e43daSPeter Zijlstra /* 2132391e43daSPeter Zijlstra * find_lock_lowest_rq releases rq->lock 2133391e43daSPeter Zijlstra * so it is possible that next_task has migrated. 2134391e43daSPeter Zijlstra * 2135391e43daSPeter Zijlstra * We need to make sure that the task is still on the same 2136391e43daSPeter Zijlstra * run-queue and is also still the next task eligible for 2137391e43daSPeter Zijlstra * pushing. 2138391e43daSPeter Zijlstra */ 2139391e43daSPeter Zijlstra task = pick_next_pushable_task(rq); 2140de16b91eSByungchul Park if (task == next_task) { 2141391e43daSPeter Zijlstra /* 2142391e43daSPeter Zijlstra * The task hasn't migrated, and is still the next 2143391e43daSPeter Zijlstra * eligible task, but we failed to find a run-queue 2144391e43daSPeter Zijlstra * to push it to. Do not retry in this case, since 214597fb7a0aSIngo Molnar * other CPUs will pull from us when ready. 2146391e43daSPeter Zijlstra */ 2147391e43daSPeter Zijlstra goto out; 2148391e43daSPeter Zijlstra } 2149391e43daSPeter Zijlstra 2150391e43daSPeter Zijlstra if (!task) 2151391e43daSPeter Zijlstra /* No more tasks, just exit */ 2152391e43daSPeter Zijlstra goto out; 2153391e43daSPeter Zijlstra 2154391e43daSPeter Zijlstra /* 2155391e43daSPeter Zijlstra * Something has shifted, try again. 2156391e43daSPeter Zijlstra */ 2157391e43daSPeter Zijlstra put_task_struct(next_task); 2158391e43daSPeter Zijlstra next_task = task; 2159391e43daSPeter Zijlstra goto retry; 2160391e43daSPeter Zijlstra } 2161391e43daSPeter Zijlstra 2162391e43daSPeter Zijlstra deactivate_task(rq, next_task, 0); 2163391e43daSPeter Zijlstra set_task_cpu(next_task, lowest_rq->cpu); 2164391e43daSPeter Zijlstra activate_task(lowest_rq, next_task, 0); 2165a7c81556SPeter Zijlstra resched_curr(lowest_rq); 2166391e43daSPeter Zijlstra ret = 1; 2167391e43daSPeter Zijlstra 2168391e43daSPeter Zijlstra double_unlock_balance(rq, lowest_rq); 2169391e43daSPeter Zijlstra out: 2170391e43daSPeter Zijlstra put_task_struct(next_task); 2171391e43daSPeter Zijlstra 2172391e43daSPeter Zijlstra return ret; 2173391e43daSPeter Zijlstra } 2174391e43daSPeter Zijlstra 2175391e43daSPeter Zijlstra static void push_rt_tasks(struct rq *rq) 2176391e43daSPeter Zijlstra { 2177391e43daSPeter Zijlstra /* push_rt_task will return true if it moved an RT */ 2178a7c81556SPeter Zijlstra while (push_rt_task(rq, false)) 2179391e43daSPeter Zijlstra ; 2180391e43daSPeter Zijlstra } 2181391e43daSPeter Zijlstra 2182b6366f04SSteven Rostedt #ifdef HAVE_RT_PUSH_IPI 2183b6366f04SSteven Rostedt 21843e777f99SSteven Rostedt (VMware) /* 21853e777f99SSteven Rostedt (VMware) * When a high priority task schedules out from a CPU and a lower priority 21863e777f99SSteven Rostedt (VMware) * task is scheduled in, a check is made to see if there's any RT tasks 21873e777f99SSteven Rostedt (VMware) * on other CPUs that are waiting to run because a higher priority RT task 21883e777f99SSteven Rostedt (VMware) * is currently running on its CPU. In this case, the CPU with multiple RT 21893e777f99SSteven Rostedt (VMware) * tasks queued on it (overloaded) needs to be notified that a CPU has opened 21903e777f99SSteven Rostedt (VMware) * up that may be able to run one of its non-running queued RT tasks. 21913e777f99SSteven Rostedt (VMware) * 21924bdced5cSSteven Rostedt (Red Hat) * All CPUs with overloaded RT tasks need to be notified as there is currently 21934bdced5cSSteven Rostedt (Red Hat) * no way to know which of these CPUs have the highest priority task waiting 21944bdced5cSSteven Rostedt (Red Hat) * to run. Instead of trying to take a spinlock on each of these CPUs, 21954bdced5cSSteven Rostedt (Red Hat) * which has shown to cause large latency when done on machines with many 21964bdced5cSSteven Rostedt (Red Hat) * CPUs, sending an IPI to the CPUs to have them push off the overloaded 21974bdced5cSSteven Rostedt (Red Hat) * RT tasks waiting to run. 21983e777f99SSteven Rostedt (VMware) * 21994bdced5cSSteven Rostedt (Red Hat) * Just sending an IPI to each of the CPUs is also an issue, as on large 22004bdced5cSSteven Rostedt (Red Hat) * count CPU machines, this can cause an IPI storm on a CPU, especially 22014bdced5cSSteven Rostedt (Red Hat) * if its the only CPU with multiple RT tasks queued, and a large number 22024bdced5cSSteven Rostedt (Red Hat) * of CPUs scheduling a lower priority task at the same time. 22033e777f99SSteven Rostedt (VMware) * 22044bdced5cSSteven Rostedt (Red Hat) * Each root domain has its own irq work function that can iterate over 22054bdced5cSSteven Rostedt (Red Hat) * all CPUs with RT overloaded tasks. Since all CPUs with overloaded RT 22063b03706fSIngo Molnar * task must be checked if there's one or many CPUs that are lowering 22074bdced5cSSteven Rostedt (Red Hat) * their priority, there's a single irq work iterator that will try to 22084bdced5cSSteven Rostedt (Red Hat) * push off RT tasks that are waiting to run. 22093e777f99SSteven Rostedt (VMware) * 22104bdced5cSSteven Rostedt (Red Hat) * When a CPU schedules a lower priority task, it will kick off the 22114bdced5cSSteven Rostedt (Red Hat) * irq work iterator that will jump to each CPU with overloaded RT tasks. 22124bdced5cSSteven Rostedt (Red Hat) * As it only takes the first CPU that schedules a lower priority task 22134bdced5cSSteven Rostedt (Red Hat) * to start the process, the rto_start variable is incremented and if 22144bdced5cSSteven Rostedt (Red Hat) * the atomic result is one, then that CPU will try to take the rto_lock. 22154bdced5cSSteven Rostedt (Red Hat) * This prevents high contention on the lock as the process handles all 22164bdced5cSSteven Rostedt (Red Hat) * CPUs scheduling lower priority tasks. 22173e777f99SSteven Rostedt (VMware) * 22184bdced5cSSteven Rostedt (Red Hat) * All CPUs that are scheduling a lower priority task will increment the 22194bdced5cSSteven Rostedt (Red Hat) * rt_loop_next variable. This will make sure that the irq work iterator 22204bdced5cSSteven Rostedt (Red Hat) * checks all RT overloaded CPUs whenever a CPU schedules a new lower 22214bdced5cSSteven Rostedt (Red Hat) * priority task, even if the iterator is in the middle of a scan. Incrementing 22224bdced5cSSteven Rostedt (Red Hat) * the rt_loop_next will cause the iterator to perform another scan. 22233e777f99SSteven Rostedt (VMware) * 22243e777f99SSteven Rostedt (VMware) */ 2225ad0f1d9dSSteven Rostedt (VMware) static int rto_next_cpu(struct root_domain *rd) 2226b6366f04SSteven Rostedt { 22274bdced5cSSteven Rostedt (Red Hat) int next; 2228b6366f04SSteven Rostedt int cpu; 2229b6366f04SSteven Rostedt 2230b6366f04SSteven Rostedt /* 22314bdced5cSSteven Rostedt (Red Hat) * When starting the IPI RT pushing, the rto_cpu is set to -1, 22324bdced5cSSteven Rostedt (Red Hat) * rt_next_cpu() will simply return the first CPU found in 22334bdced5cSSteven Rostedt (Red Hat) * the rto_mask. 22344bdced5cSSteven Rostedt (Red Hat) * 223597fb7a0aSIngo Molnar * If rto_next_cpu() is called with rto_cpu is a valid CPU, it 22364bdced5cSSteven Rostedt (Red Hat) * will return the next CPU found in the rto_mask. 22374bdced5cSSteven Rostedt (Red Hat) * 22384bdced5cSSteven Rostedt (Red Hat) * If there are no more CPUs left in the rto_mask, then a check is made 22394bdced5cSSteven Rostedt (Red Hat) * against rto_loop and rto_loop_next. rto_loop is only updated with 22404bdced5cSSteven Rostedt (Red Hat) * the rto_lock held, but any CPU may increment the rto_loop_next 22414bdced5cSSteven Rostedt (Red Hat) * without any locking. 2242b6366f04SSteven Rostedt */ 22434bdced5cSSteven Rostedt (Red Hat) for (;;) { 22444bdced5cSSteven Rostedt (Red Hat) 22454bdced5cSSteven Rostedt (Red Hat) /* When rto_cpu is -1 this acts like cpumask_first() */ 22464bdced5cSSteven Rostedt (Red Hat) cpu = cpumask_next(rd->rto_cpu, rd->rto_mask); 22474bdced5cSSteven Rostedt (Red Hat) 22484bdced5cSSteven Rostedt (Red Hat) rd->rto_cpu = cpu; 22494bdced5cSSteven Rostedt (Red Hat) 22504bdced5cSSteven Rostedt (Red Hat) if (cpu < nr_cpu_ids) 22514bdced5cSSteven Rostedt (Red Hat) return cpu; 22524bdced5cSSteven Rostedt (Red Hat) 22534bdced5cSSteven Rostedt (Red Hat) rd->rto_cpu = -1; 22544bdced5cSSteven Rostedt (Red Hat) 22554bdced5cSSteven Rostedt (Red Hat) /* 22564bdced5cSSteven Rostedt (Red Hat) * ACQUIRE ensures we see the @rto_mask changes 22574bdced5cSSteven Rostedt (Red Hat) * made prior to the @next value observed. 22584bdced5cSSteven Rostedt (Red Hat) * 22594bdced5cSSteven Rostedt (Red Hat) * Matches WMB in rt_set_overload(). 22604bdced5cSSteven Rostedt (Red Hat) */ 22614bdced5cSSteven Rostedt (Red Hat) next = atomic_read_acquire(&rd->rto_loop_next); 22624bdced5cSSteven Rostedt (Red Hat) 22634bdced5cSSteven Rostedt (Red Hat) if (rd->rto_loop == next) 22644bdced5cSSteven Rostedt (Red Hat) break; 22654bdced5cSSteven Rostedt (Red Hat) 22664bdced5cSSteven Rostedt (Red Hat) rd->rto_loop = next; 2267b6366f04SSteven Rostedt } 2268b6366f04SSteven Rostedt 22694bdced5cSSteven Rostedt (Red Hat) return -1; 22704bdced5cSSteven Rostedt (Red Hat) } 2271b6366f04SSteven Rostedt 22724bdced5cSSteven Rostedt (Red Hat) static inline bool rto_start_trylock(atomic_t *v) 22734bdced5cSSteven Rostedt (Red Hat) { 22744bdced5cSSteven Rostedt (Red Hat) return !atomic_cmpxchg_acquire(v, 0, 1); 22754bdced5cSSteven Rostedt (Red Hat) } 22764bdced5cSSteven Rostedt (Red Hat) 22774bdced5cSSteven Rostedt (Red Hat) static inline void rto_start_unlock(atomic_t *v) 22784bdced5cSSteven Rostedt (Red Hat) { 22794bdced5cSSteven Rostedt (Red Hat) atomic_set_release(v, 0); 22804bdced5cSSteven Rostedt (Red Hat) } 22814bdced5cSSteven Rostedt (Red Hat) 22824bdced5cSSteven Rostedt (Red Hat) static void tell_cpu_to_push(struct rq *rq) 22834bdced5cSSteven Rostedt (Red Hat) { 22844bdced5cSSteven Rostedt (Red Hat) int cpu = -1; 22854bdced5cSSteven Rostedt (Red Hat) 22864bdced5cSSteven Rostedt (Red Hat) /* Keep the loop going if the IPI is currently active */ 22874bdced5cSSteven Rostedt (Red Hat) atomic_inc(&rq->rd->rto_loop_next); 22884bdced5cSSteven Rostedt (Red Hat) 22894bdced5cSSteven Rostedt (Red Hat) /* Only one CPU can initiate a loop at a time */ 22904bdced5cSSteven Rostedt (Red Hat) if (!rto_start_trylock(&rq->rd->rto_loop_start)) 2291b6366f04SSteven Rostedt return; 2292b6366f04SSteven Rostedt 22934bdced5cSSteven Rostedt (Red Hat) raw_spin_lock(&rq->rd->rto_lock); 2294b6366f04SSteven Rostedt 22954bdced5cSSteven Rostedt (Red Hat) /* 229697fb7a0aSIngo Molnar * The rto_cpu is updated under the lock, if it has a valid CPU 22974bdced5cSSteven Rostedt (Red Hat) * then the IPI is still running and will continue due to the 22984bdced5cSSteven Rostedt (Red Hat) * update to loop_next, and nothing needs to be done here. 22994bdced5cSSteven Rostedt (Red Hat) * Otherwise it is finishing up and an ipi needs to be sent. 23004bdced5cSSteven Rostedt (Red Hat) */ 23014bdced5cSSteven Rostedt (Red Hat) if (rq->rd->rto_cpu < 0) 2302ad0f1d9dSSteven Rostedt (VMware) cpu = rto_next_cpu(rq->rd); 23034bdced5cSSteven Rostedt (Red Hat) 23044bdced5cSSteven Rostedt (Red Hat) raw_spin_unlock(&rq->rd->rto_lock); 23054bdced5cSSteven Rostedt (Red Hat) 23064bdced5cSSteven Rostedt (Red Hat) rto_start_unlock(&rq->rd->rto_loop_start); 23074bdced5cSSteven Rostedt (Red Hat) 2308364f5665SSteven Rostedt (VMware) if (cpu >= 0) { 2309364f5665SSteven Rostedt (VMware) /* Make sure the rd does not get freed while pushing */ 2310364f5665SSteven Rostedt (VMware) sched_get_rd(rq->rd); 23114bdced5cSSteven Rostedt (Red Hat) irq_work_queue_on(&rq->rd->rto_push_work, cpu); 2312b6366f04SSteven Rostedt } 2313364f5665SSteven Rostedt (VMware) } 2314b6366f04SSteven Rostedt 2315b6366f04SSteven Rostedt /* Called from hardirq context */ 23164bdced5cSSteven Rostedt (Red Hat) void rto_push_irq_work_func(struct irq_work *work) 2317b6366f04SSteven Rostedt { 2318ad0f1d9dSSteven Rostedt (VMware) struct root_domain *rd = 2319ad0f1d9dSSteven Rostedt (VMware) container_of(work, struct root_domain, rto_push_work); 23204bdced5cSSteven Rostedt (Red Hat) struct rq *rq; 2321b6366f04SSteven Rostedt int cpu; 2322b6366f04SSteven Rostedt 23234bdced5cSSteven Rostedt (Red Hat) rq = this_rq(); 2324b6366f04SSteven Rostedt 23254bdced5cSSteven Rostedt (Red Hat) /* 23264bdced5cSSteven Rostedt (Red Hat) * We do not need to grab the lock to check for has_pushable_tasks. 23274bdced5cSSteven Rostedt (Red Hat) * When it gets updated, a check is made if a push is possible. 23284bdced5cSSteven Rostedt (Red Hat) */ 2329b6366f04SSteven Rostedt if (has_pushable_tasks(rq)) { 23305cb9eaa3SPeter Zijlstra raw_spin_rq_lock(rq); 2331a7c81556SPeter Zijlstra while (push_rt_task(rq, true)) 2332a7c81556SPeter Zijlstra ; 23335cb9eaa3SPeter Zijlstra raw_spin_rq_unlock(rq); 2334b6366f04SSteven Rostedt } 2335b6366f04SSteven Rostedt 2336ad0f1d9dSSteven Rostedt (VMware) raw_spin_lock(&rd->rto_lock); 23374bdced5cSSteven Rostedt (Red Hat) 2338b6366f04SSteven Rostedt /* Pass the IPI to the next rt overloaded queue */ 2339ad0f1d9dSSteven Rostedt (VMware) cpu = rto_next_cpu(rd); 2340b6366f04SSteven Rostedt 2341ad0f1d9dSSteven Rostedt (VMware) raw_spin_unlock(&rd->rto_lock); 2342b6366f04SSteven Rostedt 2343364f5665SSteven Rostedt (VMware) if (cpu < 0) { 2344364f5665SSteven Rostedt (VMware) sched_put_rd(rd); 2345b6366f04SSteven Rostedt return; 2346364f5665SSteven Rostedt (VMware) } 2347b6366f04SSteven Rostedt 2348b6366f04SSteven Rostedt /* Try the next RT overloaded CPU */ 2349ad0f1d9dSSteven Rostedt (VMware) irq_work_queue_on(&rd->rto_push_work, cpu); 2350b6366f04SSteven Rostedt } 2351b6366f04SSteven Rostedt #endif /* HAVE_RT_PUSH_IPI */ 2352b6366f04SSteven Rostedt 23538046d680SPeter Zijlstra static void pull_rt_task(struct rq *this_rq) 2354391e43daSPeter Zijlstra { 23558046d680SPeter Zijlstra int this_cpu = this_rq->cpu, cpu; 23568046d680SPeter Zijlstra bool resched = false; 2357a7c81556SPeter Zijlstra struct task_struct *p, *push_task; 2358391e43daSPeter Zijlstra struct rq *src_rq; 2359f73c52a5SSteven Rostedt int rt_overload_count = rt_overloaded(this_rq); 2360391e43daSPeter Zijlstra 2361f73c52a5SSteven Rostedt if (likely(!rt_overload_count)) 23628046d680SPeter Zijlstra return; 2363391e43daSPeter Zijlstra 23647c3f2ab7SPeter Zijlstra /* 23657c3f2ab7SPeter Zijlstra * Match the barrier from rt_set_overloaded; this guarantees that if we 23667c3f2ab7SPeter Zijlstra * see overloaded we must also see the rto_mask bit. 23677c3f2ab7SPeter Zijlstra */ 23687c3f2ab7SPeter Zijlstra smp_rmb(); 23697c3f2ab7SPeter Zijlstra 2370f73c52a5SSteven Rostedt /* If we are the only overloaded CPU do nothing */ 2371f73c52a5SSteven Rostedt if (rt_overload_count == 1 && 2372f73c52a5SSteven Rostedt cpumask_test_cpu(this_rq->cpu, this_rq->rd->rto_mask)) 2373f73c52a5SSteven Rostedt return; 2374f73c52a5SSteven Rostedt 2375b6366f04SSteven Rostedt #ifdef HAVE_RT_PUSH_IPI 2376b6366f04SSteven Rostedt if (sched_feat(RT_PUSH_IPI)) { 2377b6366f04SSteven Rostedt tell_cpu_to_push(this_rq); 23788046d680SPeter Zijlstra return; 2379b6366f04SSteven Rostedt } 2380b6366f04SSteven Rostedt #endif 2381b6366f04SSteven Rostedt 2382391e43daSPeter Zijlstra for_each_cpu(cpu, this_rq->rd->rto_mask) { 2383391e43daSPeter Zijlstra if (this_cpu == cpu) 2384391e43daSPeter Zijlstra continue; 2385391e43daSPeter Zijlstra 2386391e43daSPeter Zijlstra src_rq = cpu_rq(cpu); 2387391e43daSPeter Zijlstra 2388391e43daSPeter Zijlstra /* 2389391e43daSPeter Zijlstra * Don't bother taking the src_rq->lock if the next highest 2390391e43daSPeter Zijlstra * task is known to be lower-priority than our current task. 2391391e43daSPeter Zijlstra * This may look racy, but if this value is about to go 2392391e43daSPeter Zijlstra * logically higher, the src_rq will push this task away. 2393391e43daSPeter Zijlstra * And if its going logically lower, we do not care 2394391e43daSPeter Zijlstra */ 2395391e43daSPeter Zijlstra if (src_rq->rt.highest_prio.next >= 2396391e43daSPeter Zijlstra this_rq->rt.highest_prio.curr) 2397391e43daSPeter Zijlstra continue; 2398391e43daSPeter Zijlstra 2399391e43daSPeter Zijlstra /* 2400391e43daSPeter Zijlstra * We can potentially drop this_rq's lock in 2401391e43daSPeter Zijlstra * double_lock_balance, and another CPU could 2402391e43daSPeter Zijlstra * alter this_rq 2403391e43daSPeter Zijlstra */ 2404a7c81556SPeter Zijlstra push_task = NULL; 2405391e43daSPeter Zijlstra double_lock_balance(this_rq, src_rq); 2406391e43daSPeter Zijlstra 2407391e43daSPeter Zijlstra /* 2408e23ee747SKirill Tkhai * We can pull only a task, which is pushable 2409e23ee747SKirill Tkhai * on its rq, and no others. 2410391e43daSPeter Zijlstra */ 2411e23ee747SKirill Tkhai p = pick_highest_pushable_task(src_rq, this_cpu); 2412391e43daSPeter Zijlstra 2413391e43daSPeter Zijlstra /* 2414391e43daSPeter Zijlstra * Do we have an RT task that preempts 2415391e43daSPeter Zijlstra * the to-be-scheduled task? 2416391e43daSPeter Zijlstra */ 2417391e43daSPeter Zijlstra if (p && (p->prio < this_rq->rt.highest_prio.curr)) { 2418391e43daSPeter Zijlstra WARN_ON(p == src_rq->curr); 2419da0c1e65SKirill Tkhai WARN_ON(!task_on_rq_queued(p)); 2420391e43daSPeter Zijlstra 2421391e43daSPeter Zijlstra /* 2422391e43daSPeter Zijlstra * There's a chance that p is higher in priority 242397fb7a0aSIngo Molnar * than what's currently running on its CPU. 24243b03706fSIngo Molnar * This is just that p is waking up and hasn't 2425391e43daSPeter Zijlstra * had a chance to schedule. We only pull 2426391e43daSPeter Zijlstra * p if it is lower in priority than the 2427391e43daSPeter Zijlstra * current task on the run queue 2428391e43daSPeter Zijlstra */ 2429391e43daSPeter Zijlstra if (p->prio < src_rq->curr->prio) 2430391e43daSPeter Zijlstra goto skip; 2431391e43daSPeter Zijlstra 2432a7c81556SPeter Zijlstra if (is_migration_disabled(p)) { 2433a7c81556SPeter Zijlstra push_task = get_push_task(src_rq); 2434a7c81556SPeter Zijlstra } else { 2435391e43daSPeter Zijlstra deactivate_task(src_rq, p, 0); 2436391e43daSPeter Zijlstra set_task_cpu(p, this_cpu); 2437391e43daSPeter Zijlstra activate_task(this_rq, p, 0); 2438a7c81556SPeter Zijlstra resched = true; 2439a7c81556SPeter Zijlstra } 2440391e43daSPeter Zijlstra /* 2441391e43daSPeter Zijlstra * We continue with the search, just in 2442391e43daSPeter Zijlstra * case there's an even higher prio task 2443391e43daSPeter Zijlstra * in another runqueue. (low likelihood 2444391e43daSPeter Zijlstra * but possible) 2445391e43daSPeter Zijlstra */ 2446391e43daSPeter Zijlstra } 2447391e43daSPeter Zijlstra skip: 2448391e43daSPeter Zijlstra double_unlock_balance(this_rq, src_rq); 2449a7c81556SPeter Zijlstra 2450a7c81556SPeter Zijlstra if (push_task) { 24515cb9eaa3SPeter Zijlstra raw_spin_rq_unlock(this_rq); 2452a7c81556SPeter Zijlstra stop_one_cpu_nowait(src_rq->cpu, push_cpu_stop, 2453a7c81556SPeter Zijlstra push_task, &src_rq->push_work); 24545cb9eaa3SPeter Zijlstra raw_spin_rq_lock(this_rq); 2455a7c81556SPeter Zijlstra } 2456391e43daSPeter Zijlstra } 2457391e43daSPeter Zijlstra 24588046d680SPeter Zijlstra if (resched) 24598046d680SPeter Zijlstra resched_curr(this_rq); 2460391e43daSPeter Zijlstra } 2461391e43daSPeter Zijlstra 2462391e43daSPeter Zijlstra /* 2463391e43daSPeter Zijlstra * If we are not running and we are not going to reschedule soon, we should 2464391e43daSPeter Zijlstra * try to push tasks away now 2465391e43daSPeter Zijlstra */ 2466391e43daSPeter Zijlstra static void task_woken_rt(struct rq *rq, struct task_struct *p) 2467391e43daSPeter Zijlstra { 24680b9d46fcSPeter Zijlstra bool need_to_push = !task_on_cpu(rq, p) && 2469391e43daSPeter Zijlstra !test_tsk_need_resched(rq->curr) && 24704b53a341SIngo Molnar p->nr_cpus_allowed > 1 && 24711baca4ceSJuri Lelli (dl_task(rq->curr) || rt_task(rq->curr)) && 24724b53a341SIngo Molnar (rq->curr->nr_cpus_allowed < 2 || 2473804d402fSQais Yousef rq->curr->prio <= p->prio); 2474804d402fSQais Yousef 2475d94a9df4SQais Yousef if (need_to_push) 2476391e43daSPeter Zijlstra push_rt_tasks(rq); 2477391e43daSPeter Zijlstra } 2478391e43daSPeter Zijlstra 2479391e43daSPeter Zijlstra /* Assumes rq->lock is held */ 2480391e43daSPeter Zijlstra static void rq_online_rt(struct rq *rq) 2481391e43daSPeter Zijlstra { 2482391e43daSPeter Zijlstra if (rq->rt.overloaded) 2483391e43daSPeter Zijlstra rt_set_overload(rq); 2484391e43daSPeter Zijlstra 2485391e43daSPeter Zijlstra __enable_runtime(rq); 2486391e43daSPeter Zijlstra 2487391e43daSPeter Zijlstra cpupri_set(&rq->rd->cpupri, rq->cpu, rq->rt.highest_prio.curr); 2488391e43daSPeter Zijlstra } 2489391e43daSPeter Zijlstra 2490391e43daSPeter Zijlstra /* Assumes rq->lock is held */ 2491391e43daSPeter Zijlstra static void rq_offline_rt(struct rq *rq) 2492391e43daSPeter Zijlstra { 2493391e43daSPeter Zijlstra if (rq->rt.overloaded) 2494391e43daSPeter Zijlstra rt_clear_overload(rq); 2495391e43daSPeter Zijlstra 2496391e43daSPeter Zijlstra __disable_runtime(rq); 2497391e43daSPeter Zijlstra 2498391e43daSPeter Zijlstra cpupri_set(&rq->rd->cpupri, rq->cpu, CPUPRI_INVALID); 2499391e43daSPeter Zijlstra } 2500391e43daSPeter Zijlstra 2501391e43daSPeter Zijlstra /* 2502391e43daSPeter Zijlstra * When switch from the rt queue, we bring ourselves to a position 2503391e43daSPeter Zijlstra * that we might want to pull RT tasks from other runqueues. 2504391e43daSPeter Zijlstra */ 2505391e43daSPeter Zijlstra static void switched_from_rt(struct rq *rq, struct task_struct *p) 2506391e43daSPeter Zijlstra { 2507391e43daSPeter Zijlstra /* 2508391e43daSPeter Zijlstra * If there are other RT tasks then we will reschedule 2509391e43daSPeter Zijlstra * and the scheduling of the other RT tasks will handle 2510391e43daSPeter Zijlstra * the balancing. But if we are the last RT task 2511391e43daSPeter Zijlstra * we may need to handle the pulling of RT tasks 2512391e43daSPeter Zijlstra * now. 2513391e43daSPeter Zijlstra */ 2514da0c1e65SKirill Tkhai if (!task_on_rq_queued(p) || rq->rt.rt_nr_running) 25151158ddb5SKirill Tkhai return; 25161158ddb5SKirill Tkhai 251702d8ec94SIngo Molnar rt_queue_pull_task(rq); 2518391e43daSPeter Zijlstra } 2519391e43daSPeter Zijlstra 252011c785b7SLi Zefan void __init init_sched_rt_class(void) 2521391e43daSPeter Zijlstra { 2522391e43daSPeter Zijlstra unsigned int i; 2523391e43daSPeter Zijlstra 2524391e43daSPeter Zijlstra for_each_possible_cpu(i) { 2525391e43daSPeter Zijlstra zalloc_cpumask_var_node(&per_cpu(local_cpu_mask, i), 2526391e43daSPeter Zijlstra GFP_KERNEL, cpu_to_node(i)); 2527391e43daSPeter Zijlstra } 2528391e43daSPeter Zijlstra } 2529391e43daSPeter Zijlstra #endif /* CONFIG_SMP */ 2530391e43daSPeter Zijlstra 2531391e43daSPeter Zijlstra /* 2532391e43daSPeter Zijlstra * When switching a task to RT, we may overload the runqueue 2533391e43daSPeter Zijlstra * with RT tasks. In this case we try to push them off to 2534391e43daSPeter Zijlstra * other runqueues. 2535391e43daSPeter Zijlstra */ 2536391e43daSPeter Zijlstra static void switched_to_rt(struct rq *rq, struct task_struct *p) 2537391e43daSPeter Zijlstra { 2538391e43daSPeter Zijlstra /* 2539fecfcbc2SVincent Donnefort * If we are running, update the avg_rt tracking, as the running time 2540fecfcbc2SVincent Donnefort * will now on be accounted into the latter. 2541fecfcbc2SVincent Donnefort */ 2542fecfcbc2SVincent Donnefort if (task_current(rq, p)) { 2543fecfcbc2SVincent Donnefort update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 0); 2544fecfcbc2SVincent Donnefort return; 2545fecfcbc2SVincent Donnefort } 2546fecfcbc2SVincent Donnefort 2547fecfcbc2SVincent Donnefort /* 2548fecfcbc2SVincent Donnefort * If we are not running we may need to preempt the current 2549fecfcbc2SVincent Donnefort * running task. If that current running task is also an RT task 2550391e43daSPeter Zijlstra * then see if we can move to another run queue. 2551391e43daSPeter Zijlstra */ 2552fecfcbc2SVincent Donnefort if (task_on_rq_queued(p)) { 2553391e43daSPeter Zijlstra #ifdef CONFIG_SMP 2554d94a9df4SQais Yousef if (p->nr_cpus_allowed > 1 && rq->rt.overloaded) 255502d8ec94SIngo Molnar rt_queue_push_tasks(rq); 2556619bd4a7SSebastian Andrzej Siewior #endif /* CONFIG_SMP */ 25572fe25826SPaul E. McKenney if (p->prio < rq->curr->prio && cpu_online(cpu_of(rq))) 25588875125eSKirill Tkhai resched_curr(rq); 2559391e43daSPeter Zijlstra } 2560391e43daSPeter Zijlstra } 2561391e43daSPeter Zijlstra 2562391e43daSPeter Zijlstra /* 2563391e43daSPeter Zijlstra * Priority of the task has changed. This may cause 2564391e43daSPeter Zijlstra * us to initiate a push or pull. 2565391e43daSPeter Zijlstra */ 2566391e43daSPeter Zijlstra static void 2567391e43daSPeter Zijlstra prio_changed_rt(struct rq *rq, struct task_struct *p, int oldprio) 2568391e43daSPeter Zijlstra { 2569da0c1e65SKirill Tkhai if (!task_on_rq_queued(p)) 2570391e43daSPeter Zijlstra return; 2571391e43daSPeter Zijlstra 257265bcf072SHui Su if (task_current(rq, p)) { 2573391e43daSPeter Zijlstra #ifdef CONFIG_SMP 2574391e43daSPeter Zijlstra /* 2575391e43daSPeter Zijlstra * If our priority decreases while running, we 2576391e43daSPeter Zijlstra * may need to pull tasks to this runqueue. 2577391e43daSPeter Zijlstra */ 2578391e43daSPeter Zijlstra if (oldprio < p->prio) 257902d8ec94SIngo Molnar rt_queue_pull_task(rq); 2580fd7a4bedSPeter Zijlstra 2581391e43daSPeter Zijlstra /* 2582391e43daSPeter Zijlstra * If there's a higher priority task waiting to run 2583fd7a4bedSPeter Zijlstra * then reschedule. 2584391e43daSPeter Zijlstra */ 2585fd7a4bedSPeter Zijlstra if (p->prio > rq->rt.highest_prio.curr) 25868875125eSKirill Tkhai resched_curr(rq); 2587391e43daSPeter Zijlstra #else 2588391e43daSPeter Zijlstra /* For UP simply resched on drop of prio */ 2589391e43daSPeter Zijlstra if (oldprio < p->prio) 25908875125eSKirill Tkhai resched_curr(rq); 2591391e43daSPeter Zijlstra #endif /* CONFIG_SMP */ 2592391e43daSPeter Zijlstra } else { 2593391e43daSPeter Zijlstra /* 2594391e43daSPeter Zijlstra * This task is not running, but if it is 2595391e43daSPeter Zijlstra * greater than the current running task 2596391e43daSPeter Zijlstra * then reschedule. 2597391e43daSPeter Zijlstra */ 2598391e43daSPeter Zijlstra if (p->prio < rq->curr->prio) 25998875125eSKirill Tkhai resched_curr(rq); 2600391e43daSPeter Zijlstra } 2601391e43daSPeter Zijlstra } 2602391e43daSPeter Zijlstra 2603b18b6a9cSNicolas Pitre #ifdef CONFIG_POSIX_TIMERS 2604391e43daSPeter Zijlstra static void watchdog(struct rq *rq, struct task_struct *p) 2605391e43daSPeter Zijlstra { 2606391e43daSPeter Zijlstra unsigned long soft, hard; 2607391e43daSPeter Zijlstra 2608391e43daSPeter Zijlstra /* max may change after cur was read, this will be fixed next tick */ 2609391e43daSPeter Zijlstra soft = task_rlimit(p, RLIMIT_RTTIME); 2610391e43daSPeter Zijlstra hard = task_rlimit_max(p, RLIMIT_RTTIME); 2611391e43daSPeter Zijlstra 2612391e43daSPeter Zijlstra if (soft != RLIM_INFINITY) { 2613391e43daSPeter Zijlstra unsigned long next; 2614391e43daSPeter Zijlstra 261557d2aa00SYing Xue if (p->rt.watchdog_stamp != jiffies) { 2616391e43daSPeter Zijlstra p->rt.timeout++; 261757d2aa00SYing Xue p->rt.watchdog_stamp = jiffies; 261857d2aa00SYing Xue } 261957d2aa00SYing Xue 2620391e43daSPeter Zijlstra next = DIV_ROUND_UP(min(soft, hard), USEC_PER_SEC/HZ); 26213a245c0fSThomas Gleixner if (p->rt.timeout > next) { 26223a245c0fSThomas Gleixner posix_cputimers_rt_watchdog(&p->posix_cputimers, 26233a245c0fSThomas Gleixner p->se.sum_exec_runtime); 26243a245c0fSThomas Gleixner } 2625391e43daSPeter Zijlstra } 2626391e43daSPeter Zijlstra } 2627b18b6a9cSNicolas Pitre #else 2628b18b6a9cSNicolas Pitre static inline void watchdog(struct rq *rq, struct task_struct *p) { } 2629b18b6a9cSNicolas Pitre #endif 2630391e43daSPeter Zijlstra 2631d84b3131SFrederic Weisbecker /* 2632d84b3131SFrederic Weisbecker * scheduler tick hitting a task of our scheduling class. 2633d84b3131SFrederic Weisbecker * 2634d84b3131SFrederic Weisbecker * NOTE: This function can be called remotely by the tick offload that 2635d84b3131SFrederic Weisbecker * goes along full dynticks. Therefore no local assumption can be made 2636d84b3131SFrederic Weisbecker * and everything must be accessed through the @rq and @curr passed in 2637d84b3131SFrederic Weisbecker * parameters. 2638d84b3131SFrederic Weisbecker */ 2639391e43daSPeter Zijlstra static void task_tick_rt(struct rq *rq, struct task_struct *p, int queued) 2640391e43daSPeter Zijlstra { 2641454c7999SColin Cross struct sched_rt_entity *rt_se = &p->rt; 2642454c7999SColin Cross 2643391e43daSPeter Zijlstra update_curr_rt(rq); 264423127296SVincent Guittot update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 1); 2645391e43daSPeter Zijlstra 2646391e43daSPeter Zijlstra watchdog(rq, p); 2647391e43daSPeter Zijlstra 2648391e43daSPeter Zijlstra /* 2649391e43daSPeter Zijlstra * RR tasks need a special form of timeslice management. 2650391e43daSPeter Zijlstra * FIFO tasks have no timeslices. 2651391e43daSPeter Zijlstra */ 2652391e43daSPeter Zijlstra if (p->policy != SCHED_RR) 2653391e43daSPeter Zijlstra return; 2654391e43daSPeter Zijlstra 2655391e43daSPeter Zijlstra if (--p->rt.time_slice) 2656391e43daSPeter Zijlstra return; 2657391e43daSPeter Zijlstra 2658ce0dbbbbSClark Williams p->rt.time_slice = sched_rr_timeslice; 2659391e43daSPeter Zijlstra 2660391e43daSPeter Zijlstra /* 2661e9aa39bbSLi Bin * Requeue to the end of queue if we (and all of our ancestors) are not 2662e9aa39bbSLi Bin * the only element on the queue 2663391e43daSPeter Zijlstra */ 2664454c7999SColin Cross for_each_sched_rt_entity(rt_se) { 2665454c7999SColin Cross if (rt_se->run_list.prev != rt_se->run_list.next) { 2666391e43daSPeter Zijlstra requeue_task_rt(rq, p, 0); 26678aa6f0ebSKirill Tkhai resched_curr(rq); 2668454c7999SColin Cross return; 2669454c7999SColin Cross } 2670391e43daSPeter Zijlstra } 2671391e43daSPeter Zijlstra } 2672391e43daSPeter Zijlstra 2673391e43daSPeter Zijlstra static unsigned int get_rr_interval_rt(struct rq *rq, struct task_struct *task) 2674391e43daSPeter Zijlstra { 2675391e43daSPeter Zijlstra /* 2676391e43daSPeter Zijlstra * Time slice is 0 for SCHED_FIFO tasks 2677391e43daSPeter Zijlstra */ 2678391e43daSPeter Zijlstra if (task->policy == SCHED_RR) 2679ce0dbbbbSClark Williams return sched_rr_timeslice; 2680391e43daSPeter Zijlstra else 2681391e43daSPeter Zijlstra return 0; 2682391e43daSPeter Zijlstra } 2683391e43daSPeter Zijlstra 2684530bfad1SHao Jia #ifdef CONFIG_SCHED_CORE 2685530bfad1SHao Jia static int task_is_throttled_rt(struct task_struct *p, int cpu) 2686530bfad1SHao Jia { 2687530bfad1SHao Jia struct rt_rq *rt_rq; 2688530bfad1SHao Jia 2689530bfad1SHao Jia #ifdef CONFIG_RT_GROUP_SCHED 2690530bfad1SHao Jia rt_rq = task_group(p)->rt_rq[cpu]; 2691530bfad1SHao Jia #else 2692530bfad1SHao Jia rt_rq = &cpu_rq(cpu)->rt; 2693530bfad1SHao Jia #endif 2694530bfad1SHao Jia 2695530bfad1SHao Jia return rt_rq_throttled(rt_rq); 2696530bfad1SHao Jia } 2697530bfad1SHao Jia #endif 2698530bfad1SHao Jia 269943c31ac0SPeter Zijlstra DEFINE_SCHED_CLASS(rt) = { 270043c31ac0SPeter Zijlstra 2701391e43daSPeter Zijlstra .enqueue_task = enqueue_task_rt, 2702391e43daSPeter Zijlstra .dequeue_task = dequeue_task_rt, 2703391e43daSPeter Zijlstra .yield_task = yield_task_rt, 2704391e43daSPeter Zijlstra 2705391e43daSPeter Zijlstra .check_preempt_curr = check_preempt_curr_rt, 2706391e43daSPeter Zijlstra 2707391e43daSPeter Zijlstra .pick_next_task = pick_next_task_rt, 2708391e43daSPeter Zijlstra .put_prev_task = put_prev_task_rt, 270903b7fad1SPeter Zijlstra .set_next_task = set_next_task_rt, 2710391e43daSPeter Zijlstra 2711391e43daSPeter Zijlstra #ifdef CONFIG_SMP 27126e2df058SPeter Zijlstra .balance = balance_rt, 271321f56ffeSPeter Zijlstra .pick_task = pick_task_rt, 2714391e43daSPeter Zijlstra .select_task_rq = select_task_rq_rt, 27156c37067eSPeter Zijlstra .set_cpus_allowed = set_cpus_allowed_common, 2716391e43daSPeter Zijlstra .rq_online = rq_online_rt, 2717391e43daSPeter Zijlstra .rq_offline = rq_offline_rt, 2718391e43daSPeter Zijlstra .task_woken = task_woken_rt, 2719391e43daSPeter Zijlstra .switched_from = switched_from_rt, 2720a7c81556SPeter Zijlstra .find_lock_rq = find_lock_lowest_rq, 2721391e43daSPeter Zijlstra #endif 2722391e43daSPeter Zijlstra 2723391e43daSPeter Zijlstra .task_tick = task_tick_rt, 2724391e43daSPeter Zijlstra 2725391e43daSPeter Zijlstra .get_rr_interval = get_rr_interval_rt, 2726391e43daSPeter Zijlstra 2727391e43daSPeter Zijlstra .prio_changed = prio_changed_rt, 2728391e43daSPeter Zijlstra .switched_to = switched_to_rt, 27296e998916SStanislaw Gruszka 27306e998916SStanislaw Gruszka .update_curr = update_curr_rt, 2731982d9cdcSPatrick Bellasi 2732530bfad1SHao Jia #ifdef CONFIG_SCHED_CORE 2733530bfad1SHao Jia .task_is_throttled = task_is_throttled_rt, 2734530bfad1SHao Jia #endif 2735530bfad1SHao Jia 2736982d9cdcSPatrick Bellasi #ifdef CONFIG_UCLAMP_TASK 2737982d9cdcSPatrick Bellasi .uclamp_enabled = 1, 2738982d9cdcSPatrick Bellasi #endif 2739391e43daSPeter Zijlstra }; 2740391e43daSPeter Zijlstra 27418887cd99SNicolas Pitre #ifdef CONFIG_RT_GROUP_SCHED 27428887cd99SNicolas Pitre /* 27438887cd99SNicolas Pitre * Ensure that the real time constraints are schedulable. 27448887cd99SNicolas Pitre */ 27458887cd99SNicolas Pitre static DEFINE_MUTEX(rt_constraints_mutex); 27468887cd99SNicolas Pitre 27478887cd99SNicolas Pitre static inline int tg_has_rt_tasks(struct task_group *tg) 27488887cd99SNicolas Pitre { 2749b4fb015eSKonstantin Khlebnikov struct task_struct *task; 2750b4fb015eSKonstantin Khlebnikov struct css_task_iter it; 2751b4fb015eSKonstantin Khlebnikov int ret = 0; 27528887cd99SNicolas Pitre 27538887cd99SNicolas Pitre /* 27548887cd99SNicolas Pitre * Autogroups do not have RT tasks; see autogroup_create(). 27558887cd99SNicolas Pitre */ 27568887cd99SNicolas Pitre if (task_group_is_autogroup(tg)) 27578887cd99SNicolas Pitre return 0; 27588887cd99SNicolas Pitre 2759b4fb015eSKonstantin Khlebnikov css_task_iter_start(&tg->css, 0, &it); 2760b4fb015eSKonstantin Khlebnikov while (!ret && (task = css_task_iter_next(&it))) 2761b4fb015eSKonstantin Khlebnikov ret |= rt_task(task); 2762b4fb015eSKonstantin Khlebnikov css_task_iter_end(&it); 27638887cd99SNicolas Pitre 2764b4fb015eSKonstantin Khlebnikov return ret; 27658887cd99SNicolas Pitre } 27668887cd99SNicolas Pitre 27678887cd99SNicolas Pitre struct rt_schedulable_data { 27688887cd99SNicolas Pitre struct task_group *tg; 27698887cd99SNicolas Pitre u64 rt_period; 27708887cd99SNicolas Pitre u64 rt_runtime; 27718887cd99SNicolas Pitre }; 27728887cd99SNicolas Pitre 27738887cd99SNicolas Pitre static int tg_rt_schedulable(struct task_group *tg, void *data) 27748887cd99SNicolas Pitre { 27758887cd99SNicolas Pitre struct rt_schedulable_data *d = data; 27768887cd99SNicolas Pitre struct task_group *child; 27778887cd99SNicolas Pitre unsigned long total, sum = 0; 27788887cd99SNicolas Pitre u64 period, runtime; 27798887cd99SNicolas Pitre 27808887cd99SNicolas Pitre period = ktime_to_ns(tg->rt_bandwidth.rt_period); 27818887cd99SNicolas Pitre runtime = tg->rt_bandwidth.rt_runtime; 27828887cd99SNicolas Pitre 27838887cd99SNicolas Pitre if (tg == d->tg) { 27848887cd99SNicolas Pitre period = d->rt_period; 27858887cd99SNicolas Pitre runtime = d->rt_runtime; 27868887cd99SNicolas Pitre } 27878887cd99SNicolas Pitre 27888887cd99SNicolas Pitre /* 27898887cd99SNicolas Pitre * Cannot have more runtime than the period. 27908887cd99SNicolas Pitre */ 27918887cd99SNicolas Pitre if (runtime > period && runtime != RUNTIME_INF) 27928887cd99SNicolas Pitre return -EINVAL; 27938887cd99SNicolas Pitre 27948887cd99SNicolas Pitre /* 2795b4fb015eSKonstantin Khlebnikov * Ensure we don't starve existing RT tasks if runtime turns zero. 27968887cd99SNicolas Pitre */ 2797b4fb015eSKonstantin Khlebnikov if (rt_bandwidth_enabled() && !runtime && 2798b4fb015eSKonstantin Khlebnikov tg->rt_bandwidth.rt_runtime && tg_has_rt_tasks(tg)) 27998887cd99SNicolas Pitre return -EBUSY; 28008887cd99SNicolas Pitre 28018887cd99SNicolas Pitre total = to_ratio(period, runtime); 28028887cd99SNicolas Pitre 28038887cd99SNicolas Pitre /* 28048887cd99SNicolas Pitre * Nobody can have more than the global setting allows. 28058887cd99SNicolas Pitre */ 28068887cd99SNicolas Pitre if (total > to_ratio(global_rt_period(), global_rt_runtime())) 28078887cd99SNicolas Pitre return -EINVAL; 28088887cd99SNicolas Pitre 28098887cd99SNicolas Pitre /* 28108887cd99SNicolas Pitre * The sum of our children's runtime should not exceed our own. 28118887cd99SNicolas Pitre */ 28128887cd99SNicolas Pitre list_for_each_entry_rcu(child, &tg->children, siblings) { 28138887cd99SNicolas Pitre period = ktime_to_ns(child->rt_bandwidth.rt_period); 28148887cd99SNicolas Pitre runtime = child->rt_bandwidth.rt_runtime; 28158887cd99SNicolas Pitre 28168887cd99SNicolas Pitre if (child == d->tg) { 28178887cd99SNicolas Pitre period = d->rt_period; 28188887cd99SNicolas Pitre runtime = d->rt_runtime; 28198887cd99SNicolas Pitre } 28208887cd99SNicolas Pitre 28218887cd99SNicolas Pitre sum += to_ratio(period, runtime); 28228887cd99SNicolas Pitre } 28238887cd99SNicolas Pitre 28248887cd99SNicolas Pitre if (sum > total) 28258887cd99SNicolas Pitre return -EINVAL; 28268887cd99SNicolas Pitre 28278887cd99SNicolas Pitre return 0; 28288887cd99SNicolas Pitre } 28298887cd99SNicolas Pitre 28308887cd99SNicolas Pitre static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime) 28318887cd99SNicolas Pitre { 28328887cd99SNicolas Pitre int ret; 28338887cd99SNicolas Pitre 28348887cd99SNicolas Pitre struct rt_schedulable_data data = { 28358887cd99SNicolas Pitre .tg = tg, 28368887cd99SNicolas Pitre .rt_period = period, 28378887cd99SNicolas Pitre .rt_runtime = runtime, 28388887cd99SNicolas Pitre }; 28398887cd99SNicolas Pitre 28408887cd99SNicolas Pitre rcu_read_lock(); 28418887cd99SNicolas Pitre ret = walk_tg_tree(tg_rt_schedulable, tg_nop, &data); 28428887cd99SNicolas Pitre rcu_read_unlock(); 28438887cd99SNicolas Pitre 28448887cd99SNicolas Pitre return ret; 28458887cd99SNicolas Pitre } 28468887cd99SNicolas Pitre 28478887cd99SNicolas Pitre static int tg_set_rt_bandwidth(struct task_group *tg, 28488887cd99SNicolas Pitre u64 rt_period, u64 rt_runtime) 28498887cd99SNicolas Pitre { 28508887cd99SNicolas Pitre int i, err = 0; 28518887cd99SNicolas Pitre 28528887cd99SNicolas Pitre /* 28538887cd99SNicolas Pitre * Disallowing the root group RT runtime is BAD, it would disallow the 28548887cd99SNicolas Pitre * kernel creating (and or operating) RT threads. 28558887cd99SNicolas Pitre */ 28568887cd99SNicolas Pitre if (tg == &root_task_group && rt_runtime == 0) 28578887cd99SNicolas Pitre return -EINVAL; 28588887cd99SNicolas Pitre 28598887cd99SNicolas Pitre /* No period doesn't make any sense. */ 28608887cd99SNicolas Pitre if (rt_period == 0) 28618887cd99SNicolas Pitre return -EINVAL; 28628887cd99SNicolas Pitre 2863d505b8afSHuaixin Chang /* 2864d505b8afSHuaixin Chang * Bound quota to defend quota against overflow during bandwidth shift. 2865d505b8afSHuaixin Chang */ 2866d505b8afSHuaixin Chang if (rt_runtime != RUNTIME_INF && rt_runtime > max_rt_runtime) 2867d505b8afSHuaixin Chang return -EINVAL; 2868d505b8afSHuaixin Chang 28698887cd99SNicolas Pitre mutex_lock(&rt_constraints_mutex); 28708887cd99SNicolas Pitre err = __rt_schedulable(tg, rt_period, rt_runtime); 28718887cd99SNicolas Pitre if (err) 28728887cd99SNicolas Pitre goto unlock; 28738887cd99SNicolas Pitre 28748887cd99SNicolas Pitre raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock); 28758887cd99SNicolas Pitre tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period); 28768887cd99SNicolas Pitre tg->rt_bandwidth.rt_runtime = rt_runtime; 28778887cd99SNicolas Pitre 28788887cd99SNicolas Pitre for_each_possible_cpu(i) { 28798887cd99SNicolas Pitre struct rt_rq *rt_rq = tg->rt_rq[i]; 28808887cd99SNicolas Pitre 28818887cd99SNicolas Pitre raw_spin_lock(&rt_rq->rt_runtime_lock); 28828887cd99SNicolas Pitre rt_rq->rt_runtime = rt_runtime; 28838887cd99SNicolas Pitre raw_spin_unlock(&rt_rq->rt_runtime_lock); 28848887cd99SNicolas Pitre } 28858887cd99SNicolas Pitre raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock); 28868887cd99SNicolas Pitre unlock: 28878887cd99SNicolas Pitre mutex_unlock(&rt_constraints_mutex); 28888887cd99SNicolas Pitre 28898887cd99SNicolas Pitre return err; 28908887cd99SNicolas Pitre } 28918887cd99SNicolas Pitre 28928887cd99SNicolas Pitre int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us) 28938887cd99SNicolas Pitre { 28948887cd99SNicolas Pitre u64 rt_runtime, rt_period; 28958887cd99SNicolas Pitre 28968887cd99SNicolas Pitre rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period); 28978887cd99SNicolas Pitre rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC; 28988887cd99SNicolas Pitre if (rt_runtime_us < 0) 28998887cd99SNicolas Pitre rt_runtime = RUNTIME_INF; 29001a010e29SKonstantin Khlebnikov else if ((u64)rt_runtime_us > U64_MAX / NSEC_PER_USEC) 29011a010e29SKonstantin Khlebnikov return -EINVAL; 29028887cd99SNicolas Pitre 29038887cd99SNicolas Pitre return tg_set_rt_bandwidth(tg, rt_period, rt_runtime); 29048887cd99SNicolas Pitre } 29058887cd99SNicolas Pitre 29068887cd99SNicolas Pitre long sched_group_rt_runtime(struct task_group *tg) 29078887cd99SNicolas Pitre { 29088887cd99SNicolas Pitre u64 rt_runtime_us; 29098887cd99SNicolas Pitre 29108887cd99SNicolas Pitre if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF) 29118887cd99SNicolas Pitre return -1; 29128887cd99SNicolas Pitre 29138887cd99SNicolas Pitre rt_runtime_us = tg->rt_bandwidth.rt_runtime; 29148887cd99SNicolas Pitre do_div(rt_runtime_us, NSEC_PER_USEC); 29158887cd99SNicolas Pitre return rt_runtime_us; 29168887cd99SNicolas Pitre } 29178887cd99SNicolas Pitre 29188887cd99SNicolas Pitre int sched_group_set_rt_period(struct task_group *tg, u64 rt_period_us) 29198887cd99SNicolas Pitre { 29208887cd99SNicolas Pitre u64 rt_runtime, rt_period; 29218887cd99SNicolas Pitre 29221a010e29SKonstantin Khlebnikov if (rt_period_us > U64_MAX / NSEC_PER_USEC) 29231a010e29SKonstantin Khlebnikov return -EINVAL; 29241a010e29SKonstantin Khlebnikov 29258887cd99SNicolas Pitre rt_period = rt_period_us * NSEC_PER_USEC; 29268887cd99SNicolas Pitre rt_runtime = tg->rt_bandwidth.rt_runtime; 29278887cd99SNicolas Pitre 29288887cd99SNicolas Pitre return tg_set_rt_bandwidth(tg, rt_period, rt_runtime); 29298887cd99SNicolas Pitre } 29308887cd99SNicolas Pitre 29318887cd99SNicolas Pitre long sched_group_rt_period(struct task_group *tg) 29328887cd99SNicolas Pitre { 29338887cd99SNicolas Pitre u64 rt_period_us; 29348887cd99SNicolas Pitre 29358887cd99SNicolas Pitre rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period); 29368887cd99SNicolas Pitre do_div(rt_period_us, NSEC_PER_USEC); 29378887cd99SNicolas Pitre return rt_period_us; 29388887cd99SNicolas Pitre } 29398887cd99SNicolas Pitre 294028f152cdSBaisong Zhong #ifdef CONFIG_SYSCTL 29418887cd99SNicolas Pitre static int sched_rt_global_constraints(void) 29428887cd99SNicolas Pitre { 29438887cd99SNicolas Pitre int ret = 0; 29448887cd99SNicolas Pitre 29458887cd99SNicolas Pitre mutex_lock(&rt_constraints_mutex); 29468887cd99SNicolas Pitre ret = __rt_schedulable(NULL, 0, 0); 29478887cd99SNicolas Pitre mutex_unlock(&rt_constraints_mutex); 29488887cd99SNicolas Pitre 29498887cd99SNicolas Pitre return ret; 29508887cd99SNicolas Pitre } 295128f152cdSBaisong Zhong #endif /* CONFIG_SYSCTL */ 29528887cd99SNicolas Pitre 29538887cd99SNicolas Pitre int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk) 29548887cd99SNicolas Pitre { 29558887cd99SNicolas Pitre /* Don't accept realtime tasks when there is no way for them to run */ 29568887cd99SNicolas Pitre if (rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0) 29578887cd99SNicolas Pitre return 0; 29588887cd99SNicolas Pitre 29598887cd99SNicolas Pitre return 1; 29608887cd99SNicolas Pitre } 29618887cd99SNicolas Pitre 29628887cd99SNicolas Pitre #else /* !CONFIG_RT_GROUP_SCHED */ 296328f152cdSBaisong Zhong 296428f152cdSBaisong Zhong #ifdef CONFIG_SYSCTL 29658887cd99SNicolas Pitre static int sched_rt_global_constraints(void) 29668887cd99SNicolas Pitre { 29678887cd99SNicolas Pitre unsigned long flags; 29688887cd99SNicolas Pitre int i; 29698887cd99SNicolas Pitre 29708887cd99SNicolas Pitre raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags); 29718887cd99SNicolas Pitre for_each_possible_cpu(i) { 29728887cd99SNicolas Pitre struct rt_rq *rt_rq = &cpu_rq(i)->rt; 29738887cd99SNicolas Pitre 29748887cd99SNicolas Pitre raw_spin_lock(&rt_rq->rt_runtime_lock); 29758887cd99SNicolas Pitre rt_rq->rt_runtime = global_rt_runtime(); 29768887cd99SNicolas Pitre raw_spin_unlock(&rt_rq->rt_runtime_lock); 29778887cd99SNicolas Pitre } 29788887cd99SNicolas Pitre raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags); 29798887cd99SNicolas Pitre 29808887cd99SNicolas Pitre return 0; 29818887cd99SNicolas Pitre } 298228f152cdSBaisong Zhong #endif /* CONFIG_SYSCTL */ 29838887cd99SNicolas Pitre #endif /* CONFIG_RT_GROUP_SCHED */ 29848887cd99SNicolas Pitre 298528f152cdSBaisong Zhong #ifdef CONFIG_SYSCTL 29868887cd99SNicolas Pitre static int sched_rt_global_validate(void) 29878887cd99SNicolas Pitre { 29888887cd99SNicolas Pitre if (sysctl_sched_rt_period <= 0) 29898887cd99SNicolas Pitre return -EINVAL; 29908887cd99SNicolas Pitre 29918887cd99SNicolas Pitre if ((sysctl_sched_rt_runtime != RUNTIME_INF) && 2992d505b8afSHuaixin Chang ((sysctl_sched_rt_runtime > sysctl_sched_rt_period) || 2993d505b8afSHuaixin Chang ((u64)sysctl_sched_rt_runtime * 2994d505b8afSHuaixin Chang NSEC_PER_USEC > max_rt_runtime))) 29958887cd99SNicolas Pitre return -EINVAL; 29968887cd99SNicolas Pitre 29978887cd99SNicolas Pitre return 0; 29988887cd99SNicolas Pitre } 29998887cd99SNicolas Pitre 30008887cd99SNicolas Pitre static void sched_rt_do_global(void) 30018887cd99SNicolas Pitre { 30029b58e976SLi Hua unsigned long flags; 30039b58e976SLi Hua 30049b58e976SLi Hua raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags); 30058887cd99SNicolas Pitre def_rt_bandwidth.rt_runtime = global_rt_runtime(); 30068887cd99SNicolas Pitre def_rt_bandwidth.rt_period = ns_to_ktime(global_rt_period()); 30079b58e976SLi Hua raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags); 30088887cd99SNicolas Pitre } 30098887cd99SNicolas Pitre 3010d9ab0e63SZhen Ni static int sched_rt_handler(struct ctl_table *table, int write, void *buffer, 301132927393SChristoph Hellwig size_t *lenp, loff_t *ppos) 30128887cd99SNicolas Pitre { 30138887cd99SNicolas Pitre int old_period, old_runtime; 30148887cd99SNicolas Pitre static DEFINE_MUTEX(mutex); 30158887cd99SNicolas Pitre int ret; 30168887cd99SNicolas Pitre 30178887cd99SNicolas Pitre mutex_lock(&mutex); 30188887cd99SNicolas Pitre old_period = sysctl_sched_rt_period; 30198887cd99SNicolas Pitre old_runtime = sysctl_sched_rt_runtime; 30208887cd99SNicolas Pitre 30218887cd99SNicolas Pitre ret = proc_dointvec(table, write, buffer, lenp, ppos); 30228887cd99SNicolas Pitre 30238887cd99SNicolas Pitre if (!ret && write) { 30248887cd99SNicolas Pitre ret = sched_rt_global_validate(); 30258887cd99SNicolas Pitre if (ret) 30268887cd99SNicolas Pitre goto undo; 30278887cd99SNicolas Pitre 30288887cd99SNicolas Pitre ret = sched_dl_global_validate(); 30298887cd99SNicolas Pitre if (ret) 30308887cd99SNicolas Pitre goto undo; 30318887cd99SNicolas Pitre 30328887cd99SNicolas Pitre ret = sched_rt_global_constraints(); 30338887cd99SNicolas Pitre if (ret) 30348887cd99SNicolas Pitre goto undo; 30358887cd99SNicolas Pitre 30368887cd99SNicolas Pitre sched_rt_do_global(); 30378887cd99SNicolas Pitre sched_dl_do_global(); 30388887cd99SNicolas Pitre } 30398887cd99SNicolas Pitre if (0) { 30408887cd99SNicolas Pitre undo: 30418887cd99SNicolas Pitre sysctl_sched_rt_period = old_period; 30428887cd99SNicolas Pitre sysctl_sched_rt_runtime = old_runtime; 30438887cd99SNicolas Pitre } 30448887cd99SNicolas Pitre mutex_unlock(&mutex); 30458887cd99SNicolas Pitre 30468887cd99SNicolas Pitre return ret; 30478887cd99SNicolas Pitre } 30488887cd99SNicolas Pitre 3049dafd7a9dSZhen Ni static int sched_rr_handler(struct ctl_table *table, int write, void *buffer, 305032927393SChristoph Hellwig size_t *lenp, loff_t *ppos) 30518887cd99SNicolas Pitre { 30528887cd99SNicolas Pitre int ret; 30538887cd99SNicolas Pitre static DEFINE_MUTEX(mutex); 30548887cd99SNicolas Pitre 30558887cd99SNicolas Pitre mutex_lock(&mutex); 30568887cd99SNicolas Pitre ret = proc_dointvec(table, write, buffer, lenp, ppos); 30578887cd99SNicolas Pitre /* 30588887cd99SNicolas Pitre * Make sure that internally we keep jiffies. 30598887cd99SNicolas Pitre * Also, writing zero resets the timeslice to default: 30608887cd99SNicolas Pitre */ 30618887cd99SNicolas Pitre if (!ret && write) { 30628887cd99SNicolas Pitre sched_rr_timeslice = 30638887cd99SNicolas Pitre sysctl_sched_rr_timeslice <= 0 ? RR_TIMESLICE : 30648887cd99SNicolas Pitre msecs_to_jiffies(sysctl_sched_rr_timeslice); 30658887cd99SNicolas Pitre } 30668887cd99SNicolas Pitre mutex_unlock(&mutex); 306797fb7a0aSIngo Molnar 30688887cd99SNicolas Pitre return ret; 30698887cd99SNicolas Pitre } 307028f152cdSBaisong Zhong #endif /* CONFIG_SYSCTL */ 30718887cd99SNicolas Pitre 3072391e43daSPeter Zijlstra #ifdef CONFIG_SCHED_DEBUG 3073391e43daSPeter Zijlstra void print_rt_stats(struct seq_file *m, int cpu) 3074391e43daSPeter Zijlstra { 3075391e43daSPeter Zijlstra rt_rq_iter_t iter; 3076391e43daSPeter Zijlstra struct rt_rq *rt_rq; 3077391e43daSPeter Zijlstra 3078391e43daSPeter Zijlstra rcu_read_lock(); 3079391e43daSPeter Zijlstra for_each_rt_rq(rt_rq, iter, cpu_rq(cpu)) 3080391e43daSPeter Zijlstra print_rt_rq(m, cpu, rt_rq); 3081391e43daSPeter Zijlstra rcu_read_unlock(); 3082391e43daSPeter Zijlstra } 3083391e43daSPeter Zijlstra #endif /* CONFIG_SCHED_DEBUG */ 3084