1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright 2018 Linaro Limited 4 * 5 * Author: Daniel Lezcano <daniel.lezcano@linaro.org> 6 * 7 * The idle injection framework provides a way to force CPUs to enter idle 8 * states for a specified fraction of time over a specified period. 9 * 10 * It relies on the smpboot kthreads feature providing common code for CPU 11 * hotplug and thread [un]parking. 12 * 13 * All of the kthreads used for idle injection are created at init time. 14 * 15 * Next, the users of the the idle injection framework provide a cpumask via 16 * its register function. The kthreads will be synchronized with respect to 17 * this cpumask. 18 * 19 * The idle + run duration is specified via separate helpers and that allows 20 * idle injection to be started. 21 * 22 * The idle injection kthreads will call play_idle() with the idle duration 23 * specified as per the above. 24 * 25 * After all of them have been woken up, a timer is set to start the next idle 26 * injection cycle. 27 * 28 * The timer interrupt handler will wake up the idle injection kthreads for 29 * all of the CPUs in the cpumask provided by the user. 30 * 31 * Idle injection is stopped synchronously and no leftover idle injection 32 * kthread activity after its completion is guaranteed. 33 * 34 * It is up to the user of this framework to provide a lock for higher-level 35 * synchronization to prevent race conditions like starting idle injection 36 * while unregistering from the framework. 37 */ 38 #define pr_fmt(fmt) "ii_dev: " fmt 39 40 #include <linux/cpu.h> 41 #include <linux/hrtimer.h> 42 #include <linux/kthread.h> 43 #include <linux/sched.h> 44 #include <linux/slab.h> 45 #include <linux/smpboot.h> 46 47 #include <uapi/linux/sched/types.h> 48 49 /** 50 * struct idle_inject_thread - task on/off switch structure 51 * @tsk: task injecting the idle cycles 52 * @should_run: whether or not to run the task (for the smpboot kthread API) 53 */ 54 struct idle_inject_thread { 55 struct task_struct *tsk; 56 int should_run; 57 }; 58 59 /** 60 * struct idle_inject_device - idle injection data 61 * @timer: idle injection period timer 62 * @idle_duration_us: duration of CPU idle time to inject 63 * @run_duration_us: duration of CPU run time to allow 64 * @cpumask: mask of CPUs affected by idle injection 65 */ 66 struct idle_inject_device { 67 struct hrtimer timer; 68 unsigned int idle_duration_us; 69 unsigned int run_duration_us; 70 unsigned long cpumask[]; 71 }; 72 73 static DEFINE_PER_CPU(struct idle_inject_thread, idle_inject_thread); 74 static DEFINE_PER_CPU(struct idle_inject_device *, idle_inject_device); 75 76 /** 77 * idle_inject_wakeup - Wake up idle injection threads 78 * @ii_dev: target idle injection device 79 * 80 * Every idle injection task associated with the given idle injection device 81 * and running on an online CPU will be woken up. 82 */ 83 static void idle_inject_wakeup(struct idle_inject_device *ii_dev) 84 { 85 struct idle_inject_thread *iit; 86 unsigned int cpu; 87 88 for_each_cpu_and(cpu, to_cpumask(ii_dev->cpumask), cpu_online_mask) { 89 iit = per_cpu_ptr(&idle_inject_thread, cpu); 90 iit->should_run = 1; 91 wake_up_process(iit->tsk); 92 } 93 } 94 95 /** 96 * idle_inject_timer_fn - idle injection timer function 97 * @timer: idle injection hrtimer 98 * 99 * This function is called when the idle injection timer expires. It wakes up 100 * idle injection tasks associated with the timer and they, in turn, invoke 101 * play_idle() to inject a specified amount of CPU idle time. 102 * 103 * Return: HRTIMER_RESTART. 104 */ 105 static enum hrtimer_restart idle_inject_timer_fn(struct hrtimer *timer) 106 { 107 unsigned int duration_us; 108 struct idle_inject_device *ii_dev = 109 container_of(timer, struct idle_inject_device, timer); 110 111 duration_us = READ_ONCE(ii_dev->run_duration_us); 112 duration_us += READ_ONCE(ii_dev->idle_duration_us); 113 114 idle_inject_wakeup(ii_dev); 115 116 hrtimer_forward_now(timer, ns_to_ktime(duration_us * NSEC_PER_USEC)); 117 118 return HRTIMER_RESTART; 119 } 120 121 /** 122 * idle_inject_fn - idle injection work function 123 * @cpu: the CPU owning the task 124 * 125 * This function calls play_idle() to inject a specified amount of CPU idle 126 * time. 127 */ 128 static void idle_inject_fn(unsigned int cpu) 129 { 130 struct idle_inject_device *ii_dev; 131 struct idle_inject_thread *iit; 132 133 ii_dev = per_cpu(idle_inject_device, cpu); 134 iit = per_cpu_ptr(&idle_inject_thread, cpu); 135 136 /* 137 * Let the smpboot main loop know that the task should not run again. 138 */ 139 iit->should_run = 0; 140 141 play_idle(READ_ONCE(ii_dev->idle_duration_us)); 142 } 143 144 /** 145 * idle_inject_set_duration - idle and run duration update helper 146 * @run_duration_us: CPU run time to allow in microseconds 147 * @idle_duration_us: CPU idle time to inject in microseconds 148 */ 149 void idle_inject_set_duration(struct idle_inject_device *ii_dev, 150 unsigned int run_duration_us, 151 unsigned int idle_duration_us) 152 { 153 if (run_duration_us && idle_duration_us) { 154 WRITE_ONCE(ii_dev->run_duration_us, run_duration_us); 155 WRITE_ONCE(ii_dev->idle_duration_us, idle_duration_us); 156 } 157 } 158 159 /** 160 * idle_inject_get_duration - idle and run duration retrieval helper 161 * @run_duration_us: memory location to store the current CPU run time 162 * @idle_duration_us: memory location to store the current CPU idle time 163 */ 164 void idle_inject_get_duration(struct idle_inject_device *ii_dev, 165 unsigned int *run_duration_us, 166 unsigned int *idle_duration_us) 167 { 168 *run_duration_us = READ_ONCE(ii_dev->run_duration_us); 169 *idle_duration_us = READ_ONCE(ii_dev->idle_duration_us); 170 } 171 172 /** 173 * idle_inject_start - start idle injections 174 * @ii_dev: idle injection control device structure 175 * 176 * The function starts idle injection by first waking up all of the idle 177 * injection kthreads associated with @ii_dev to let them inject CPU idle time 178 * sets up a timer to start the next idle injection period. 179 * 180 * Return: -EINVAL if the CPU idle or CPU run time is not set or 0 on success. 181 */ 182 int idle_inject_start(struct idle_inject_device *ii_dev) 183 { 184 unsigned int idle_duration_us = READ_ONCE(ii_dev->idle_duration_us); 185 unsigned int run_duration_us = READ_ONCE(ii_dev->run_duration_us); 186 187 if (!idle_duration_us || !run_duration_us) 188 return -EINVAL; 189 190 pr_debug("Starting injecting idle cycles on CPUs '%*pbl'\n", 191 cpumask_pr_args(to_cpumask(ii_dev->cpumask))); 192 193 idle_inject_wakeup(ii_dev); 194 195 hrtimer_start(&ii_dev->timer, 196 ns_to_ktime((idle_duration_us + run_duration_us) * 197 NSEC_PER_USEC), 198 HRTIMER_MODE_REL); 199 200 return 0; 201 } 202 203 /** 204 * idle_inject_stop - stops idle injections 205 * @ii_dev: idle injection control device structure 206 * 207 * The function stops idle injection and waits for the threads to finish work. 208 * If CPU idle time is being injected when this function runs, then it will 209 * wait until the end of the cycle. 210 * 211 * When it returns, there is no more idle injection kthread activity. The 212 * kthreads are scheduled out and the periodic timer is off. 213 */ 214 void idle_inject_stop(struct idle_inject_device *ii_dev) 215 { 216 struct idle_inject_thread *iit; 217 unsigned int cpu; 218 219 pr_debug("Stopping idle injection on CPUs '%*pbl'\n", 220 cpumask_pr_args(to_cpumask(ii_dev->cpumask))); 221 222 hrtimer_cancel(&ii_dev->timer); 223 224 /* 225 * Stopping idle injection requires all of the idle injection kthreads 226 * associated with the given cpumask to be parked and stay that way, so 227 * prevent CPUs from going online at this point. Any CPUs going online 228 * after the loop below will be covered by clearing the should_run flag 229 * that will cause the smpboot main loop to schedule them out. 230 */ 231 cpu_hotplug_disable(); 232 233 /* 234 * Iterate over all (online + offline) CPUs here in case one of them 235 * goes offline with the should_run flag set so as to prevent its idle 236 * injection kthread from running when the CPU goes online again after 237 * the ii_dev has been freed. 238 */ 239 for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) { 240 iit = per_cpu_ptr(&idle_inject_thread, cpu); 241 iit->should_run = 0; 242 243 wait_task_inactive(iit->tsk, 0); 244 } 245 246 cpu_hotplug_enable(); 247 } 248 249 /** 250 * idle_inject_setup - prepare the current task for idle injection 251 * @cpu: not used 252 * 253 * Called once, this function is in charge of setting the current task's 254 * scheduler parameters to make it an RT task. 255 */ 256 static void idle_inject_setup(unsigned int cpu) 257 { 258 struct sched_param param = { .sched_priority = MAX_USER_RT_PRIO / 2 }; 259 260 sched_setscheduler(current, SCHED_FIFO, ¶m); 261 } 262 263 /** 264 * idle_inject_should_run - function helper for the smpboot API 265 * @cpu: CPU the kthread is running on 266 * 267 * Return: whether or not the thread can run. 268 */ 269 static int idle_inject_should_run(unsigned int cpu) 270 { 271 struct idle_inject_thread *iit = 272 per_cpu_ptr(&idle_inject_thread, cpu); 273 274 return iit->should_run; 275 } 276 277 /** 278 * idle_inject_register - initialize idle injection on a set of CPUs 279 * @cpumask: CPUs to be affected by idle injection 280 * 281 * This function creates an idle injection control device structure for the 282 * given set of CPUs and initializes the timer associated with it. It does not 283 * start any injection cycles. 284 * 285 * Return: NULL if memory allocation fails, idle injection control device 286 * pointer on success. 287 */ 288 struct idle_inject_device *idle_inject_register(struct cpumask *cpumask) 289 { 290 struct idle_inject_device *ii_dev; 291 int cpu, cpu_rb; 292 293 ii_dev = kzalloc(sizeof(*ii_dev) + cpumask_size(), GFP_KERNEL); 294 if (!ii_dev) 295 return NULL; 296 297 cpumask_copy(to_cpumask(ii_dev->cpumask), cpumask); 298 hrtimer_init(&ii_dev->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 299 ii_dev->timer.function = idle_inject_timer_fn; 300 301 for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) { 302 303 if (per_cpu(idle_inject_device, cpu)) { 304 pr_err("cpu%d is already registered\n", cpu); 305 goto out_rollback; 306 } 307 308 per_cpu(idle_inject_device, cpu) = ii_dev; 309 } 310 311 return ii_dev; 312 313 out_rollback: 314 for_each_cpu(cpu_rb, to_cpumask(ii_dev->cpumask)) { 315 if (cpu == cpu_rb) 316 break; 317 per_cpu(idle_inject_device, cpu_rb) = NULL; 318 } 319 320 kfree(ii_dev); 321 322 return NULL; 323 } 324 325 /** 326 * idle_inject_unregister - unregister idle injection control device 327 * @ii_dev: idle injection control device to unregister 328 * 329 * The function stops idle injection for the given control device, 330 * unregisters its kthreads and frees memory allocated when that device was 331 * created. 332 */ 333 void idle_inject_unregister(struct idle_inject_device *ii_dev) 334 { 335 unsigned int cpu; 336 337 idle_inject_stop(ii_dev); 338 339 for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) 340 per_cpu(idle_inject_device, cpu) = NULL; 341 342 kfree(ii_dev); 343 } 344 345 static struct smp_hotplug_thread idle_inject_threads = { 346 .store = &idle_inject_thread.tsk, 347 .setup = idle_inject_setup, 348 .thread_fn = idle_inject_fn, 349 .thread_comm = "idle_inject/%u", 350 .thread_should_run = idle_inject_should_run, 351 }; 352 353 static int __init idle_inject_init(void) 354 { 355 return smpboot_register_percpu_thread(&idle_inject_threads); 356 } 357 early_initcall(idle_inject_init); 358