1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright 2018 Linaro Limited 4 * 5 * Author: Daniel Lezcano <daniel.lezcano@linaro.org> 6 * 7 * The idle injection framework provides a way to force CPUs to enter idle 8 * states for a specified fraction of time over a specified period. 9 * 10 * It relies on the smpboot kthreads feature providing common code for CPU 11 * hotplug and thread [un]parking. 12 * 13 * All of the kthreads used for idle injection are created at init time. 14 * 15 * Next, the users of the idle injection framework provide a cpumask via 16 * its register function. The kthreads will be synchronized with respect to 17 * this cpumask. 18 * 19 * The idle + run duration is specified via separate helpers and that allows 20 * idle injection to be started. 21 * 22 * The idle injection kthreads will call play_idle_precise() with the idle 23 * duration and max allowed latency specified as per the above. 24 * 25 * After all of them have been woken up, a timer is set to start the next idle 26 * injection cycle. 27 * 28 * The timer interrupt handler will wake up the idle injection kthreads for 29 * all of the CPUs in the cpumask provided by the user. 30 * 31 * Idle injection is stopped synchronously and no leftover idle injection 32 * kthread activity after its completion is guaranteed. 33 * 34 * It is up to the user of this framework to provide a lock for higher-level 35 * synchronization to prevent race conditions like starting idle injection 36 * while unregistering from the framework. 37 */ 38 #define pr_fmt(fmt) "ii_dev: " fmt 39 40 #include <linux/cpu.h> 41 #include <linux/hrtimer.h> 42 #include <linux/kthread.h> 43 #include <linux/sched.h> 44 #include <linux/slab.h> 45 #include <linux/smpboot.h> 46 #include <linux/idle_inject.h> 47 48 #include <uapi/linux/sched/types.h> 49 50 /** 51 * struct idle_inject_thread - task on/off switch structure 52 * @tsk: task injecting the idle cycles 53 * @should_run: whether or not to run the task (for the smpboot kthread API) 54 */ 55 struct idle_inject_thread { 56 struct task_struct *tsk; 57 int should_run; 58 }; 59 60 /** 61 * struct idle_inject_device - idle injection data 62 * @timer: idle injection period timer 63 * @idle_duration_us: duration of CPU idle time to inject 64 * @run_duration_us: duration of CPU run time to allow 65 * @latency_us: max allowed latency 66 * @cpumask: mask of CPUs affected by idle injection 67 */ 68 struct idle_inject_device { 69 struct hrtimer timer; 70 unsigned int idle_duration_us; 71 unsigned int run_duration_us; 72 unsigned int latency_us; 73 unsigned long cpumask[]; 74 }; 75 76 static DEFINE_PER_CPU(struct idle_inject_thread, idle_inject_thread); 77 static DEFINE_PER_CPU(struct idle_inject_device *, idle_inject_device); 78 79 /** 80 * idle_inject_wakeup - Wake up idle injection threads 81 * @ii_dev: target idle injection device 82 * 83 * Every idle injection task associated with the given idle injection device 84 * and running on an online CPU will be woken up. 85 */ 86 static void idle_inject_wakeup(struct idle_inject_device *ii_dev) 87 { 88 struct idle_inject_thread *iit; 89 unsigned int cpu; 90 91 for_each_cpu_and(cpu, to_cpumask(ii_dev->cpumask), cpu_online_mask) { 92 iit = per_cpu_ptr(&idle_inject_thread, cpu); 93 iit->should_run = 1; 94 wake_up_process(iit->tsk); 95 } 96 } 97 98 /** 99 * idle_inject_timer_fn - idle injection timer function 100 * @timer: idle injection hrtimer 101 * 102 * This function is called when the idle injection timer expires. It wakes up 103 * idle injection tasks associated with the timer and they, in turn, invoke 104 * play_idle_precise() to inject a specified amount of CPU idle time. 105 * 106 * Return: HRTIMER_RESTART. 107 */ 108 static enum hrtimer_restart idle_inject_timer_fn(struct hrtimer *timer) 109 { 110 unsigned int duration_us; 111 struct idle_inject_device *ii_dev = 112 container_of(timer, struct idle_inject_device, timer); 113 114 duration_us = READ_ONCE(ii_dev->run_duration_us); 115 duration_us += READ_ONCE(ii_dev->idle_duration_us); 116 117 idle_inject_wakeup(ii_dev); 118 119 hrtimer_forward_now(timer, ns_to_ktime(duration_us * NSEC_PER_USEC)); 120 121 return HRTIMER_RESTART; 122 } 123 124 /** 125 * idle_inject_fn - idle injection work function 126 * @cpu: the CPU owning the task 127 * 128 * This function calls play_idle_precise() to inject a specified amount of CPU 129 * idle time. 130 */ 131 static void idle_inject_fn(unsigned int cpu) 132 { 133 struct idle_inject_device *ii_dev; 134 struct idle_inject_thread *iit; 135 136 ii_dev = per_cpu(idle_inject_device, cpu); 137 iit = per_cpu_ptr(&idle_inject_thread, cpu); 138 139 /* 140 * Let the smpboot main loop know that the task should not run again. 141 */ 142 iit->should_run = 0; 143 144 play_idle_precise(READ_ONCE(ii_dev->idle_duration_us) * NSEC_PER_USEC, 145 READ_ONCE(ii_dev->latency_us) * NSEC_PER_USEC); 146 } 147 148 /** 149 * idle_inject_set_duration - idle and run duration update helper 150 * @ii_dev: idle injection control device structure 151 * @run_duration_us: CPU run time to allow in microseconds 152 * @idle_duration_us: CPU idle time to inject in microseconds 153 */ 154 void idle_inject_set_duration(struct idle_inject_device *ii_dev, 155 unsigned int run_duration_us, 156 unsigned int idle_duration_us) 157 { 158 if (run_duration_us && idle_duration_us) { 159 WRITE_ONCE(ii_dev->run_duration_us, run_duration_us); 160 WRITE_ONCE(ii_dev->idle_duration_us, idle_duration_us); 161 } 162 } 163 164 /** 165 * idle_inject_get_duration - idle and run duration retrieval helper 166 * @ii_dev: idle injection control device structure 167 * @run_duration_us: memory location to store the current CPU run time 168 * @idle_duration_us: memory location to store the current CPU idle time 169 */ 170 void idle_inject_get_duration(struct idle_inject_device *ii_dev, 171 unsigned int *run_duration_us, 172 unsigned int *idle_duration_us) 173 { 174 *run_duration_us = READ_ONCE(ii_dev->run_duration_us); 175 *idle_duration_us = READ_ONCE(ii_dev->idle_duration_us); 176 } 177 178 /** 179 * idle_inject_set_latency - set the maximum latency allowed 180 * @ii_dev: idle injection control device structure 181 * @latency_us: set the latency requirement for the idle state 182 */ 183 void idle_inject_set_latency(struct idle_inject_device *ii_dev, 184 unsigned int latency_us) 185 { 186 WRITE_ONCE(ii_dev->latency_us, latency_us); 187 } 188 189 /** 190 * idle_inject_start - start idle injections 191 * @ii_dev: idle injection control device structure 192 * 193 * The function starts idle injection by first waking up all of the idle 194 * injection kthreads associated with @ii_dev to let them inject CPU idle time 195 * sets up a timer to start the next idle injection period. 196 * 197 * Return: -EINVAL if the CPU idle or CPU run time is not set or 0 on success. 198 */ 199 int idle_inject_start(struct idle_inject_device *ii_dev) 200 { 201 unsigned int idle_duration_us = READ_ONCE(ii_dev->idle_duration_us); 202 unsigned int run_duration_us = READ_ONCE(ii_dev->run_duration_us); 203 204 if (!idle_duration_us || !run_duration_us) 205 return -EINVAL; 206 207 pr_debug("Starting injecting idle cycles on CPUs '%*pbl'\n", 208 cpumask_pr_args(to_cpumask(ii_dev->cpumask))); 209 210 idle_inject_wakeup(ii_dev); 211 212 hrtimer_start(&ii_dev->timer, 213 ns_to_ktime((idle_duration_us + run_duration_us) * 214 NSEC_PER_USEC), 215 HRTIMER_MODE_REL); 216 217 return 0; 218 } 219 220 /** 221 * idle_inject_stop - stops idle injections 222 * @ii_dev: idle injection control device structure 223 * 224 * The function stops idle injection and waits for the threads to finish work. 225 * If CPU idle time is being injected when this function runs, then it will 226 * wait until the end of the cycle. 227 * 228 * When it returns, there is no more idle injection kthread activity. The 229 * kthreads are scheduled out and the periodic timer is off. 230 */ 231 void idle_inject_stop(struct idle_inject_device *ii_dev) 232 { 233 struct idle_inject_thread *iit; 234 unsigned int cpu; 235 236 pr_debug("Stopping idle injection on CPUs '%*pbl'\n", 237 cpumask_pr_args(to_cpumask(ii_dev->cpumask))); 238 239 hrtimer_cancel(&ii_dev->timer); 240 241 /* 242 * Stopping idle injection requires all of the idle injection kthreads 243 * associated with the given cpumask to be parked and stay that way, so 244 * prevent CPUs from going online at this point. Any CPUs going online 245 * after the loop below will be covered by clearing the should_run flag 246 * that will cause the smpboot main loop to schedule them out. 247 */ 248 cpu_hotplug_disable(); 249 250 /* 251 * Iterate over all (online + offline) CPUs here in case one of them 252 * goes offline with the should_run flag set so as to prevent its idle 253 * injection kthread from running when the CPU goes online again after 254 * the ii_dev has been freed. 255 */ 256 for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) { 257 iit = per_cpu_ptr(&idle_inject_thread, cpu); 258 iit->should_run = 0; 259 260 wait_task_inactive(iit->tsk, TASK_ANY); 261 } 262 263 cpu_hotplug_enable(); 264 } 265 266 /** 267 * idle_inject_setup - prepare the current task for idle injection 268 * @cpu: not used 269 * 270 * Called once, this function is in charge of setting the current task's 271 * scheduler parameters to make it an RT task. 272 */ 273 static void idle_inject_setup(unsigned int cpu) 274 { 275 sched_set_fifo(current); 276 } 277 278 /** 279 * idle_inject_should_run - function helper for the smpboot API 280 * @cpu: CPU the kthread is running on 281 * 282 * Return: whether or not the thread can run. 283 */ 284 static int idle_inject_should_run(unsigned int cpu) 285 { 286 struct idle_inject_thread *iit = 287 per_cpu_ptr(&idle_inject_thread, cpu); 288 289 return iit->should_run; 290 } 291 292 /** 293 * idle_inject_register - initialize idle injection on a set of CPUs 294 * @cpumask: CPUs to be affected by idle injection 295 * 296 * This function creates an idle injection control device structure for the 297 * given set of CPUs and initializes the timer associated with it. It does not 298 * start any injection cycles. 299 * 300 * Return: NULL if memory allocation fails, idle injection control device 301 * pointer on success. 302 */ 303 struct idle_inject_device *idle_inject_register(struct cpumask *cpumask) 304 { 305 struct idle_inject_device *ii_dev; 306 int cpu, cpu_rb; 307 308 ii_dev = kzalloc(sizeof(*ii_dev) + cpumask_size(), GFP_KERNEL); 309 if (!ii_dev) 310 return NULL; 311 312 cpumask_copy(to_cpumask(ii_dev->cpumask), cpumask); 313 hrtimer_init(&ii_dev->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 314 ii_dev->timer.function = idle_inject_timer_fn; 315 ii_dev->latency_us = UINT_MAX; 316 317 for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) { 318 319 if (per_cpu(idle_inject_device, cpu)) { 320 pr_err("cpu%d is already registered\n", cpu); 321 goto out_rollback; 322 } 323 324 per_cpu(idle_inject_device, cpu) = ii_dev; 325 } 326 327 return ii_dev; 328 329 out_rollback: 330 for_each_cpu(cpu_rb, to_cpumask(ii_dev->cpumask)) { 331 if (cpu == cpu_rb) 332 break; 333 per_cpu(idle_inject_device, cpu_rb) = NULL; 334 } 335 336 kfree(ii_dev); 337 338 return NULL; 339 } 340 341 /** 342 * idle_inject_unregister - unregister idle injection control device 343 * @ii_dev: idle injection control device to unregister 344 * 345 * The function stops idle injection for the given control device, 346 * unregisters its kthreads and frees memory allocated when that device was 347 * created. 348 */ 349 void idle_inject_unregister(struct idle_inject_device *ii_dev) 350 { 351 unsigned int cpu; 352 353 idle_inject_stop(ii_dev); 354 355 for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) 356 per_cpu(idle_inject_device, cpu) = NULL; 357 358 kfree(ii_dev); 359 } 360 361 static struct smp_hotplug_thread idle_inject_threads = { 362 .store = &idle_inject_thread.tsk, 363 .setup = idle_inject_setup, 364 .thread_fn = idle_inject_fn, 365 .thread_comm = "idle_inject/%u", 366 .thread_should_run = idle_inject_should_run, 367 }; 368 369 static int __init idle_inject_init(void) 370 { 371 return smpboot_register_percpu_thread(&idle_inject_threads); 372 } 373 early_initcall(idle_inject_init); 374