1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright 2018 Linaro Limited
4  *
5  * Author: Daniel Lezcano <daniel.lezcano@linaro.org>
6  *
7  * The idle injection framework provides a way to force CPUs to enter idle
8  * states for a specified fraction of time over a specified period.
9  *
10  * It relies on the smpboot kthreads feature providing common code for CPU
11  * hotplug and thread [un]parking.
12  *
13  * All of the kthreads used for idle injection are created at init time.
14  *
15  * Next, the users of the idle injection framework provide a cpumask via
16  * its register function. The kthreads will be synchronized with respect to
17  * this cpumask.
18  *
19  * The idle + run duration is specified via separate helpers and that allows
20  * idle injection to be started.
21  *
22  * The idle injection kthreads will call play_idle_precise() with the idle
23  * duration and max allowed latency specified as per the above.
24  *
25  * After all of them have been woken up, a timer is set to start the next idle
26  * injection cycle.
27  *
28  * The timer interrupt handler will wake up the idle injection kthreads for
29  * all of the CPUs in the cpumask provided by the user.
30  *
31  * Idle injection is stopped synchronously and no leftover idle injection
32  * kthread activity after its completion is guaranteed.
33  *
34  * It is up to the user of this framework to provide a lock for higher-level
35  * synchronization to prevent race conditions like starting idle injection
36  * while unregistering from the framework.
37  */
38 #define pr_fmt(fmt) "ii_dev: " fmt
39 
40 #include <linux/cpu.h>
41 #include <linux/hrtimer.h>
42 #include <linux/kthread.h>
43 #include <linux/sched.h>
44 #include <linux/slab.h>
45 #include <linux/smpboot.h>
46 #include <linux/idle_inject.h>
47 
48 #include <uapi/linux/sched/types.h>
49 
50 /**
51  * struct idle_inject_thread - task on/off switch structure
52  * @tsk: task injecting the idle cycles
53  * @should_run: whether or not to run the task (for the smpboot kthread API)
54  */
55 struct idle_inject_thread {
56 	struct task_struct *tsk;
57 	int should_run;
58 };
59 
60 /**
61  * struct idle_inject_device - idle injection data
62  * @timer: idle injection period timer
63  * @idle_duration_us: duration of CPU idle time to inject
64  * @run_duration_us: duration of CPU run time to allow
65  * @latency_us: max allowed latency
66  * @cpumask: mask of CPUs affected by idle injection
67  */
68 struct idle_inject_device {
69 	struct hrtimer timer;
70 	unsigned int idle_duration_us;
71 	unsigned int run_duration_us;
72 	unsigned int latency_us;
73 	unsigned long cpumask[];
74 };
75 
76 static DEFINE_PER_CPU(struct idle_inject_thread, idle_inject_thread);
77 static DEFINE_PER_CPU(struct idle_inject_device *, idle_inject_device);
78 
79 /**
80  * idle_inject_wakeup - Wake up idle injection threads
81  * @ii_dev: target idle injection device
82  *
83  * Every idle injection task associated with the given idle injection device
84  * and running on an online CPU will be woken up.
85  */
86 static void idle_inject_wakeup(struct idle_inject_device *ii_dev)
87 {
88 	struct idle_inject_thread *iit;
89 	unsigned int cpu;
90 
91 	for_each_cpu_and(cpu, to_cpumask(ii_dev->cpumask), cpu_online_mask) {
92 		iit = per_cpu_ptr(&idle_inject_thread, cpu);
93 		iit->should_run = 1;
94 		wake_up_process(iit->tsk);
95 	}
96 }
97 
98 /**
99  * idle_inject_timer_fn - idle injection timer function
100  * @timer: idle injection hrtimer
101  *
102  * This function is called when the idle injection timer expires.  It wakes up
103  * idle injection tasks associated with the timer and they, in turn, invoke
104  * play_idle_precise() to inject a specified amount of CPU idle time.
105  *
106  * Return: HRTIMER_RESTART.
107  */
108 static enum hrtimer_restart idle_inject_timer_fn(struct hrtimer *timer)
109 {
110 	unsigned int duration_us;
111 	struct idle_inject_device *ii_dev =
112 		container_of(timer, struct idle_inject_device, timer);
113 
114 	duration_us = READ_ONCE(ii_dev->run_duration_us);
115 	duration_us += READ_ONCE(ii_dev->idle_duration_us);
116 
117 	idle_inject_wakeup(ii_dev);
118 
119 	hrtimer_forward_now(timer, ns_to_ktime(duration_us * NSEC_PER_USEC));
120 
121 	return HRTIMER_RESTART;
122 }
123 
124 /**
125  * idle_inject_fn - idle injection work function
126  * @cpu: the CPU owning the task
127  *
128  * This function calls play_idle_precise() to inject a specified amount of CPU
129  * idle time.
130  */
131 static void idle_inject_fn(unsigned int cpu)
132 {
133 	struct idle_inject_device *ii_dev;
134 	struct idle_inject_thread *iit;
135 
136 	ii_dev = per_cpu(idle_inject_device, cpu);
137 	iit = per_cpu_ptr(&idle_inject_thread, cpu);
138 
139 	/*
140 	 * Let the smpboot main loop know that the task should not run again.
141 	 */
142 	iit->should_run = 0;
143 
144 	play_idle_precise(READ_ONCE(ii_dev->idle_duration_us) * NSEC_PER_USEC,
145 			  READ_ONCE(ii_dev->latency_us) * NSEC_PER_USEC);
146 }
147 
148 /**
149  * idle_inject_set_duration - idle and run duration update helper
150  * @ii_dev: idle injection control device structure
151  * @run_duration_us: CPU run time to allow in microseconds
152  * @idle_duration_us: CPU idle time to inject in microseconds
153  */
154 void idle_inject_set_duration(struct idle_inject_device *ii_dev,
155 			      unsigned int run_duration_us,
156 			      unsigned int idle_duration_us)
157 {
158 	if (run_duration_us && idle_duration_us) {
159 		WRITE_ONCE(ii_dev->run_duration_us, run_duration_us);
160 		WRITE_ONCE(ii_dev->idle_duration_us, idle_duration_us);
161 	}
162 }
163 
164 /**
165  * idle_inject_get_duration - idle and run duration retrieval helper
166  * @ii_dev: idle injection control device structure
167  * @run_duration_us: memory location to store the current CPU run time
168  * @idle_duration_us: memory location to store the current CPU idle time
169  */
170 void idle_inject_get_duration(struct idle_inject_device *ii_dev,
171 			      unsigned int *run_duration_us,
172 			      unsigned int *idle_duration_us)
173 {
174 	*run_duration_us = READ_ONCE(ii_dev->run_duration_us);
175 	*idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);
176 }
177 
178 /**
179  * idle_inject_set_latency - set the maximum latency allowed
180  * @ii_dev: idle injection control device structure
181  * @latency_us: set the latency requirement for the idle state
182  */
183 void idle_inject_set_latency(struct idle_inject_device *ii_dev,
184 			     unsigned int latency_us)
185 {
186 	WRITE_ONCE(ii_dev->latency_us, latency_us);
187 }
188 
189 /**
190  * idle_inject_start - start idle injections
191  * @ii_dev: idle injection control device structure
192  *
193  * The function starts idle injection by first waking up all of the idle
194  * injection kthreads associated with @ii_dev to let them inject CPU idle time
195  * sets up a timer to start the next idle injection period.
196  *
197  * Return: -EINVAL if the CPU idle or CPU run time is not set or 0 on success.
198  */
199 int idle_inject_start(struct idle_inject_device *ii_dev)
200 {
201 	unsigned int idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);
202 	unsigned int run_duration_us = READ_ONCE(ii_dev->run_duration_us);
203 
204 	if (!idle_duration_us || !run_duration_us)
205 		return -EINVAL;
206 
207 	pr_debug("Starting injecting idle cycles on CPUs '%*pbl'\n",
208 		 cpumask_pr_args(to_cpumask(ii_dev->cpumask)));
209 
210 	idle_inject_wakeup(ii_dev);
211 
212 	hrtimer_start(&ii_dev->timer,
213 		      ns_to_ktime((idle_duration_us + run_duration_us) *
214 				  NSEC_PER_USEC),
215 		      HRTIMER_MODE_REL);
216 
217 	return 0;
218 }
219 
220 /**
221  * idle_inject_stop - stops idle injections
222  * @ii_dev: idle injection control device structure
223  *
224  * The function stops idle injection and waits for the threads to finish work.
225  * If CPU idle time is being injected when this function runs, then it will
226  * wait until the end of the cycle.
227  *
228  * When it returns, there is no more idle injection kthread activity.  The
229  * kthreads are scheduled out and the periodic timer is off.
230  */
231 void idle_inject_stop(struct idle_inject_device *ii_dev)
232 {
233 	struct idle_inject_thread *iit;
234 	unsigned int cpu;
235 
236 	pr_debug("Stopping idle injection on CPUs '%*pbl'\n",
237 		 cpumask_pr_args(to_cpumask(ii_dev->cpumask)));
238 
239 	hrtimer_cancel(&ii_dev->timer);
240 
241 	/*
242 	 * Stopping idle injection requires all of the idle injection kthreads
243 	 * associated with the given cpumask to be parked and stay that way, so
244 	 * prevent CPUs from going online at this point.  Any CPUs going online
245 	 * after the loop below will be covered by clearing the should_run flag
246 	 * that will cause the smpboot main loop to schedule them out.
247 	 */
248 	cpu_hotplug_disable();
249 
250 	/*
251 	 * Iterate over all (online + offline) CPUs here in case one of them
252 	 * goes offline with the should_run flag set so as to prevent its idle
253 	 * injection kthread from running when the CPU goes online again after
254 	 * the ii_dev has been freed.
255 	 */
256 	for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {
257 		iit = per_cpu_ptr(&idle_inject_thread, cpu);
258 		iit->should_run = 0;
259 
260 		wait_task_inactive(iit->tsk, TASK_ANY);
261 	}
262 
263 	cpu_hotplug_enable();
264 }
265 
266 /**
267  * idle_inject_setup - prepare the current task for idle injection
268  * @cpu: not used
269  *
270  * Called once, this function is in charge of setting the current task's
271  * scheduler parameters to make it an RT task.
272  */
273 static void idle_inject_setup(unsigned int cpu)
274 {
275 	sched_set_fifo(current);
276 }
277 
278 /**
279  * idle_inject_should_run - function helper for the smpboot API
280  * @cpu: CPU the kthread is running on
281  *
282  * Return: whether or not the thread can run.
283  */
284 static int idle_inject_should_run(unsigned int cpu)
285 {
286 	struct idle_inject_thread *iit =
287 		per_cpu_ptr(&idle_inject_thread, cpu);
288 
289 	return iit->should_run;
290 }
291 
292 /**
293  * idle_inject_register - initialize idle injection on a set of CPUs
294  * @cpumask: CPUs to be affected by idle injection
295  *
296  * This function creates an idle injection control device structure for the
297  * given set of CPUs and initializes the timer associated with it.  It does not
298  * start any injection cycles.
299  *
300  * Return: NULL if memory allocation fails, idle injection control device
301  * pointer on success.
302  */
303 struct idle_inject_device *idle_inject_register(struct cpumask *cpumask)
304 {
305 	struct idle_inject_device *ii_dev;
306 	int cpu, cpu_rb;
307 
308 	ii_dev = kzalloc(sizeof(*ii_dev) + cpumask_size(), GFP_KERNEL);
309 	if (!ii_dev)
310 		return NULL;
311 
312 	cpumask_copy(to_cpumask(ii_dev->cpumask), cpumask);
313 	hrtimer_init(&ii_dev->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
314 	ii_dev->timer.function = idle_inject_timer_fn;
315 	ii_dev->latency_us = UINT_MAX;
316 
317 	for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {
318 
319 		if (per_cpu(idle_inject_device, cpu)) {
320 			pr_err("cpu%d is already registered\n", cpu);
321 			goto out_rollback;
322 		}
323 
324 		per_cpu(idle_inject_device, cpu) = ii_dev;
325 	}
326 
327 	return ii_dev;
328 
329 out_rollback:
330 	for_each_cpu(cpu_rb, to_cpumask(ii_dev->cpumask)) {
331 		if (cpu == cpu_rb)
332 			break;
333 		per_cpu(idle_inject_device, cpu_rb) = NULL;
334 	}
335 
336 	kfree(ii_dev);
337 
338 	return NULL;
339 }
340 
341 /**
342  * idle_inject_unregister - unregister idle injection control device
343  * @ii_dev: idle injection control device to unregister
344  *
345  * The function stops idle injection for the given control device,
346  * unregisters its kthreads and frees memory allocated when that device was
347  * created.
348  */
349 void idle_inject_unregister(struct idle_inject_device *ii_dev)
350 {
351 	unsigned int cpu;
352 
353 	idle_inject_stop(ii_dev);
354 
355 	for_each_cpu(cpu, to_cpumask(ii_dev->cpumask))
356 		per_cpu(idle_inject_device, cpu) = NULL;
357 
358 	kfree(ii_dev);
359 }
360 
361 static struct smp_hotplug_thread idle_inject_threads = {
362 	.store = &idle_inject_thread.tsk,
363 	.setup = idle_inject_setup,
364 	.thread_fn = idle_inject_fn,
365 	.thread_comm = "idle_inject/%u",
366 	.thread_should_run = idle_inject_should_run,
367 };
368 
369 static int __init idle_inject_init(void)
370 {
371 	return smpboot_register_percpu_thread(&idle_inject_threads);
372 }
373 early_initcall(idle_inject_init);
374