drivers/powercap/idle_inject.c

1 // SPDX-License-Identifier: GPL-2.0
7  * The idle injection framework provides a way to force CPUs to enter idle
10  * It relies on the smpboot kthreads feature providing common code for CPU
13  * All of the kthreads used for idle injection are created at init time.
15  * Next, the users of the idle injection framework provide a cpumask via
19  * The idle + run duration is specified via separate helpers and that allows
20  * idle injection to be started.
22  * The idle injection kthreads will call play_idle_precise() with the idle
25  * After all of them have been woken up, a timer is set to start the next idle
28  * The timer interrupt handler will wake up the idle injection kthreads for
31  * Idle injection is stopped synchronously and no leftover idle injection
34  * It is up to the user of this framework to provide a lock for higher-level
35  * synchronization to prevent race conditions like starting idle injection
51  * struct idle_inject_thread - task on/off switch structure
52  * @tsk: task injecting the idle cycles
61  * struct idle_inject_device - idle injection data
62  * @timer: idle injection period timer
63  * @idle_duration_us: duration of CPU idle time to inject
66  * @update: Optional callback deciding whether or not to skip idle
68  * @cpumask: mask of CPUs affected by idle injection
70  * This structure is used to define per instance idle inject device data. Each
71  * instance has an idle duration, a run duration and mask of CPUs to inject
72  * idle.
74  * Actual CPU idle time is injected by calling kernel scheduler interface
78  * update() - This callback is invoked just before waking up CPUs to inject
79  * idle. If it returns false, CPUs are not woken up to inject idle in the given
80  * cycle. It also allows the caller to readjust the idle and run duration by
96  * idle_inject_wakeup - Wake up idle injection threads
97  * @ii_dev: target idle injection device
99  * Every idle injection task associated with the given idle injection device
100  * and running on an online CPU will be woken up.
107 	for_each_cpu_and(cpu, to_cpumask(ii_dev->cpumask), cpu_online_mask) {  in idle_inject_wakeup()
109 		iit->should_run = 1;  in idle_inject_wakeup()
110 		wake_up_process(iit->tsk);  in idle_inject_wakeup()
115  * idle_inject_timer_fn - idle injection timer function
116  * @timer: idle injection hrtimer
118  * This function is called when the idle injection timer expires.  It wakes up
119  * idle injection tasks associated with the timer and they, in turn, invoke
120  * play_idle_precise() to inject a specified amount of CPU idle time.
130 	if (!ii_dev->update || (ii_dev->update && ii_dev->update()))  in idle_inject_timer_fn()
133 	duration_us = READ_ONCE(ii_dev->run_duration_us);  in idle_inject_timer_fn()
134 	duration_us += READ_ONCE(ii_dev->idle_duration_us);  in idle_inject_timer_fn()
142  * idle_inject_fn - idle injection work function
146  * idle time.
159 	iit->should_run = 0;  in idle_inject_fn()
161 	play_idle_precise(READ_ONCE(ii_dev->idle_duration_us) * NSEC_PER_USEC,  in idle_inject_fn()
162 			  READ_ONCE(ii_dev->latency_us) * NSEC_PER_USEC);  in idle_inject_fn()
166  * idle_inject_set_duration - idle and run duration update helper
167  * @ii_dev: idle injection control device structure
169  * @idle_duration_us: CPU idle time to inject in microseconds
176 		WRITE_ONCE(ii_dev->run_duration_us, run_duration_us);  in idle_inject_set_duration()
177 		WRITE_ONCE(ii_dev->idle_duration_us, idle_duration_us);  in idle_inject_set_duration()
180 		pr_debug("CPU is forced to 100 percent idle\n");  in idle_inject_set_duration()
185  * idle_inject_get_duration - idle and run duration retrieval helper
186  * @ii_dev: idle injection control device structure
188  * @idle_duration_us: memory location to store the current CPU idle time
194 	*run_duration_us = READ_ONCE(ii_dev->run_duration_us);  in idle_inject_get_duration()
195 	*idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);  in idle_inject_get_duration()
200  * idle_inject_set_latency - set the maximum latency allowed
201  * @ii_dev: idle injection control device structure
202  * @latency_us: set the latency requirement for the idle state
207 	WRITE_ONCE(ii_dev->latency_us, latency_us);  in idle_inject_set_latency()
212  * idle_inject_start - start idle injections
213  * @ii_dev: idle injection control device structure
215  * The function starts idle injection by first waking up all of the idle
216  * injection kthreads associated with @ii_dev to let them inject CPU idle time
217  * sets up a timer to start the next idle injection period.
219  * Return: -EINVAL if the CPU idle or CPU run time is not set or 0 on success.
223 	unsigned int idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);  in idle_inject_start()
224 	unsigned int run_duration_us = READ_ONCE(ii_dev->run_duration_us);  in idle_inject_start()
227 		return -EINVAL;  in idle_inject_start()
229 	pr_debug("Starting injecting idle cycles on CPUs '%*pbl'\n",  in idle_inject_start()
230 		 cpumask_pr_args(to_cpumask(ii_dev->cpumask)));  in idle_inject_start()
234 	hrtimer_start(&ii_dev->timer,  in idle_inject_start()
244  * idle_inject_stop - stops idle injections
245  * @ii_dev: idle injection control device structure
247  * The function stops idle injection and waits for the threads to finish work.
248  * If CPU idle time is being injected when this function runs, then it will
251  * When it returns, there is no more idle injection kthread activity.  The
259 	pr_debug("Stopping idle injection on CPUs '%*pbl'\n",  in idle_inject_stop()
260 		 cpumask_pr_args(to_cpumask(ii_dev->cpumask)));  in idle_inject_stop()
262 	hrtimer_cancel(&ii_dev->timer);  in idle_inject_stop()
265 	 * Stopping idle injection requires all of the idle injection kthreads  in idle_inject_stop()
275 	 * goes offline with the should_run flag set so as to prevent its idle  in idle_inject_stop()
279 	for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {  in idle_inject_stop()
281 		iit->should_run = 0;  in idle_inject_stop()
283 		wait_task_inactive(iit->tsk, TASK_ANY);  in idle_inject_stop()
291  * idle_inject_setup - prepare the current task for idle injection
303  * idle_inject_should_run - function helper for the smpboot API
304  * @cpu: CPU the kthread is running on
313 	return iit->should_run;  in idle_inject_should_run()
317  * idle_inject_register_full - initialize idle injection on a set of CPUs
318  * @cpumask: CPUs to be affected by idle injection
320  * idle
322  * This function creates an idle injection control device structure for the
327  * Return: NULL if memory allocation fails, idle injection control device
328  * pointer on success.
341 	cpumask_copy(to_cpumask(ii_dev->cpumask), cpumask);  in idle_inject_register_full()
342 	hrtimer_init(&ii_dev->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);  in idle_inject_register_full()
343 	ii_dev->timer.function = idle_inject_timer_fn;  in idle_inject_register_full()
344 	ii_dev->latency_us = UINT_MAX;  in idle_inject_register_full()
345 	ii_dev->update = update;  in idle_inject_register_full()
347 	for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {  in idle_inject_register_full()
360 	for_each_cpu(cpu_rb, to_cpumask(ii_dev->cpumask)) {  in idle_inject_register_full()
373  * idle_inject_register - initialize idle injection on a set of CPUs
374  * @cpumask: CPUs to be affected by idle injection
376  * This function creates an idle injection control device structure for the
380  * Return: NULL if memory allocation fails, idle injection control device
381  * pointer on success.
390  * idle_inject_unregister - unregister idle injection control device
391  * @ii_dev: idle injection control device to unregister
393  * The function stops idle injection for the given control device,
403 	for_each_cpu(cpu, to_cpumask(ii_dev->cpumask))  in idle_inject_unregister()