xref: /openbmc/linux/drivers/cpuidle/cpuidle.c (revision b627b4ed) 
1 /*
2  * cpuidle.c - core cpuidle infrastructure
3  *
4  * (C) 2006-2007 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
5  *               Shaohua Li <shaohua.li@intel.com>
6  *               Adam Belay <abelay@novell.com>
7  *
8  * This code is licenced under the GPL.
9  */
10 
11 #include <linux/kernel.h>
12 #include <linux/mutex.h>
13 #include <linux/sched.h>
14 #include <linux/notifier.h>
15 #include <linux/pm_qos_params.h>
16 #include <linux/cpu.h>
17 #include <linux/cpuidle.h>
18 #include <linux/ktime.h>
19 #include <linux/hrtimer.h>
20 
21 #include "cpuidle.h"
22 
23 DEFINE_PER_CPU(struct cpuidle_device *, cpuidle_devices);
24 
25 DEFINE_MUTEX(cpuidle_lock);
26 LIST_HEAD(cpuidle_detected_devices);
27 static void (*pm_idle_old)(void);
28 
29 static int enabled_devices;
30 
31 #if defined(CONFIG_ARCH_HAS_CPU_IDLE_WAIT)
32 static void cpuidle_kick_cpus(void)
33 {
34 	cpu_idle_wait();
35 }
36 #elif defined(CONFIG_SMP)
37 # error "Arch needs cpu_idle_wait() equivalent here"
38 #else /* !CONFIG_ARCH_HAS_CPU_IDLE_WAIT && !CONFIG_SMP */
39 static void cpuidle_kick_cpus(void) {}
40 #endif
41 
42 static int __cpuidle_register_device(struct cpuidle_device *dev);
43 
44 /**
45  * cpuidle_idle_call - the main idle loop
46  *
47  * NOTE: no locks or semaphores should be used here
48  */
49 static void cpuidle_idle_call(void)
50 {
51 	struct cpuidle_device *dev = __get_cpu_var(cpuidle_devices);
52 	struct cpuidle_state *target_state;
53 	int next_state;
54 
55 	/* check if the device is ready */
56 	if (!dev || !dev->enabled) {
57 		if (pm_idle_old)
58 			pm_idle_old();
59 		else
60 #if defined(CONFIG_ARCH_HAS_DEFAULT_IDLE)
61 			default_idle();
62 #else
63 			local_irq_enable();
64 #endif
65 		return;
66 	}
67 
68 #if 0
69 	/* shows regressions, re-enable for 2.6.29 */
70 	/*
71 	 * run any timers that can be run now, at this point
72 	 * before calculating the idle duration etc.
73 	 */
74 	hrtimer_peek_ahead_timers();
75 #endif
76 	/* ask the governor for the next state */
77 	next_state = cpuidle_curr_governor->select(dev);
78 	if (need_resched())
79 		return;
80 	target_state = &dev->states[next_state];
81 
82 	/* enter the state and update stats */
83 	dev->last_state = target_state;
84 	dev->last_residency = target_state->enter(dev, target_state);
85 	if (dev->last_state)
86 		target_state = dev->last_state;
87 
88 	target_state->time += (unsigned long long)dev->last_residency;
89 	target_state->usage++;
90 
91 	/* give the governor an opportunity to reflect on the outcome */
92 	if (cpuidle_curr_governor->reflect)
93 		cpuidle_curr_governor->reflect(dev);
94 }
95 
96 /**
97  * cpuidle_install_idle_handler - installs the cpuidle idle loop handler
98  */
99 void cpuidle_install_idle_handler(void)
100 {
101 	if (enabled_devices && (pm_idle != cpuidle_idle_call)) {
102 		/* Make sure all changes finished before we switch to new idle */
103 		smp_wmb();
104 		pm_idle = cpuidle_idle_call;
105 	}
106 }
107 
108 /**
109  * cpuidle_uninstall_idle_handler - uninstalls the cpuidle idle loop handler
110  */
111 void cpuidle_uninstall_idle_handler(void)
112 {
113 	if (enabled_devices && pm_idle_old && (pm_idle != pm_idle_old)) {
114 		pm_idle = pm_idle_old;
115 		cpuidle_kick_cpus();
116 	}
117 }
118 
119 /**
120  * cpuidle_pause_and_lock - temporarily disables CPUIDLE
121  */
122 void cpuidle_pause_and_lock(void)
123 {
124 	mutex_lock(&cpuidle_lock);
125 	cpuidle_uninstall_idle_handler();
126 }
127 
128 EXPORT_SYMBOL_GPL(cpuidle_pause_and_lock);
129 
130 /**
131  * cpuidle_resume_and_unlock - resumes CPUIDLE operation
132  */
133 void cpuidle_resume_and_unlock(void)
134 {
135 	cpuidle_install_idle_handler();
136 	mutex_unlock(&cpuidle_lock);
137 }
138 
139 EXPORT_SYMBOL_GPL(cpuidle_resume_and_unlock);
140 
141 /**
142  * cpuidle_enable_device - enables idle PM for a CPU
143  * @dev: the CPU
144  *
145  * This function must be called between cpuidle_pause_and_lock and
146  * cpuidle_resume_and_unlock when used externally.
147  */
148 int cpuidle_enable_device(struct cpuidle_device *dev)
149 {
150 	int ret, i;
151 
152 	if (dev->enabled)
153 		return 0;
154 	if (!cpuidle_curr_driver || !cpuidle_curr_governor)
155 		return -EIO;
156 	if (!dev->state_count)
157 		return -EINVAL;
158 
159 	if (dev->registered == 0) {
160 		ret = __cpuidle_register_device(dev);
161 		if (ret)
162 			return ret;
163 	}
164 
165 	if ((ret = cpuidle_add_state_sysfs(dev)))
166 		return ret;
167 
168 	if (cpuidle_curr_governor->enable &&
169 	    (ret = cpuidle_curr_governor->enable(dev)))
170 		goto fail_sysfs;
171 
172 	for (i = 0; i < dev->state_count; i++) {
173 		dev->states[i].usage = 0;
174 		dev->states[i].time = 0;
175 	}
176 	dev->last_residency = 0;
177 	dev->last_state = NULL;
178 
179 	smp_wmb();
180 
181 	dev->enabled = 1;
182 
183 	enabled_devices++;
184 	return 0;
185 
186 fail_sysfs:
187 	cpuidle_remove_state_sysfs(dev);
188 
189 	return ret;
190 }
191 
192 EXPORT_SYMBOL_GPL(cpuidle_enable_device);
193 
194 /**
195  * cpuidle_disable_device - disables idle PM for a CPU
196  * @dev: the CPU
197  *
198  * This function must be called between cpuidle_pause_and_lock and
199  * cpuidle_resume_and_unlock when used externally.
200  */
201 void cpuidle_disable_device(struct cpuidle_device *dev)
202 {
203 	if (!dev->enabled)
204 		return;
205 	if (!cpuidle_curr_driver || !cpuidle_curr_governor)
206 		return;
207 
208 	dev->enabled = 0;
209 
210 	if (cpuidle_curr_governor->disable)
211 		cpuidle_curr_governor->disable(dev);
212 
213 	cpuidle_remove_state_sysfs(dev);
214 	enabled_devices--;
215 }
216 
217 EXPORT_SYMBOL_GPL(cpuidle_disable_device);
218 
219 #ifdef CONFIG_ARCH_HAS_CPU_RELAX
220 static int poll_idle(struct cpuidle_device *dev, struct cpuidle_state *st)
221 {
222 	ktime_t	t1, t2;
223 	s64 diff;
224 	int ret;
225 
226 	t1 = ktime_get();
227 	local_irq_enable();
228 	while (!need_resched())
229 		cpu_relax();
230 
231 	t2 = ktime_get();
232 	diff = ktime_to_us(ktime_sub(t2, t1));
233 	if (diff > INT_MAX)
234 		diff = INT_MAX;
235 
236 	ret = (int) diff;
237 	return ret;
238 }
239 
240 static void poll_idle_init(struct cpuidle_device *dev)
241 {
242 	struct cpuidle_state *state = &dev->states[0];
243 
244 	cpuidle_set_statedata(state, NULL);
245 
246 	snprintf(state->name, CPUIDLE_NAME_LEN, "C0");
247 	snprintf(state->desc, CPUIDLE_DESC_LEN, "CPUIDLE CORE POLL IDLE");
248 	state->exit_latency = 0;
249 	state->target_residency = 0;
250 	state->power_usage = -1;
251 	state->flags = CPUIDLE_FLAG_POLL;
252 	state->enter = poll_idle;
253 }
254 #else
255 static void poll_idle_init(struct cpuidle_device *dev) {}
256 #endif /* CONFIG_ARCH_HAS_CPU_RELAX */
257 
258 /**
259  * __cpuidle_register_device - internal register function called before register
260  * and enable routines
261  * @dev: the cpu
262  *
263  * cpuidle_lock mutex must be held before this is called
264  */
265 static int __cpuidle_register_device(struct cpuidle_device *dev)
266 {
267 	int ret;
268 	struct sys_device *sys_dev = get_cpu_sysdev((unsigned long)dev->cpu);
269 
270 	if (!sys_dev)
271 		return -EINVAL;
272 	if (!try_module_get(cpuidle_curr_driver->owner))
273 		return -EINVAL;
274 
275 	init_completion(&dev->kobj_unregister);
276 
277 	poll_idle_init(dev);
278 
279 	per_cpu(cpuidle_devices, dev->cpu) = dev;
280 	list_add(&dev->device_list, &cpuidle_detected_devices);
281 	if ((ret = cpuidle_add_sysfs(sys_dev))) {
282 		module_put(cpuidle_curr_driver->owner);
283 		return ret;
284 	}
285 
286 	dev->registered = 1;
287 	return 0;
288 }
289 
290 /**
291  * cpuidle_register_device - registers a CPU's idle PM feature
292  * @dev: the cpu
293  */
294 int cpuidle_register_device(struct cpuidle_device *dev)
295 {
296 	int ret;
297 
298 	mutex_lock(&cpuidle_lock);
299 
300 	if ((ret = __cpuidle_register_device(dev))) {
301 		mutex_unlock(&cpuidle_lock);
302 		return ret;
303 	}
304 
305 	cpuidle_enable_device(dev);
306 	cpuidle_install_idle_handler();
307 
308 	mutex_unlock(&cpuidle_lock);
309 
310 	return 0;
311 
312 }
313 
314 EXPORT_SYMBOL_GPL(cpuidle_register_device);
315 
316 /**
317  * cpuidle_unregister_device - unregisters a CPU's idle PM feature
318  * @dev: the cpu
319  */
320 void cpuidle_unregister_device(struct cpuidle_device *dev)
321 {
322 	struct sys_device *sys_dev = get_cpu_sysdev((unsigned long)dev->cpu);
323 
324 	if (dev->registered == 0)
325 		return;
326 
327 	cpuidle_pause_and_lock();
328 
329 	cpuidle_disable_device(dev);
330 
331 	cpuidle_remove_sysfs(sys_dev);
332 	list_del(&dev->device_list);
333 	wait_for_completion(&dev->kobj_unregister);
334 	per_cpu(cpuidle_devices, dev->cpu) = NULL;
335 
336 	cpuidle_resume_and_unlock();
337 
338 	module_put(cpuidle_curr_driver->owner);
339 }
340 
341 EXPORT_SYMBOL_GPL(cpuidle_unregister_device);
342 
343 #ifdef CONFIG_SMP
344 
345 static void smp_callback(void *v)
346 {
347 	/* we already woke the CPU up, nothing more to do */
348 }
349 
350 /*
351  * This function gets called when a part of the kernel has a new latency
352  * requirement.  This means we need to get all processors out of their C-state,
353  * and then recalculate a new suitable C-state. Just do a cross-cpu IPI; that
354  * wakes them all right up.
355  */
356 static int cpuidle_latency_notify(struct notifier_block *b,
357 		unsigned long l, void *v)
358 {
359 	smp_call_function(smp_callback, NULL, 1);
360 	return NOTIFY_OK;
361 }
362 
363 static struct notifier_block cpuidle_latency_notifier = {
364 	.notifier_call = cpuidle_latency_notify,
365 };
366 
367 static inline void latency_notifier_init(struct notifier_block *n)
368 {
369 	pm_qos_add_notifier(PM_QOS_CPU_DMA_LATENCY, n);
370 }
371 
372 #else /* CONFIG_SMP */
373 
374 #define latency_notifier_init(x) do { } while (0)
375 
376 #endif /* CONFIG_SMP */
377 
378 /**
379  * cpuidle_init - core initializer
380  */
381 static int __init cpuidle_init(void)
382 {
383 	int ret;
384 
385 	pm_idle_old = pm_idle;
386 
387 	ret = cpuidle_add_class_sysfs(&cpu_sysdev_class);
388 	if (ret)
389 		return ret;
390 
391 	latency_notifier_init(&cpuidle_latency_notifier);
392 
393 	return 0;
394 }
395 
396 core_initcall(cpuidle_init);
397