1 /* 2 * cpuidle.c - core cpuidle infrastructure 3 * 4 * (C) 2006-2007 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com> 5 * Shaohua Li <shaohua.li@intel.com> 6 * Adam Belay <abelay@novell.com> 7 * 8 * This code is licenced under the GPL. 9 */ 10 11 #include <linux/kernel.h> 12 #include <linux/mutex.h> 13 #include <linux/sched.h> 14 #include <linux/notifier.h> 15 #include <linux/pm_qos_params.h> 16 #include <linux/cpu.h> 17 #include <linux/cpuidle.h> 18 #include <linux/ktime.h> 19 20 #include "cpuidle.h" 21 22 DEFINE_PER_CPU(struct cpuidle_device *, cpuidle_devices); 23 24 DEFINE_MUTEX(cpuidle_lock); 25 LIST_HEAD(cpuidle_detected_devices); 26 static void (*pm_idle_old)(void); 27 28 static int enabled_devices; 29 30 #if defined(CONFIG_ARCH_HAS_CPU_IDLE_WAIT) 31 static void cpuidle_kick_cpus(void) 32 { 33 cpu_idle_wait(); 34 } 35 #elif defined(CONFIG_SMP) 36 # error "Arch needs cpu_idle_wait() equivalent here" 37 #else /* !CONFIG_ARCH_HAS_CPU_IDLE_WAIT && !CONFIG_SMP */ 38 static void cpuidle_kick_cpus(void) {} 39 #endif 40 41 static int __cpuidle_register_device(struct cpuidle_device *dev); 42 43 /** 44 * cpuidle_idle_call - the main idle loop 45 * 46 * NOTE: no locks or semaphores should be used here 47 */ 48 static void cpuidle_idle_call(void) 49 { 50 struct cpuidle_device *dev = __get_cpu_var(cpuidle_devices); 51 struct cpuidle_state *target_state; 52 int next_state; 53 54 /* check if the device is ready */ 55 if (!dev || !dev->enabled) { 56 if (pm_idle_old) 57 pm_idle_old(); 58 else 59 local_irq_enable(); 60 return; 61 } 62 63 /* ask the governor for the next state */ 64 next_state = cpuidle_curr_governor->select(dev); 65 if (need_resched()) 66 return; 67 target_state = &dev->states[next_state]; 68 69 /* enter the state and update stats */ 70 dev->last_residency = target_state->enter(dev, target_state); 71 dev->last_state = target_state; 72 target_state->time += (unsigned long long)dev->last_residency; 73 target_state->usage++; 74 75 /* give the governor an opportunity to reflect on the outcome */ 76 if (cpuidle_curr_governor->reflect) 77 cpuidle_curr_governor->reflect(dev); 78 } 79 80 /** 81 * cpuidle_install_idle_handler - installs the cpuidle idle loop handler 82 */ 83 void cpuidle_install_idle_handler(void) 84 { 85 if (enabled_devices && (pm_idle != cpuidle_idle_call)) { 86 /* Make sure all changes finished before we switch to new idle */ 87 smp_wmb(); 88 pm_idle = cpuidle_idle_call; 89 } 90 } 91 92 /** 93 * cpuidle_uninstall_idle_handler - uninstalls the cpuidle idle loop handler 94 */ 95 void cpuidle_uninstall_idle_handler(void) 96 { 97 if (enabled_devices && (pm_idle != pm_idle_old)) { 98 pm_idle = pm_idle_old; 99 cpuidle_kick_cpus(); 100 } 101 } 102 103 /** 104 * cpuidle_pause_and_lock - temporarily disables CPUIDLE 105 */ 106 void cpuidle_pause_and_lock(void) 107 { 108 mutex_lock(&cpuidle_lock); 109 cpuidle_uninstall_idle_handler(); 110 } 111 112 EXPORT_SYMBOL_GPL(cpuidle_pause_and_lock); 113 114 /** 115 * cpuidle_resume_and_unlock - resumes CPUIDLE operation 116 */ 117 void cpuidle_resume_and_unlock(void) 118 { 119 cpuidle_install_idle_handler(); 120 mutex_unlock(&cpuidle_lock); 121 } 122 123 EXPORT_SYMBOL_GPL(cpuidle_resume_and_unlock); 124 125 /** 126 * cpuidle_enable_device - enables idle PM for a CPU 127 * @dev: the CPU 128 * 129 * This function must be called between cpuidle_pause_and_lock and 130 * cpuidle_resume_and_unlock when used externally. 131 */ 132 int cpuidle_enable_device(struct cpuidle_device *dev) 133 { 134 int ret, i; 135 136 if (dev->enabled) 137 return 0; 138 if (!cpuidle_curr_driver || !cpuidle_curr_governor) 139 return -EIO; 140 if (!dev->state_count) 141 return -EINVAL; 142 143 if (dev->registered == 0) { 144 ret = __cpuidle_register_device(dev); 145 if (ret) 146 return ret; 147 } 148 149 if ((ret = cpuidle_add_state_sysfs(dev))) 150 return ret; 151 152 if (cpuidle_curr_governor->enable && 153 (ret = cpuidle_curr_governor->enable(dev))) 154 goto fail_sysfs; 155 156 for (i = 0; i < dev->state_count; i++) { 157 dev->states[i].usage = 0; 158 dev->states[i].time = 0; 159 } 160 dev->last_residency = 0; 161 dev->last_state = NULL; 162 163 smp_wmb(); 164 165 dev->enabled = 1; 166 167 enabled_devices++; 168 return 0; 169 170 fail_sysfs: 171 cpuidle_remove_state_sysfs(dev); 172 173 return ret; 174 } 175 176 EXPORT_SYMBOL_GPL(cpuidle_enable_device); 177 178 /** 179 * cpuidle_disable_device - disables idle PM for a CPU 180 * @dev: the CPU 181 * 182 * This function must be called between cpuidle_pause_and_lock and 183 * cpuidle_resume_and_unlock when used externally. 184 */ 185 void cpuidle_disable_device(struct cpuidle_device *dev) 186 { 187 if (!dev->enabled) 188 return; 189 if (!cpuidle_curr_driver || !cpuidle_curr_governor) 190 return; 191 192 dev->enabled = 0; 193 194 if (cpuidle_curr_governor->disable) 195 cpuidle_curr_governor->disable(dev); 196 197 cpuidle_remove_state_sysfs(dev); 198 enabled_devices--; 199 } 200 201 EXPORT_SYMBOL_GPL(cpuidle_disable_device); 202 203 #ifdef CONFIG_ARCH_HAS_CPU_RELAX 204 static int poll_idle(struct cpuidle_device *dev, struct cpuidle_state *st) 205 { 206 ktime_t t1, t2; 207 s64 diff; 208 int ret; 209 210 t1 = ktime_get(); 211 local_irq_enable(); 212 while (!need_resched()) 213 cpu_relax(); 214 215 t2 = ktime_get(); 216 diff = ktime_to_us(ktime_sub(t2, t1)); 217 if (diff > INT_MAX) 218 diff = INT_MAX; 219 220 ret = (int) diff; 221 return ret; 222 } 223 224 static void poll_idle_init(struct cpuidle_device *dev) 225 { 226 struct cpuidle_state *state = &dev->states[0]; 227 228 cpuidle_set_statedata(state, NULL); 229 230 snprintf(state->name, CPUIDLE_NAME_LEN, "C0"); 231 snprintf(state->desc, CPUIDLE_DESC_LEN, "CPUIDLE CORE POLL IDLE"); 232 state->exit_latency = 0; 233 state->target_residency = 0; 234 state->power_usage = -1; 235 state->flags = CPUIDLE_FLAG_POLL; 236 state->enter = poll_idle; 237 } 238 #else 239 static void poll_idle_init(struct cpuidle_device *dev) {} 240 #endif /* CONFIG_ARCH_HAS_CPU_RELAX */ 241 242 /** 243 * __cpuidle_register_device - internal register function called before register 244 * and enable routines 245 * @dev: the cpu 246 * 247 * cpuidle_lock mutex must be held before this is called 248 */ 249 static int __cpuidle_register_device(struct cpuidle_device *dev) 250 { 251 int ret; 252 struct sys_device *sys_dev = get_cpu_sysdev((unsigned long)dev->cpu); 253 254 if (!sys_dev) 255 return -EINVAL; 256 if (!try_module_get(cpuidle_curr_driver->owner)) 257 return -EINVAL; 258 259 init_completion(&dev->kobj_unregister); 260 261 poll_idle_init(dev); 262 263 per_cpu(cpuidle_devices, dev->cpu) = dev; 264 list_add(&dev->device_list, &cpuidle_detected_devices); 265 if ((ret = cpuidle_add_sysfs(sys_dev))) { 266 module_put(cpuidle_curr_driver->owner); 267 return ret; 268 } 269 270 dev->registered = 1; 271 return 0; 272 } 273 274 /** 275 * cpuidle_register_device - registers a CPU's idle PM feature 276 * @dev: the cpu 277 */ 278 int cpuidle_register_device(struct cpuidle_device *dev) 279 { 280 int ret; 281 282 mutex_lock(&cpuidle_lock); 283 284 if ((ret = __cpuidle_register_device(dev))) { 285 mutex_unlock(&cpuidle_lock); 286 return ret; 287 } 288 289 cpuidle_enable_device(dev); 290 cpuidle_install_idle_handler(); 291 292 mutex_unlock(&cpuidle_lock); 293 294 return 0; 295 296 } 297 298 EXPORT_SYMBOL_GPL(cpuidle_register_device); 299 300 /** 301 * cpuidle_unregister_device - unregisters a CPU's idle PM feature 302 * @dev: the cpu 303 */ 304 void cpuidle_unregister_device(struct cpuidle_device *dev) 305 { 306 struct sys_device *sys_dev = get_cpu_sysdev((unsigned long)dev->cpu); 307 308 if (dev->registered == 0) 309 return; 310 311 cpuidle_pause_and_lock(); 312 313 cpuidle_disable_device(dev); 314 315 cpuidle_remove_sysfs(sys_dev); 316 list_del(&dev->device_list); 317 wait_for_completion(&dev->kobj_unregister); 318 per_cpu(cpuidle_devices, dev->cpu) = NULL; 319 320 cpuidle_resume_and_unlock(); 321 322 module_put(cpuidle_curr_driver->owner); 323 } 324 325 EXPORT_SYMBOL_GPL(cpuidle_unregister_device); 326 327 #ifdef CONFIG_SMP 328 329 static void smp_callback(void *v) 330 { 331 /* we already woke the CPU up, nothing more to do */ 332 } 333 334 /* 335 * This function gets called when a part of the kernel has a new latency 336 * requirement. This means we need to get all processors out of their C-state, 337 * and then recalculate a new suitable C-state. Just do a cross-cpu IPI; that 338 * wakes them all right up. 339 */ 340 static int cpuidle_latency_notify(struct notifier_block *b, 341 unsigned long l, void *v) 342 { 343 smp_call_function(smp_callback, NULL, 1); 344 return NOTIFY_OK; 345 } 346 347 static struct notifier_block cpuidle_latency_notifier = { 348 .notifier_call = cpuidle_latency_notify, 349 }; 350 351 static inline void latency_notifier_init(struct notifier_block *n) 352 { 353 pm_qos_add_notifier(PM_QOS_CPU_DMA_LATENCY, n); 354 } 355 356 #else /* CONFIG_SMP */ 357 358 #define latency_notifier_init(x) do { } while (0) 359 360 #endif /* CONFIG_SMP */ 361 362 /** 363 * cpuidle_init - core initializer 364 */ 365 static int __init cpuidle_init(void) 366 { 367 int ret; 368 369 pm_idle_old = pm_idle; 370 371 ret = cpuidle_add_class_sysfs(&cpu_sysdev_class); 372 if (ret) 373 return ret; 374 375 latency_notifier_init(&cpuidle_latency_notifier); 376 377 return 0; 378 } 379 380 core_initcall(cpuidle_init); 381